Merge branch 'master' into modfqmmgr-389

folio-org · Jul 25, 2024 · 3d81181 · 3d81181
2 parents d7763d5 + ae0c2f6
commit 3d81181
Show file tree

Hide file tree

Showing 39 changed files with 2,294 additions and 1,827 deletions.
diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json
@@ -334,6 +334,12 @@
       "version": "1.1"
     }
   ],
+  "optional": [
+    {
+      "id": "consortia",
+      "version": "1.0"
+    }
+  ],
   "launchDescriptor": {
     "dockerImage": "@artifactId@:@version@",
     "dockerPull": false,

diff --git a/src/main/java/org/folio/fqm/repository/IdStreamer.java b/src/main/java/org/folio/fqm/repository/IdStreamer.java
@@ -10,7 +10,6 @@
 import org.folio.fql.model.Fql;
 import org.folio.querytool.domain.dto.EntityType;
 import org.folio.querytool.domain.dto.EntityTypeDefaultSort;
-import org.folio.querytool.domain.dto.EntityTypeSource;
 import org.jooq.Condition;
 import org.jooq.Cursor;
 import org.jooq.DSLContext;
@@ -82,12 +81,6 @@ private int streamIdsInBatch(EntityType entityType,
                                Consumer<IdsWithCancelCallback> idsConsumer) {
     String finalJoinClause = entityTypeFlatteningService.getJoinClause(entityType);
     Field<String[]> idValueGetter = IdColumnUtils.getResultIdValueGetter(entityType);
-    String finalWhereClause = sqlWhereClause.toString();
-    for (EntityTypeSource source : entityType.getSources()) {
-      String toReplace = ":" + source.getAlias();
-      String alias = "\"" + source.getAlias() + "\"";
-      finalWhereClause = finalWhereClause.replace(toReplace, alias);
-    }
     ResultQuery<Record1<String[]>> query = null;
     if (!isEmpty(entityType.getGroupByFields())) {
       Field<?>[] groupByFields = entityType
@@ -100,15 +93,15 @@ private int streamIdsInBatch(EntityType entityType,
       query = jooqContext.dsl()
         .select(field(idValueGetter))
         .from(finalJoinClause)
-        .where(finalWhereClause)
+        .where(sqlWhereClause)
         .groupBy(groupByFields)
         .orderBy(getSortFields(entityType, sortResults))
         .fetchSize(batchSize);
     } else {
       query = jooqContext.dsl()
         .select(field(idValueGetter))
         .from(finalJoinClause)
-        .where(finalWhereClause)
+        .where(sqlWhereClause)
         .orderBy(getSortFields(entityType, sortResults))
         .fetchSize(batchSize);
     }

diff --git a/src/main/java/org/folio/fqm/repository/ResultSetRepository.java b/src/main/java/org/folio/fqm/repository/ResultSetRepository.java
@@ -13,7 +13,6 @@
 
 import org.folio.fql.model.Fql;
 import org.folio.fqm.exception.FieldNotFoundException;
-import org.folio.fqm.exception.EntityTypeNotFoundException;
 import org.folio.fqm.service.EntityTypeFlatteningService;
 import org.folio.fqm.service.FqlToSqlConverterService;
 import org.folio.fqm.utils.IdColumnUtils;
@@ -23,7 +22,6 @@
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.folio.querytool.domain.dto.EntityTypeColumn;
-import org.folio.querytool.domain.dto.EntityTypeSource;
 import org.jooq.Condition;
 import org.jooq.DSLContext;
 import org.jooq.Field;
@@ -124,13 +122,6 @@ public List<Map<String, Object>> getResultSet(UUID entityTypeId, Fql fql, List<S
     }
 
     Condition condition = FqlToSqlConverterService.getSqlCondition(fql.fqlCondition(), entityType);
-    // TODO: might want to put next 5 lines in its own method in EntityTypeFlatteningService
-    String finalWhereClause = condition.toString();
-    for (EntityTypeSource source : entityType.getSources()) {
-      String toReplace = ":" + source.getAlias();
-      String alias = "\"" + source.getAlias() + "\"";
-      finalWhereClause = finalWhereClause.replace(toReplace, alias);
-    }
     var fieldsToSelect = getSqlFields(entityType, fields);
     String idColumnName = entityType
       .getColumns()
@@ -143,7 +134,7 @@ public List<Map<String, Object>> getResultSet(UUID entityTypeId, Fql fql, List<S
     String fromClause = entityTypeFlatteningService.getJoinClause(entityType);
     var result = jooqContext.select(fieldsToSelect)
       .from(fromClause)
-      .where(finalWhereClause)
+      .where(condition)
       .and(afterIdCondition)
       .orderBy(sortCriteria)
       .limit(limit)

diff --git a/src/main/java/org/folio/fqm/resource/MigrationController.java b/src/main/java/org/folio/fqm/resource/MigrationController.java
@@ -0,0 +1,18 @@
+package org.folio.fqm.resource;
+
+import lombok.RequiredArgsConstructor;
+import org.folio.fqm.service.MigrationService;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequiredArgsConstructor
+public class MigrationController implements FqmVersionApi {
+  private final MigrationService migrationService;
+
+  @Override
+  public ResponseEntity<String> getFqmVersion() {
+    return new ResponseEntity<>(migrationService.getLatestVersion(), HttpStatus.OK);
+  }
+}
diff --git a/src/main/java/org/folio/fqm/service/EntityTypeFlatteningService.java b/src/main/java/org/folio/fqm/service/EntityTypeFlatteningService.java
@@ -307,6 +307,7 @@ private List<EntityTypeColumn> getFilteredColumns(List<EntityTypeColumn> unfilte
     return unfilteredColumns
       .stream()
       .filter(column -> ecsEnabled || !Boolean.TRUE.equals(column.getEcsOnly()))
+      .map(column -> column.getValues() == null ? column : column.values(column.getValues().stream().distinct().toList()))
       .toList();
   }
 

diff --git a/src/main/java/org/folio/fqm/service/EntityTypeInitializationService.java b/src/main/java/org/folio/fqm/service/EntityTypeInitializationService.java
@@ -8,8 +8,14 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Stream;
+
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
+import feign.FeignException;
 import lombok.extern.log4j.Log4j2;
+import org.folio.fqm.client.SimpleHttpClient;
 import org.folio.fqm.repository.EntityTypeRepository;
 import org.folio.querytool.domain.dto.EntityType;
 import org.folio.spring.FolioExecutionContext;
@@ -28,16 +34,19 @@ public class EntityTypeInitializationService {
 
   private final ObjectMapper objectMapper;
   private final ResourcePatternResolver resourceResolver;
+  private final SimpleHttpClient ecsClient;
 
   @Autowired
   public EntityTypeInitializationService(
     EntityTypeRepository entityTypeRepository,
     FolioExecutionContext folioExecutionContext,
-    ResourcePatternResolver resourceResolver
+    ResourcePatternResolver resourceResolver,
+    SimpleHttpClient ecsClient
   ) {
     this.entityTypeRepository = entityTypeRepository;
     this.folioExecutionContext = folioExecutionContext;
     this.resourceResolver = resourceResolver;
+    this.ecsClient = ecsClient;
 
     // this enables all JSON5 features, except for numeric ones (hex, starting/trailing
     // decimal points, use of NaN, etc), as those are not relevant for our use
@@ -62,7 +71,16 @@ public EntityTypeInitializationService(
   // called as part of tenant install/upgrade (see FqmTenantService)
   public void initializeEntityTypes() throws IOException {
     log.info("Initializing entity types");
-
+    String centralTenantId = "${central_tenant_id}";
+    try {
+      String rawJson = ecsClient.get("consortia-configuration", Map.of("limit", String.valueOf(100)));
+      DocumentContext parsedJson = JsonPath.parse(rawJson);
+      centralTenantId = parsedJson.read("centralTenantId");
+      log.info("ECS central tenant ID: {}", centralTenantId);
+    } catch (FeignException.NotFound | IllegalArgumentException e) {
+      log.info("ECS is not enabled for tenant {}", folioExecutionContext.getTenantId());
+    }
+    String finalCentralTenantId = centralTenantId;
     List<EntityType> desiredEntityTypes = Stream
       .concat(
         Arrays.stream(resourceResolver.getResources("classpath:/entity-types/**/*.json")),
@@ -74,7 +92,8 @@ public void initializeEntityTypes() throws IOException {
           return objectMapper.readValue(
             resource
               .getContentAsString(StandardCharsets.UTF_8)
-              .replace("${tenant_id}", folioExecutionContext.getTenantId()),
+              .replace("${tenant_id}", folioExecutionContext.getTenantId())
+              .replace("${central_tenant_id}", finalCentralTenantId),
             EntityType.class
           );
         } catch (IOException e) {

diff --git a/src/main/java/org/folio/fqm/service/EntityTypeService.java b/src/main/java/org/folio/fqm/service/EntityTypeService.java
@@ -129,6 +129,7 @@ private ColumnValues getFieldValuesFromEntityTypeDefinition(Field field, String
       .getValues()
       .stream()
       .filter(valueWithLabel -> valueWithLabel.getLabel().contains(searchText))
+      .distinct()
       .sorted(Comparator.comparing(ValueWithLabel::getLabel, String.CASE_INSENSITIVE_ORDER))
       .toList();
     return new ColumnValues().content(filteredValues);

diff --git a/src/main/resources/entity-types/circulation/simple_loan.json5 b/src/main/resources/entity-types/circulation/simple_loan.json5
@@ -368,5 +368,16 @@
       visibleByDefault: false,
       valueGetter: ":sourceAlias.jsonb->'reminders'->'lastFeeBilled'->>'date'",
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'loan',
+      dataType: {
+        dataType: 'stringType',
+      },
+      isIdColumn: false,
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
 }
diff --git a/src/main/resources/entity-types/circulation/simple_loan_policy.json5 b/src/main/resources/entity-types/circulation/simple_loan_policy.json5
@@ -869,5 +869,16 @@
         },
       ],
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'lp',
+      dataType: {
+        dataType: 'stringType',
+      },
+      isIdColumn: false,
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
 }
diff --git a/src/main/resources/entity-types/inventory/simple_call_number_type.json5 b/src/main/resources/entity-types/inventory/simple_call_number_type.json5
@@ -118,6 +118,16 @@
       visibleByDefault: false,
       valueGetter: ":sourceAlias.jsonb->'metadata'->>'updatedByUsername'",
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'call_number_type',
+      dataType: {
+        dataType: 'stringType',
+      },
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
   defaultSort: [
     {

diff --git a/src/main/resources/entity-types/inventory/simple_holdings_records.json5 b/src/main/resources/entity-types/inventory/simple_holdings_records.json5
@@ -825,5 +825,16 @@
       visibleByDefault: false,
       valueGetter: ":sourceAlias.jsonb->>'notes'",
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'hrd',
+      dataType: {
+        dataType: 'stringType',
+      },
+      isIdColumn: false,
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
 }
diff --git a/src/main/resources/entity-types/inventory/simple_instance.json5 b/src/main/resources/entity-types/inventory/simple_instance.json5
@@ -1047,7 +1047,7 @@
       queryable: true,
       visibleByDefault: true,
       valueGetter: "CASE \
-                      WHEN (SELECT central_tenant_id from ${tenant_id}_mod_consortia.consortia_configuration) = '${tenant_id}' \
+                      WHEN '${central_tenant_id}' = '${tenant_id}' \
                         THEN 'Shared'\
                       ELSE \
                         CASE \
@@ -1056,7 +1056,17 @@
                           ELSE 'Local' \
                         END \
                     END",
-      ecsOnly: true
+      values: [
+        {
+          value: 'Shared',
+          label: 'Shared',
+        },
+        {
+          value: 'Local',
+          label: 'Local',
+        },
+      ],
+      ecsOnly: true,
     },
     {
       name: 'source_tenant_id',
@@ -1068,16 +1078,37 @@
       queryable: true,
       visibleByDefault: true,
       valueGetter: "CASE \
-                      WHEN (SELECT central_tenant_id from ${tenant_id}_mod_consortia.consortia_configuration) = '${tenant_id}' \
+                      WHEN '${central_tenant_id}' = '${tenant_id}' \
                         THEN '${tenant_id}' \
                       ELSE \
                         CASE \
                           WHEN \"left\"(lower(:sourceAlias.jsonb ->> 'source'::text), 600) = 'consortium-folio' OR \"left\"(lower(:sourceAlias.jsonb ->> 'source'::text), 600) = 'consortium-marc' \
-                            THEN (SELECT central_tenant_id FROM ${tenant_id}_mod_consortia.consortia_configuration) \
+                            THEN '${central_tenant_id}' \
                           ELSE '${tenant_id}' \
                         END \
                     END",
-      ecsOnly: true
+      values: [
+        {
+          value: '${tenant_id}',
+          label: '${tenant_id}',
+        },
+        {
+          value: '${central_tenant_id}',
+          label: '${central_tenant_id}',
+        },
+      ],
+      ecsOnly: true,
+    },
+    {
+      name: 'jsonb',
+      sourceAlias: 'inst',
+      dataType: {
+        dataType: 'stringType',
+      },
+      isIdColumn: false,
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
     }
   ],
 }
diff --git a/src/main/resources/entity-types/inventory/simple_instance_status.json5 b/src/main/resources/entity-types/inventory/simple_instance_status.json5
@@ -122,6 +122,16 @@
       visibleByDefault: false,
       valueGetter: ":sourceAlias.jsonb->'metadata'->>'updatedByUsername'",
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'instance_status',
+      dataType: {
+        dataType: 'stringType',
+      },
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
   defaultSort: [
     {

diff --git a/src/main/resources/entity-types/inventory/simple_instance_type.json5 b/src/main/resources/entity-types/inventory/simple_instance_type.json5
@@ -128,6 +128,17 @@
       visibleByDefault: false,
       valueGetter: ":sourceAlias.jsonb->'metadata'->>'updatedByUserId'",
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'instance_type',
+      dataType: {
+        dataType: 'stringType',
+      },
+      isIdColumn: false,
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
   defaultSort: [
     {

diff --git a/src/main/resources/entity-types/inventory/simple_item_details.json5 b/src/main/resources/entity-types/inventory/simple_item_details.json5
@@ -961,5 +961,16 @@
       visibleByDefault: false,
       valueGetter: ":sourceAlias.jsonb->>'purchaseOrderLineIdentifier'",
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'item',
+      dataType: {
+        dataType: 'stringType',
+      },
+      isIdColumn: false,
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
 }
diff --git a/src/main/resources/entity-types/inventory/simple_loan_types.json5 b/src/main/resources/entity-types/inventory/simple_loan_types.json5
@@ -88,5 +88,16 @@
       visibleByDefault: false,
       valueGetter: ":sourceAlias.jsonb->'metadata'->>'updatedByUserId'",
     },
+    {
+      name: 'jsonb',
+      sourceAlias: 'lt',
+      dataType: {
+        dataType: 'stringType',
+      },
+      isIdColumn: false,
+      queryable: false,
+      visibleByDefault: false,
+      valueGetter: ":sourceAlias.jsonb::text",
+    }
   ],
 }