Skip to content

Commit

Permalink
Merge branch 'b1.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
bvsharp committed Nov 14, 2023
2 parents 0932123 + ada4d0c commit b8626f8
Show file tree
Hide file tree
Showing 26 changed files with 1,307 additions and 23 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
.env

### STS ###
.apt_generated
Expand Down
16 changes: 14 additions & 2 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
# 1.0.0
- Initial release
# 1.0.x

## 1.0.3
- [MODFQMMGR-58](https://issues.folio.org/browse/MODFQMMGR-58) Refactor drv_item_callnumber_location view
- [MODFQMMGR-76](https://issues.folio.org/browse/MODFQMMGR-76) Periodically refresh materialized views

## 1.0.2
- Remove the instance_title_searchable field from the Items entity type
- Purge old query results based on query start date/time instead of the end date/time
- Fix bug in user preferred contact type
- Update the provided `_tenant` interface in the module descriptor to 2.0

## 1.0.1
- [MODFQMMGR-57](https://issues.folio.org/browse/MODFQMMGR-57) Use a different version of f_unaccent(), to allow us to make use of an index
Expand All @@ -8,3 +17,6 @@
- [MODFQMMGR-67](https://issues.folio.org/browse/MODFQMMGR-67) Enable batched inserts
- [MODFQMMGR-71](https://issues.folio.org/browse/MODFQMMGR-71) Update item and user entity types
- [MODFQMMGR-31](https://issues.folio.org/browse/MODFQMMGR-31) Fix Users dropdown

## 1.0.0
- Initial release
42 changes: 34 additions & 8 deletions descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
{
"id": "@artifactId@-@version@",
"name": "The module descriptor for mod-fqm-manager.",
"name": "FQM Manager Module",
"provides": [
{
"id": "_tenant",
"version": "1.2",
"version": "2.0",
"interfaceType": "system",
"handlers": [
{
"methods": ["POST"],
"pathPattern": "/_/tenant"
}, {
"methods": ["DELETE"],
"pathPattern": "/_/tenant"
"pathPattern": "/_/tenant",
"permissionsRequired": []
},
{
"methods": ["GET", "DELETE"],
"pathPattern": "/_/tenant/{id}",
"permissionsRequired": []
}
]
},
Expand All @@ -34,6 +37,11 @@
"methods": ["GET"],
"pathPattern": "/entity-types/{entity-type-id}/columns/{column-name}/values",
"permissionsRequired": ["fqm.entityTypes.item.columnValues.get"]
},
{
"methods": ["POST"],
"pathPattern": "/entity-types/materialized-views/refresh",
"permissionsRequired": ["fqm.materializedViews.post"]
}
]
},
Expand Down Expand Up @@ -84,10 +92,22 @@
"interfaceType": "system",
"handlers": [
{
"methods": [ "POST" ],
"methods": ["POST"],
"pathPattern": "/query/purge",
"unit": "hour",
"delay": "1"
"delay": "1",
"modulePermissions": [
"fqm.query.purge"
]
},
{
"methods": ["POST"],
"pathPattern": "/entity-types/materialized-views/refresh",
"unit": "hour",
"delay": "24",
"modulePermissions": [
"fqm.materializedViews.post"
]
}
]
}
Expand Down Expand Up @@ -141,6 +161,12 @@
"description": "Run a query synchronously and get results",
"visible": true
},
{
"permissionName": "fqm.materializedViews.post",
"displayName": "FQM - Refresh materialized views",
"description": "Refresh FQM materialized views",
"visible": true
},
{
"permissionName": "fqm.query.all",
"displayName": "FQM - All permissions",
Expand Down
6 changes: 3 additions & 3 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -373,9 +373,9 @@
</distributionManagement>

<scm>
<url>https://github.com/EBSCOIS/${project.artifactId}</url>
<connection>scm:git:git://github.com/EBSCOIS/${project.artifactId}.git</connection>
<developerConnection>scm:git:[email protected]:EBSCOIS/${project.artifactId}.git</developerConnection>
<url>https://github.com/folio-org/${artifactId}</url>
<connection>scm:git:git://github.com/folio-org/${artifactId}.git</connection>
<developerConnection>scm:git:[email protected]:folio-org/${artifactId}.git</developerConnection>
<tag>HEAD</tag>
</scm>

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package org.folio.fqm.repository;

import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.jooq.DSLContext;
import org.springframework.stereotype.Repository;

import java.util.List;

@Repository
@RequiredArgsConstructor
@Log4j2
public class MaterializedViewRefreshRepository {
private static final String REFRESH_MATERIALIZED_VIEW_SQL = "REFRESH MATERIALIZED VIEW CONCURRENTLY ";

private static final List<String> materializedViewNames = List.of(
"drv_circulation_loan_status",
"drv_inventory_item_status"
);

private final DSLContext jooqContext;

public void refreshMaterializedViews(String tenantId) {
for (String matViewName : materializedViewNames) {
String fullName = tenantId + "_mod_fqm_manager." + matViewName;
log.info("Refreshing materialized view {}", fullName);
jooqContext.execute(REFRESH_MATERIALIZED_VIEW_SQL + fullName);
}
}
}
4 changes: 2 additions & 2 deletions src/main/java/org/folio/fqm/repository/QueryRepository.java
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,10 @@ public Optional<Query> getQuery(UUID queryId, boolean useCache) {
.fetchOneInto(Query.class));
}

public List<UUID> getQueryIdsCompletedBefore(Duration duration) {
public List<UUID> getQueryIdsStartedBefore(Duration duration) {
return jooqContext.select(field(QUERY_ID))
.from(table(QUERY_DETAILS_TABLE))
.where(field("end_date").
.where(field("start_date").
lessOrEqual(OffsetDateTime.now().minus(duration)))
.fetchInto(UUID.class);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package org.folio.fqm.resource;

import lombok.RequiredArgsConstructor;
import org.folio.fqm.service.MaterializedViewRefreshService;
import org.folio.spring.FolioExecutionContext;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequiredArgsConstructor
public class MaterializedViewRefreshController implements MaterializedViewsApi {
private final FolioExecutionContext executionContext;
private final MaterializedViewRefreshService materializedViewRefreshService;

@Override
public ResponseEntity<Void> refreshMaterializedViews() {
materializedViewRefreshService.refreshMaterializedViews(executionContext.getTenantId());
return new ResponseEntity<>(HttpStatus.NO_CONTENT);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package org.folio.fqm.service;

import lombok.RequiredArgsConstructor;
import org.folio.fqm.repository.MaterializedViewRefreshRepository;
import org.springframework.stereotype.Service;

@Service
@RequiredArgsConstructor
public class MaterializedViewRefreshService {
private final MaterializedViewRefreshRepository materializedViewRefreshRepository;

public void refreshMaterializedViews(String tenantId) {
materializedViewRefreshRepository.refreshMaterializedViews(tenantId);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ public Optional<QueryDetails> getQuery(UUID queryId, boolean includeResults, int
*/
@Transactional
public PurgedQueries deleteOldQueries() {
List<UUID> queryIds = queryRepository.getQueryIdsCompletedBefore(queryRetentionDuration);
List<UUID> queryIds = queryRepository.getQueryIdsStartedBefore(queryRetentionDuration);
log.info("Deleting the queries with queryIds {}", queryIds);
deleteQueryAndResults(queryIds);
return new PurgedQueries().deletedQueryIds(queryIds);
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/db/changelog/changelog-master.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,7 @@

<include file="changes/v1.0.0/changelog-v1.0.0.xml" relativeToChangelogFile="true"/>
<include file="changes/v1.0.1/changelog-v1.0.1.xml" relativeToChangelogFile="true"/>
<include file="changes/v1.0.2/changelog-v1.0.2.xml" relativeToChangelogFile="true"/>
<include file="changes/v1.0.3/changelog-v1.0.3.xml" relativeToChangelogFile="true"/>

</databaseChangeLog>
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">

<include file="remove_items_instance_title_searchable.xml" relativeToChangelogFile="true"/>
<include file="update_item_details_entity_type_definition.xml" relativeToChangelogFile="true"/>

<include file="remove_items_holdingsrecord_instance_title_searchable.xml" relativeToChangelogFile="true"/>
<include file="update_item_holdingsrecord_entity_type_definition.xml" relativeToChangelogFile="true"/>

<include file="fix_user_details_preferred_contact_type.xml" relativeToChangelogFile="true"/>
<include file="update_user_details_entity_type_definition.xml" relativeToChangelogFile="true"/>

</databaseChangeLog>
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
<databaseChangeLog xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
<changeSet id="create-view-user-details" author="[email protected]">
<!-- ensure other schemas/tables exist (primarily to prevent invalid references in integration tests) -->
<preConditions onFail="CONTINUE">
<viewExists viewName="src_users_groups"/>
<viewExists viewName="src_users_users"/>
</preConditions>
<createView
replaceIfExists="true"
viewName="drv_user_details">
SELECT
userDetails.jsonb -> 'personal' ->> 'firstName' as user_first_name,
userDetails.jsonb -> 'personal' ->> 'lastName' as user_last_name,
userDetails.jsonb ->>'barcode' as user_barcode,
userDetails.jsonb ->>'username' as username,
userDetails.id as id,
userDetails.jsonb ->> 'externalSystemId' as user_external_system_id,
userDetails.jsonb ->> 'active' as user_active,
userDetails.jsonb -> 'personal' ->> 'email' as user_email,
userDetails.jsonb -> 'metadata' ->> 'createdDate' as user_created_date,
userDetails.jsonb -> 'metadata' ->> 'updatedDate' as user_updated_date,
userDetails.jsonb -> 'personal' ->> 'preferredFirstName' as user_preferred_first_name,
userDetails.jsonb -> 'personal' ->> 'middleName' as user_middle_name,
userDetails.jsonb -> 'personal' ->> 'phone' as user_phone,
userDetails.jsonb -> 'personal' ->> 'mobilePhone' as user_mobile_phone,
userDetails.jsonb -> 'personal' ->> 'dateOfBirth' as user_date_of_birth,
userDetails.jsonb ->> 'expirationDate'::text AS user_expiration_date,
userDetails.jsonb ->> 'enrollmentDate'::text AS user_enrollment_date,
patron_id_ref_data.jsonb ->> 'group'::text AS user_patron_group,
patron_id_ref_data.id::text AS user_patron_group_id,
UserDetails.jsonb -> 'personal' ->> 'preferredContactTypeId' as user_preferred_contact_type_id,
CASE UserDetails.jsonb -> 'personal' ->> 'preferredContactTypeId'
WHEN '001' THEN 'Mail (Primary Address)'
WHEN '002' THEN 'Email'
WHEN '003' THEN 'Text Message'
ELSE 'unknown'
END AS user_preferred_contact_type,
concat_ws(', '::text,
NULLIF(( SELECT subquery.addressline1
FROM ( SELECT add_id.value ->> 'addressLine1'::text AS addressline1,
row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
WHERE subquery.row_num = 1), ''::text),
NULLIF(( SELECT subquery.addressline2
FROM ( SELECT add_id.value ->> 'addressLine2'::text AS addressline2,
row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
WHERE subquery.row_num = 1), ''::text),
NULLIF(( SELECT subquery.city
FROM ( SELECT add_id.value ->> 'city'::text AS city,
row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
WHERE subquery.row_num = 1), ''::text),
NULLIF(( SELECT subquery.region
FROM ( SELECT add_id.value ->> 'region'::text AS region,
row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
WHERE subquery.row_num = 1), ''::text),
NULLIF(( SELECT subquery.postalcode
FROM ( SELECT add_id.value ->> 'postalCode'::text AS postalcode,
row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
WHERE subquery.row_num = 1), ''::text),
NULLIF(( SELECT subquery.countryid
FROM ( SELECT add_id.value ->> 'countryId'::text AS countryid,
row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
WHERE subquery.row_num = 1), ''::text)) AS user_primary_address,
( SELECT array_agg(add_id.value ->> 'city'::text) FILTER (WHERE (add_id.value ->> 'city'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS cities,
( SELECT array_agg(add_id.value ->> 'region'::text) FILTER (WHERE (add_id.value ->> 'region'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS regions,
( SELECT array_agg(add_id.value ->> 'countryId'::text) FILTER (WHERE (add_id.value ->> 'countryId'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS country_ids,
( SELECT array_agg(add_id.value ->> 'postalCode'::text) FILTER (WHERE (add_id.value ->> 'postalCode'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS postal_codes,
( SELECT array_agg(add_id.value ->> 'addressLine1'::text) FILTER (WHERE (add_id.value ->> 'addressLine1'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS address_line1,
( SELECT array_agg(add_id.value ->> 'addressLine2'::text) FILTER (WHERE (add_id.value ->> 'addressLine2'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS address_line2,
( SELECT array_agg(add_id.value ->> 'addressTypeId'::text) FILTER (WHERE (add_id.value ->> 'addressTypeId'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS address_ids,
( SELECT array_agg(a.jsonb ->> 'addressType'::text) FILTER (WHERE (a.jsonb ->> 'addressType'::text) IS NOT NULL) AS array_agg
FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)
JOIN src_users_addresstype a ON (add_id.value ->> 'addressTypeId'::text) = a.id::text) AS address_type_names,
array_agg(temp_departments.id::text) FILTER (where temp_departments.id is not null) as department_ids,
array_agg(temp_departments.jsonb ->> 'name') FILTER (where temp_departments.jsonb ->> 'name' is not null) as department_names
FROM src_users_users userdetails
LEFT JOIN src_users_groups patron_id_ref_data ON patron_id_ref_data.id = ((userdetails.jsonb ->> 'patronGroup'::text)::uuid)
LEFT JOIN src_users_departments as temp_departments ON userdetails.jsonb -> 'departments' ?? temp_departments.id::text
GROUP BY userdetails.id, userdetails.jsonb, patron_id_ref_data.id, patron_id_ref_data.jsonb
</createView>
</changeSet>

</databaseChangeLog>
Loading

0 comments on commit b8626f8

Please sign in to comment.