Merge pull request #752 from folio-org/release_7.0.1

Release 7.0.1

NEWS.md

@@ -1,3 +1,16 @@
+## 7.0.1 2024-04-17
+  * ERM-3193 Error saving proxy
+  * ERM-3191 Add logging to support issue triage for HibernateOptimisticLockingFailureException
+  * ERM-3190 DB Connections are not being released
+  * ERM-3185 Remove PTI/PCI filter from the /titles/electronic endpoint in Agreements
+  * ERM-3181 sourceTitleCount should not be required on uploading packages via JSON using mod-agreements-package schema
+  * ERM-3180 LOCAL external source should not offer delete option
+  * ERM-3177 Check on `noSICount` in WorkSourceIdentifierTIRSImpl is not correct
+  * ERM-3174 Review outdated/vulnerable dependencies in mod-agreements
+  * ERM-3157 Missed declaration of backend permissions
+  * ERM-3139 Propose additional info.log messages for jobs/processes
+  * ERM-3048 Add a package property to store remote source current title count
+
 ## 7.0.0 2024-03-22
   * ERM-3155 URL on PTI is limited to 255 characters
   * ERM-3154 Consolidate package upsert summary logging in info log

service/build.gradle

@@ -1,7 +1,9 @@
 /* configurations.all {
+
   // Check for updates every build
   resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
-} */
+}
+*/
 
 buildscript {
   repositories {
@@ -60,7 +62,7 @@ bootJar {
 }
 
 repositories {
-  //mavenLocal()
+//  mavenLocal()
 	maven { url "https://repo.grails.org/grails/core" }
 	maven { url "https://repo.grails.org/grails/plugins" }
 	maven { url "https://jitpack.io" }
@@ -99,7 +101,12 @@ dependencies {
 	implementation("org.grails.plugins:async")
   implementation("org.grails.plugins:events")
   implementation("org.grails.plugins:hibernate5")
-  implementation("org.grails.plugins:spring-security-core:6.1.1") // NOT IN LINE WITH GRAILS PATCH VERSION
+  implementation("org.grails.plugins:spring-security-core:6.1.1") /* { // NOT IN LINE WITH GRAILS PATCH VERSION
+    exclude group: 'org.springframework.security', module: 'spring-security-core'
+  }  */
+  // 5.8.9 affected by https://security.snyk.io/vuln/SNYK-JAVA-ORGSPRINGFRAMEWORKSECURITY-6457293
+  implementation("org.springframework.security:spring-security-core:5.8.11")
+
   implementation("org.grails.plugins:views-json")
   implementation("org.grails.plugins:views-json-templates")
   implementation("org.hibernate:hibernate-core:5.6.15.Final")
@@ -120,7 +127,7 @@ dependencies {
 
   /*  ---- Manually installed dependencies ---- */
 	implementation 'com.k_int.grails:web-toolkit-ce:9.0.0'
-	implementation('com.k_int.okapi:grails-okapi:7.0.0') {
+	implementation('com.k_int.okapi:grails-okapi:7.1.0') {
     exclude group: 'com.vaadin.external.google', module: 'android-json'
   }
 
@@ -129,17 +136,20 @@ dependencies {
   implementation "org.springframework.boot:spring-boot-starter-undertow" // Replaces spring-boot-starter-tomcat
   implementation "org.hibernate:hibernate-java8"
   runtimeOnly "com.zaxxer:HikariCP:5.1.0"                             // Replaces Tomcat JDBC pool
-  runtimeOnly "org.postgresql:postgresql:42.5.3"
+  runtimeOnly "org.postgresql:postgresql:42.7.3"
   implementation ('org.grails.plugins:database-migration:4.2.1') {
       exclude group: 'org.liquibase', module: 'liquibase-core'
   }
   implementation 'org.liquibase:liquibase-core:4.19.0'
 
 
   implementation 'com.opencsv:opencsv:5.7.1'
-  implementation 'commons-io:commons-io:2.6'
-  implementation 'io.github.virtualdogbert:logback-groovy-config:1.14.1' // Grails 5 and up no longer supports groovy files for logback config
+  implementation 'commons-io:commons-io:2.7'
+  // Grails 5 and up no longer supports groovy files for logback config
+  implementation('io.github.virtualdogbert:logback-groovy-config:1.14.1')
   compileOnly 'ch.qos.logback:logback-classic:1.4.7'
+  // Is on runtime classpath via spring-boot-starter
+  runtimeOnly 'ch.qos.logback:logback-classic:1.2.13'
 
   implementation 'uk.co.cacoethes:groovy-handlebars-engine:0.2'
   implementation 'com.github.jknack:handlebars-helpers:4.3.1'
@@ -164,8 +174,8 @@ dependencies {
 	implementation "org.seleniumhq.selenium:selenium-firefox-driver:3.14.0"
 
 	/* ---- Custom non profile deps ---- */
-	implementation 'org.apache.kafka:kafka-clients:2.3.0'
-	implementation 'com.github.everit-org.json-schema:org.everit.json.schema:1.14.3'
+	implementation 'org.apache.kafka:kafka-clients:3.7.0'
+	implementation 'com.github.everit-org.json-schema:org.everit.json.schema:1.14.4'
 	// Better test reports.
 	testImplementation( 'com.athaydes:spock-reports:2.3.2-groovy-3.0' ) {
 		transitive = false // this avoids affecting your version of Groovy/Spock

service/gradle.properties

@@ -4,7 +4,7 @@ gormVersion=8.0.3
 
 # Application
 appName=mod-agreements
-appVersion=7.0.0
+appVersion=7.0.1
 dockerTagSuffix=
 dockerRepo=folioci
 

service/grails-app/conf/logback-development.xml

@@ -35,6 +35,12 @@
   <logger name="org.olf">
     <level value="DEBUG" />
   </logger>
+
+
+
+  <logger name="org.springframework.jdbc.support" >
+    <level value="ERROR" />
+  </logger>
 
   <!-- For dev increase root to info -->
   <root level="info">

service/grails-app/conf/logback.xml

@@ -34,6 +34,10 @@
   <logger name="org.olf">
     <level value="INFO" />
   </logger>
+
+  <logger name="org.springframework.jdbc.support" >
+    <level value="ERROR" />
+  </logger>
 
 
   <!--LOG SQL - VERBOSE!!!!!!-->
@@ -125,6 +129,7 @@
   <root level="warn">
     <appender-ref ref="STDOUT" />
   </root>
+
 </configuration>
 
 <!-- GROOVY FILE

service/grails-app/controllers/org/olf/TitleController.groovy

@@ -22,38 +22,12 @@ class TitleController extends OkapiTenantAwareController<TitleInstance>  {
     super(TitleInstance)
   }
 
-  DetachedCriteria pciSubQuery = PackageContentItem.where({
-    eqProperty('pti.id', 'platformTitleInstance.id')
-    setAlias 'packageContentItem'
-    isNull 'removedTimestamp'
-
-    projections {
-      property 'id'
-    }
-  })
-
-  DetachedCriteria ptiSubQuery = PlatformTitleInstance.where({
-    eqProperty('titleInstance.id', "${DEFAULT_ROOT_ALIAS}.id") //here "this" refers to the root alias of criteria
-    setAlias 'platformTitleInstance'
-
-    exists(pciSubQuery)
-
-    projections {
-      property 'id'
-    }
-  })
-
-  // LEFT JOIN query
   def electronic () {
     respond doTheLookup {
-      and {
         eq 'subType', TitleInstance.lookupOrCreateSubType('electronic')
-
-        exists(ptiSubQuery)
-      }
     }
   }
-  
+
   def entitled() {
     respond doTheLookup (TitleInstance.entitled)
   }

service/grails-app/services/org/olf/ImportService.groovy

@@ -97,7 +97,7 @@ class ImportService implements DataBinder {
     envelope.records?.each { Map record ->
       // Ingest 1 package at a time.
       Map importResult = importPackage (record, ErmPackageImplWithContentItems)
-
+                  
       if (importResult.packageImported) {
         packageCount ++
         String packageId = importResult.packageId
@@ -115,7 +115,7 @@ class ImportService implements DataBinder {
   }
 
   private Map importPackage (final Map record, final Class<? extends PackageSchema> schemaClass) {
-    boolean packageImported = true
+    boolean packageImported = false
     String packageId = ""
 
     final PackageSchema pkg = schemaClass.newInstance()

service/grails-app/services/org/olf/KbHarvestService.groovy

@@ -163,7 +163,7 @@ where rkb.type is not null
             knowledgeBaseCacheService.runSync((String)remotekb_id)
           }
           catch ( Exception e ) {
-            log.warn("problem processing remote KB link",e)
+            log.warn("Unexpected exception encountered during runSync for ${remotekb_id}", e)
           }
           finally {
             log.info("KbHarvestService.closure completed - ${System.currentTimeMillis()-gokb_sync_start_time}ms elapsed. Release sync status");
@@ -260,7 +260,7 @@ where rkb.type is not null
       // This might ought to be in debug to bring in line with other methods in here, 
       // although it might be more useful to know it happened since the call log above is "debug" level
       // Default logback config has this service at "DEBUG" level anyway so it might not matter.
-			log.warn("KBHarvestService::handleInterruptedJob() completed");
+			log.info("KBHarvestService::handleInterruptedJob() completed");
 		}
 	}
 }

service/grails-app/services/org/olf/PackageIngestService.groovy

@@ -278,7 +278,7 @@ class PackageIngestService implements DataBinder {
               description: package_data.header.description,
         sourceDataCreated: package_data.header.sourceDataCreated,
         sourceDataUpdated: package_data.header.sourceDataUpdated,
-         sourceTitleCount: (package_data.header.sourceTitleCount ?: null),
+         sourceTitleCount: package_data.header.sourceTitleCount,
         availabilityScope: ( package_data.header.availabilityScope != null ? Pkg.lookupOrCreateAvailabilityScope(package_data.header.availabilityScope) : null ),
           lifecycleStatus: Pkg.lookupOrCreateLifecycleStatus(package_data.header.lifecycleStatus != null ? package_data.header.lifecycleStatus : 'Unknown'),
                     vendor: vendor,

service/grails-app/services/org/olf/general/StringTemplatingService.groovy

@@ -323,7 +323,6 @@ public class StringTemplatingService implements ApplicationListener<ApplicationE
     final Class theClass = eo.getClass()
 
     switch(event.getEventType()) {
-
       case PreUpdate:
       case PreInsert:
         if (PlatformTitleInstance.isAssignableFrom(theClass)) {
@@ -563,18 +562,26 @@ public class StringTemplatingService implements ApplicationListener<ApplicationE
   private void addDeferredPlatformUpdatesFromTemplates (  final AbstractPersistenceEvent event ) {
     // Multiple Platform update when String template added/removed/changed
     final StringTemplate template = event.entityObject as StringTemplate
-
+    
     // Make sure we are in the context of a tenant.
     final String currentTenant = ensureTenant()
 
     // Overrides include any Platforms for update explicitly.
     final Set<String> theScopes = (event.eventType != PreDelete ? template.idScopes : Collections.emptySet()) as Set<String>
-
     // Context
     final String theContext = template.context.value
 
     // Fetch the scopes from the Session to get "Previous" values
-    final Set<String> previousScopes = getScopesForStringTemplate(template.id)
+    /* For some reason pre insert events making a query end up re-raising
+     * the pre insert event in Grails 6, causing an infinite loop.
+     * However in this case a pre-insert event means that no
+     * "previous scopes" exist for the string template in question, so
+     * assume an empty set and move on with the logic
+     */
+    Set<String> previousScopes = [] as Set;
+    if (event.eventType != PreInsert) {
+      previousScopes = getScopesForStringTemplate(template.id)
+    }
 
     // Explicitly re-process the differences between the scope now and the previous
     final Set<String> reprocess = ( theScopes + previousScopes ) - theScopes.intersect(previousScopes)
@@ -587,9 +594,9 @@ public class StringTemplatingService implements ApplicationListener<ApplicationE
     StringTemplate explicitTemplate = event.eventType != PreDelete ? template : null
 
     // Build the work but stash it for running once we see the "post" events
-    addStashForId(template.id,
+    addStashForId(
+      template.id,
       new Stash((Closure <?>) { final String tenantId, final Set<String> scopes, final String context, final Set<String> overrides, StringTemplate tmpl ->
-
         Tenants.withId(tenantId) {
           GormStaticApi<Platform> ptiApi = GormUtils.gormStaticApi(Platform)
           List<String> platformIds = getAllPlatformIdsForTemplateParams(scopes, context, overrides)
@@ -625,6 +632,7 @@ public class StringTemplatingService implements ApplicationListener<ApplicationE
               }
             }
 
+
             executor.execute({ String tid, String pltf, Date before, Map<String, StringTemplate> inEx ->
 
               Tenants.withId(tid) {
@@ -635,12 +643,11 @@ public class StringTemplatingService implements ApplicationListener<ApplicationE
             }.curry(tenantId, platformId, now, includeExclude))
           }
         }
-
-      }.curry(currentTenant, theScopes, theContext, reprocess, explicitTemplate)))
+      }.curry(currentTenant, theScopes, theContext, reprocess, explicitTemplate))
+    )
   }
 
   private void updatePlatformsFromTemplateEvent(final AbstractPersistenceEvent event) {
-
     // Multiple Platform update when String template added/removed/changed
     final StringTemplate template = event.entityObject as StringTemplate
 
@@ -685,13 +692,12 @@ public class StringTemplatingService implements ApplicationListener<ApplicationE
   }
 
   private Set<String> getScopesForStringTemplate ( final String templateId  ) {
-
     final String hql = '''
       SELECT scope FROM StringTemplate tmp
       JOIN tmp.idScopes AS scope
       WHERE tmp.id = :templateId
     '''
 
-    GormUtils.gormStaticApi(StringTemplate).executeQuery(hql, ['templateId': templateId]) as Set
+    return GormUtils.gormStaticApi(StringTemplate).executeQuery(hql.toString(), ['templateId': templateId]) as Set
   }
 }

service/grails-app/services/org/olf/general/jobs/JobRunnerService.groovy

@@ -26,6 +26,7 @@ import org.olf.IdentifierService
 import org.olf.KbHarvestService
 import org.olf.KbManagementService
 import org.olf.general.jobs.PersistentJob.Type
+import org.springframework.jdbc.support.JdbcUtils
 
 import com.k_int.okapi.OkapiTenantAdminService
 import com.k_int.okapi.OkapiTenantResolver
@@ -260,10 +261,18 @@ order by pj.dateCreated
       }
     }
   }
+
+	private volatile boolean firstTick = true;
 
   @Subscriber('federation:tick:leader')
   void leaderTick(final String instanceId) {
 
+		if (firstTick == true) {
+			log.info("Skipping first tick in job runner to ensure datasources are present.")
+			firstTick = false
+			return
+		}
+
 		WithPromises.task { 
 			Tenants.withId(SystemDataService.DATASOURCE_SYSTEM) {
 				GormUtils.withTransaction {
@@ -278,6 +287,13 @@ order by pj.dateCreated
 
   @Subscriber('federation:tick:drone')
   void droneTick(final String instanceId) {
+
+		if (firstTick == true) {
+			log.info("Skipping first tick in job runner to ensure datasources are present.")
+			firstTick = false
+			return
+		}
+
 		WithPromises.task {
 			Tenants.withId(SystemDataService.DATASOURCE_SYSTEM) {
 				GormUtils.withTransaction {

service/src/main/groovy/org/olf/dataimport/erm/ErmPackageImpl.groovy

@@ -56,13 +56,14 @@ class ErmPackageImpl implements PackageHeaderSchema, PackageSchema, Validateable
     trustedSourceTI nullable: true
     description nullable: true
 
-    source    nullable: false, blank: false
-    reference nullable: false, blank: false
-    name      nullable: false, blank: false
-    packageProvider nullable: true
+    source             nullable: false, blank: false
+    reference          nullable: false, blank: false
+    name               nullable: false, blank: false
+    packageProvider    nullable: true
     availabilityScope  nullable: true, blank: false
     sourceDataCreated  nullable: true, blank: false
     sourceDataUpdated  nullable: true, blank: false
+    sourceTitleCount   nullable: true, blank: false
     lifecycleStatus    nullable: true, blank: false
   }
 

service/src/main/groovy/org/olf/dataimport/internal/titleInstanceResolvers/WorkSourceIdentifierTIRSImpl.groovy

@@ -45,13 +45,19 @@ class WorkSourceIdentifierTIRSImpl extends IdFirstTIRSImpl implements DataBinder
         return false;
       }
 
+      // This assumes that IdentiferOccurrences can only be attached to Works as the sourceIdentifier.
+      // If this changes then rework will be needed.
       long noSICount = Work.executeQuery("""
-        SELECT COUNT(w) FROM Work w WHERE w.sourceIdentifier = null
-      """.toString())?.get(0);
+        SELECT COUNT(w) FROM Work w
+        LEFT JOIN IdentifierOccurrence io ON io.resource = w.id
+        WHERE io.id = null
+        """.toString())?.get(0);
 
+/*
       long workCount = Work.executeQuery("""
         SELECT COUNT(w) FROM Work w
       """.toString())?.get(0);
+*/
 
       // For now keep the fallback unless there are NO Works in the system without SI
       if (noSICount == 0) {
@@ -74,7 +80,6 @@ class WorkSourceIdentifierTIRSImpl extends IdFirstTIRSImpl implements DataBinder
     // Error out if sourceIdentifier or sourceIdentifierNamespace do not exist
     ensureSourceIdentifierFields(citation);
 
-    // TODO Check this works with the switch to grabbing only id
     List<String> candidate_works = Work.executeQuery("""
       SELECT w.id FROM Work as w
       WHERE