Merge pull request #826 from folio-org/fix/concurrent_jobs

fix: concurrent jobs
folio-org · Oct 3, 2024 · 88aeaa5 · 88aeaa5
2 parents 76e7fe9 + 91c9102
commit 88aeaa5
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 18 deletions.
diff --git a/service/grails-app/conf/application-dc.yml b/service/grails-app/conf/application-dc.yml
@@ -27,6 +27,7 @@ okapi:
 logging:
   config: classpath:logback-development.xml
 
+concurrentJobsGlobal: "${CONCURRENT_JOBS_GLOBAL:3}"
 
 
 #    register: true

diff --git a/service/grails-app/init/org/olf/BootStrap.groovy b/service/grails-app/init/org/olf/BootStrap.groovy
@@ -22,17 +22,17 @@ class BootStrap {
     });
 
     log.info("mod-agreements startup report");
-    log.info("${grailsApplication.getMetadata().getApplicationName()}  (${grailsApplication.config?.info?.app?.version}) initialising");
+    log.info("${grailsApplication.getMetadata().getApplicationName()} (${grailsApplication.config?.getProperty('info.app.version')}) initialising");
     log.info("          build number -> ${grailsApplication.metadata['build.number']}");
     log.info("        build revision -> ${grailsApplication.metadata['build.git.revision']}");
     log.info("          build branch -> ${grailsApplication.metadata['build.git.branch']}");
     log.info("          build commit -> ${grailsApplication.metadata['build.git.commit']}");
     log.info("            build time -> ${grailsApplication.metadata['build.time']}");
     log.info("            build host -> ${grailsApplication.metadata['build.host']}");
-    log.info("         Base JDBC URL -> ${grailsApplication.config.dataSource.url} / ${grailsApplication.config.dataSource.username}");
-    log.info("    default_aws_region -> ${grailsApplication.config.kiwt?.filestore?.aws_region}");
-    log.info("       default_aws_url -> ${grailsApplication.config.kiwt?.filestore?.aws_url}");
-    log.info("    default_aws_bucket -> ${grailsApplication.config.kiwt?.filestore?.aws_bucket}");
+    log.info("         Base JDBC URL -> ${grailsApplication.config.getProperty('dataSource.url')} / ${grailsApplication.config.getProperty('dataSource.username')}");
+    log.info("    default_aws_region -> ${grailsApplication.config.getProperty('kiwt.filestore.aws_region')}");
+    log.info("       default_aws_url -> ${grailsApplication.config.getProperty('kiwt.filestore.aws_url')}");
+    log.info("    default_aws_bucket -> ${grailsApplication.config.getProperty('kiwt.filestore.aws_bucket')}");
 
     Map<String, String> env = System.getenv();
     env.each { name,value ->

diff --git a/service/grails-app/services/org/olf/DependentModuleProxyService.groovy b/service/grails-app/services/org/olf/DependentModuleProxyService.groovy
@@ -31,7 +31,7 @@ public class DependentModuleProxyService {
       log.debug "No local org for ${orgName}. Check vendors."
 
       def mod_vendor_lookup_result = null;
-      if ( grailsApplication.config.useModVendors ) {
+      if ( grailsApplication.config.getProperty('useModVendors', boolean) ) {
         // This fetches a max of 2 (we should decide how to handle multiple matches) vendors with an exact name match.
         mod_vendor_lookup_result = okapiClient.get("/vendor", [
           limit: 2,
@@ -68,7 +68,7 @@ public class DependentModuleProxyService {
         case 0:
           // No match
           // We should add in an option to create vendors if users configure that
-          if ( grailsApplication.config.createMissingVendors ) {
+          if ( grailsApplication.config.getProperty('createMissingVendors', boolean) ) {
             throw new RuntimeException("Not yet implemented");
           }
 

diff --git a/service/grails-app/services/org/olf/ErmHousekeepingService.groovy b/service/grails-app/services/org/olf/ErmHousekeepingService.groovy
@@ -52,11 +52,11 @@ public class ErmHousekeepingService {
           RefdataValue.lookupOrCreate('FileStorageEngines', 'LOB');
           RefdataValue.lookupOrCreate('FileStorageEngines', 'S3');
 
-          def default_aws_region = grailsApplication.config.kiwt?.filestore?.aws_region
-          def default_aws_url = grailsApplication.config.kiwt?.filestore?.aws_url
-          def default_aws_secret = grailsApplication.config.kiwt?.filestore?.aws_secret
-          def default_aws_bucket = grailsApplication.config.kiwt?.filestore?.aws_bucket
-          def default_aws_access_key_id = grailsApplication.config.kiwt?.filestore?.aws_access_key_id
+          def default_aws_region = grailsApplication.config.getProperty('kiwt.filestore.aws_region')
+          def default_aws_url = grailsApplication.config.getProperty('kiwt.filestore.aws_url')
+          def default_aws_secret = grailsApplication.config.getProperty('kiwt.filestore.aws_secret')
+          def default_aws_bucket = grailsApplication.config.getProperty('kiwt.filestore.aws_bucket')
+          def default_aws_access_key_id = grailsApplication.config.getProperty('kiwt.filestore.aws_access_key_id')
 
           // Bootstrap any app settings we may need
           [

diff --git a/service/grails-app/services/org/olf/general/jobs/JobRunnerService.groovy b/service/grails-app/services/org/olf/general/jobs/JobRunnerService.groovy
@@ -88,13 +88,21 @@ order by pj.dateCreated
   @PostConstruct
   void init() {
     // Set up the Executor
-    if ( grailsApplication.config.concurrentJobsGlobal instanceof Integer && grailsApplication.config.concurrentJobsGlobal > 0 )
-      CONCURRENT_JOBS_GLOBAL = grailsApplication.config.concurrentJobsGlobal;
+    try {
+      def concurrentJobsGlobalConfig = grailsApplication.config.getProperty('concurrentJobsGlobal', int);
+      if (concurrentJobsGlobalConfig > 0) {
+        CONCURRENT_JOBS_GLOBAL = concurrentJobsGlobalConfig;
+      }
+    } catch (Exception e) {
+      log.error("Failed to read concurrentJobsGlobal from config: ${e}")
+    }
 
+    log.info("Configured jobConcurrency: ${CONCURRENT_JOBS_GLOBAL}")
 		// Base the number of small jobs executable on the limit imposed on the default runner.
 		taskConcurrency = CONCURRENT_JOBS_GLOBAL * 2
-
-    // SO: This is not ideal. We don't want to limit jobs globally to 1 ideally. It should be 
+		log.info("Configured taskConcurrency: ${taskConcurrency}")
+
+    // SO: This is not ideal. We don't want to limit jobs globally to 1 ideally. It should be
     // 1 per tenant, but that will involve implementing custom handling for the queue and executor.
     // While we only have 1 tenant, this will suffice.
     executorSvc = new ThreadPoolExecutor(
@@ -111,7 +119,7 @@ order by pj.dateCreated
       5,
       TimeUnit.SECONDS, // Makes the above wait time in 'seconds'
       new LinkedBlockingQueue<Runnable>() // Blocking queue
-    )		
+    )
 
     // Raise an event to say we are ready.
     notify('jobs:job_runner_ready')

diff --git a/service/src/main/okapi/tenant/sample_data/_data.groovy b/service/src/main/okapi/tenant/sample_data/_data.groovy
@@ -32,7 +32,7 @@ RemoteKB.findByName('GOKb') ?: (new RemoteKB(
     uri:'https://gokb.org/gokb/oai/index',
     fullPrefix:'gokb',
     rectype: RemoteKB.RECTYPE_PACKAGE,
-    active:Boolean.FALSE,
+    active:Boolean.TRUE,
     supportsHarvesting:true,
     activationEnabled:false
 ).save(failOnError:true))