Merge branch 'master' of github.com:folio-org/mod-agreements into per…

…f/pushKB

.github/workflows/validate-module.yml

@@ -1,71 +1,71 @@
-# name: Validate module
+name: Validate module
 
-# on:
-#   push:
+on:
+  push:
 
-# jobs:
-#   run:
-#     name: Validate module descriptor
-#     runs-on: ubuntu-latest
-#     steps:
-#       - name: Checkout repository
-#         uses: actions/checkout@v4
-#       - name: Get Pull Request Number
-#         id: pr_number
-#         run: echo "pull_request_number=$(gh pr view --json number -q .number || echo "")" >> $GITHUB_OUTPUT
-#         env:
-#           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#       - name: Setup java
-#         uses: actions/setup-java@v4
-#         with:
-#           distribution: 'temurin' # See 'Supported distributions' for available options
-#           java-version: '17'
-#       - name: Set up Maven
-#         uses: stCarolas/setup-maven@v5
-#         with:
-#           maven-version: 3.8.2
-#       - name: Set up settings file
-#         uses: 1arp/[email protected]
-#         with:
-#           path: 'service'
-#           file: 'settings.xml'
-#           content:
-#             <settings>
-#               <profiles>
-#                 <profile>
-#                   <id>folioMavenProfile</id>
-#                   <pluginRepositories>
-#                     <pluginRepository>
-#                       <id>folio-nexus</id>
-#                       <name>FOLIO Maven repository</name>
-#                       <url>https://repository.folio.org/repository/maven-folio</url>
-#                     </pluginRepository>
-#                   </pluginRepositories>
-#                 </profile>
-#               </profiles>
-#               <activeProfiles>
-#                 <activeProfile>folioMavenProfile</activeProfile>
-#               </activeProfiles>
-#             </settings>
-#       - name: Run validator
-#         run: mvn org.folio:folio-module-descriptor-validator:1.0.0:validate -DmoduleDescriptorFile=service/src/main/okapi/ModuleDescriptor-template.json -s service/settings.xml -l validate_module_descriptor_output.txt
-#       - name: Upload validator result
-#         uses: actions/upload-artifact@v4
-#         if: always()
-#         with:
-#           name: validate_module_descriptor_output
-#           path: |
-#             validate_module_descriptor_output.txt
-#           retention-days: 1
-#       - name: Setup validate_module_descriptor_errors file
-#         if: failure()
-#         run: echo "$(cat validate_module_descriptor_output.txt)" | egrep "\[ERROR\]\s*(\"key\"|\"value\")" | sed 's/\[ERROR\]\(\s*\)//;s/\"value\"\(\s*\):\(\s*\)\(.*\)/\3\n/;s/"key\"\(\s*\):\(\s*\)\(.*\)/\3/' | tee validate_module_descriptor_errors.txt
-#       - name: Comment failures on PR
-#         if: failure()
-#         run: |
-#           # Use GitHub API to create a comment on the PR
-#           PR_NUMBER=${{ steps.pr_number.outputs.pull_request_number }}
-#           GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
-#           COMMENT_URL="https://api.github.com/repos/${{ github.repository }}/issues/${PR_NUMBER}/comments"
-#           echo "SENDING TO: $COMMENT_URL"
-#           curl -s -H "Authorization: token ${GITHUB_TOKEN}" -X POST $COMMENT_URL --data "{ \"body\": $(cat validate_module_descriptor_errors.txt | jq -Rs) }"
+jobs:
+  run:
+    name: Validate module descriptor
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Get Pull Request Number
+        id: pr_number
+        run: echo "pull_request_number=$(gh pr view --json number -q .number || echo "")" >> $GITHUB_OUTPUT
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '17'
+      - name: Set up Maven
+        uses: stCarolas/setup-maven@v5
+        with:
+          maven-version: 3.8.2
+      - name: Set up settings file
+        uses: 1arp/[email protected]
+        with:
+          path: 'service'
+          file: 'settings.xml'
+          content:
+            <settings>
+              <profiles>
+                <profile>
+                  <id>folioMavenProfile</id>
+                  <pluginRepositories>
+                    <pluginRepository>
+                      <id>folio-nexus</id>
+                      <name>FOLIO Maven repository</name>
+                      <url>https://repository.folio.org/repository/maven-folio</url>
+                    </pluginRepository>
+                  </pluginRepositories>
+                </profile>
+              </profiles>
+              <activeProfiles>
+                <activeProfile>folioMavenProfile</activeProfile>
+              </activeProfiles>
+            </settings>
+      - name: Run validator
+        run: mvn org.folio:folio-module-descriptor-validator:1.0.0:validate -DmoduleDescriptorFile=service/src/main/okapi/ModuleDescriptor-template.json -s service/settings.xml -l validate_module_descriptor_output.txt
+      - name: Upload validator result
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: validate_module_descriptor_output
+          path: |
+            validate_module_descriptor_output.txt
+          retention-days: 1
+      - name: Setup validate_module_descriptor_errors file
+        if: failure()
+        run: echo "$(cat validate_module_descriptor_output.txt)" | egrep "\[ERROR\]\s*(\"key\"|\"value\")" | sed 's/\[ERROR\]\(\s*\)//;s/\"value\"\(\s*\):\(\s*\)\(.*\)/\3\n/;s/"key\"\(\s*\):\(\s*\)\(.*\)/\3/' | tee validate_module_descriptor_errors.txt
+      - name: Comment failures on PR
+        if: failure()
+        run: |
+          # Use GitHub API to create a comment on the PR
+          PR_NUMBER=${{ steps.pr_number.outputs.pull_request_number }}
+          GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
+          COMMENT_URL="https://api.github.com/repos/${{ github.repository }}/issues/${PR_NUMBER}/comments"
+          echo "SENDING TO: $COMMENT_URL"
+          curl -s -H "Authorization: token ${GITHUB_TOKEN}" -X POST $COMMENT_URL --data "{ \"body\": $(cat validate_module_descriptor_errors.txt | jq -Rs) }"

README.md

@@ -168,3 +168,35 @@ If you create a Service in Kubernetes named "okapi" and expose a port for Hazelc
 
 
 
+## Validating module descriptor
+There is a github action created to run the module descriptor validation on `push`, but if a developer wishes to run the validation locally there is some setup that needs doing. The validation script is a Maven plugin which does not work natively with our gradle based apps.
+### Maven
+Developer will need maven cli `mvn` installed on their machine.
+### settings.xml
+Create a `settings.xml` file within the "service" directory (DO NOT MERGE THIS) containing the following:
+```
+<settings>
+  <profiles>
+    <profile>
+      <id>folioMavenProfile</id>
+      <pluginRepositories>
+        <pluginRepository>
+          <id>folio-nexus</id>
+          <name>FOLIO Maven repository</name>
+          <url>https://repository.folio.org/repository/maven-folio</url>
+        </pluginRepository>
+      </pluginRepositories>
+    </profile>
+  </profiles>
+  <activeProfiles>
+    <activeProfile>folioMavenProfile</activeProfile>
+  </activeProfiles>
+</settings>
+```
+### Running
+Finally the developer can run this command from the root directory (ie `mod-agreements` not `mod-agreements/service`)
+```
+mvn org.folio:folio-module-descriptor-validator:1.0.0:validate -DmoduleDescriptorFile=service/src/main/okapi/ModuleDescriptor-template.json -s service/settings.xml -l validate_module_descriptor_output.txt
+```
+
+This will create a file called `validate_module_descriptor_output.txt` containing the output of the validator. The github action does some cleanup and comments the errors on a PR (if present). The `grep`/`sed` commands with regex can be found in the workflow file `.github/validate-module`.

service/grails-app/conf/application-dc.yml

@@ -27,6 +27,7 @@ okapi:
 logging:
   config: classpath:logback-development.xml
 
+concurrentJobsGlobal: "${CONCURRENT_JOBS_GLOBAL:3}"
 
 
 #    register: true

service/grails-app/controllers/org/olf/UrlMappings.groovy

@@ -50,10 +50,10 @@ class UrlMappings {
       }
 
       '/usageDataProviders' {
-  		  controller = 'usageDataProvider'
-  		  method = 'GET'
-  		  filters = { "owner==${params.subscriptionAgreementId}" }
-  	  }
+        controller = 'usageDataProvider'
+        method = 'GET'
+        filters = { "owner==${params.subscriptionAgreementId}" }
+      }
 
       // Root level extensions
       collection {

service/grails-app/init/org/olf/BootStrap.groovy

@@ -22,17 +22,17 @@ class BootStrap {
     });
 
     log.info("mod-agreements startup report");
-    log.info("${grailsApplication.getMetadata().getApplicationName()}  (${grailsApplication.config?.info?.app?.version}) initialising");
+    log.info("${grailsApplication.getMetadata().getApplicationName()} (${grailsApplication.config?.getProperty('info.app.version')}) initialising");
     log.info("          build number -> ${grailsApplication.metadata['build.number']}");
     log.info("        build revision -> ${grailsApplication.metadata['build.git.revision']}");
     log.info("          build branch -> ${grailsApplication.metadata['build.git.branch']}");
     log.info("          build commit -> ${grailsApplication.metadata['build.git.commit']}");
     log.info("            build time -> ${grailsApplication.metadata['build.time']}");
     log.info("            build host -> ${grailsApplication.metadata['build.host']}");
-    log.info("         Base JDBC URL -> ${grailsApplication.config.dataSource.url} / ${grailsApplication.config.dataSource.username}");
-    log.info("    default_aws_region -> ${grailsApplication.config.kiwt?.filestore?.aws_region}");
-    log.info("       default_aws_url -> ${grailsApplication.config.kiwt?.filestore?.aws_url}");
-    log.info("    default_aws_bucket -> ${grailsApplication.config.kiwt?.filestore?.aws_bucket}");
+    log.info("         Base JDBC URL -> ${grailsApplication.config.getProperty('dataSource.url')} / ${grailsApplication.config.getProperty('dataSource.username')}");
+    log.info("    default_aws_region -> ${grailsApplication.config.getProperty('kiwt.filestore.aws_region')}");
+    log.info("       default_aws_url -> ${grailsApplication.config.getProperty('kiwt.filestore.aws_url')}");
+    log.info("    default_aws_bucket -> ${grailsApplication.config.getProperty('kiwt.filestore.aws_bucket')}");
 
     Map<String, String> env = System.getenv();
     env.each { name,value ->

service/grails-app/services/org/olf/DependentModuleProxyService.groovy

@@ -31,7 +31,7 @@ public class DependentModuleProxyService {
       log.debug "No local org for ${orgName}. Check vendors."
 
       def mod_vendor_lookup_result = null;
-      if ( grailsApplication.config.useModVendors ) {
+      if ( grailsApplication.config.getProperty('useModVendors', boolean) ) {
         // This fetches a max of 2 (we should decide how to handle multiple matches) vendors with an exact name match.
         mod_vendor_lookup_result = okapiClient.get("/vendor", [
           limit: 2,
@@ -68,7 +68,7 @@ public class DependentModuleProxyService {
         case 0:
           // No match
           // We should add in an option to create vendors if users configure that
-          if ( grailsApplication.config.createMissingVendors ) {
+          if ( grailsApplication.config.getProperty('createMissingVendors', boolean) ) {
             throw new RuntimeException("Not yet implemented");
           }
 

service/grails-app/services/org/olf/ErmHousekeepingService.groovy

@@ -52,11 +52,11 @@ public class ErmHousekeepingService {
           RefdataValue.lookupOrCreate('FileStorageEngines', 'LOB');
           RefdataValue.lookupOrCreate('FileStorageEngines', 'S3');
 
-          def default_aws_region = grailsApplication.config.kiwt?.filestore?.aws_region
-          def default_aws_url = grailsApplication.config.kiwt?.filestore?.aws_url
-          def default_aws_secret = grailsApplication.config.kiwt?.filestore?.aws_secret
-          def default_aws_bucket = grailsApplication.config.kiwt?.filestore?.aws_bucket
-          def default_aws_access_key_id = grailsApplication.config.kiwt?.filestore?.aws_access_key_id
+          def default_aws_region = grailsApplication.config.getProperty('kiwt.filestore.aws_region')
+          def default_aws_url = grailsApplication.config.getProperty('kiwt.filestore.aws_url')
+          def default_aws_secret = grailsApplication.config.getProperty('kiwt.filestore.aws_secret')
+          def default_aws_bucket = grailsApplication.config.getProperty('kiwt.filestore.aws_bucket')
+          def default_aws_access_key_id = grailsApplication.config.getProperty('kiwt.filestore.aws_access_key_id')
 
           // Bootstrap any app settings we may need
           [

service/grails-app/services/org/olf/general/jobs/JobRunnerService.groovy

@@ -88,13 +88,21 @@ order by pj.dateCreated
   @PostConstruct
   void init() {
     // Set up the Executor
-    if ( grailsApplication.config.concurrentJobsGlobal instanceof Integer && grailsApplication.config.concurrentJobsGlobal > 0 )
-      CONCURRENT_JOBS_GLOBAL = grailsApplication.config.concurrentJobsGlobal;
+    try {
+      def concurrentJobsGlobalConfig = grailsApplication.config.getProperty('concurrentJobsGlobal', int);
+      if (concurrentJobsGlobalConfig > 0) {
+        CONCURRENT_JOBS_GLOBAL = concurrentJobsGlobalConfig;
+      }
+    } catch (Exception e) {
+      log.error("Failed to read concurrentJobsGlobal from config: ${e}")
+    }
 
+    log.info("Configured jobConcurrency: ${CONCURRENT_JOBS_GLOBAL}")
 		// Base the number of small jobs executable on the limit imposed on the default runner.
 		taskConcurrency = CONCURRENT_JOBS_GLOBAL * 2
-
-    // SO: This is not ideal. We don't want to limit jobs globally to 1 ideally. It should be 
+		log.info("Configured taskConcurrency: ${taskConcurrency}")
+
+    // SO: This is not ideal. We don't want to limit jobs globally to 1 ideally. It should be
     // 1 per tenant, but that will involve implementing custom handling for the queue and executor.
     // While we only have 1 tenant, this will suffice.
     executorSvc = new ThreadPoolExecutor(
@@ -111,7 +119,7 @@ order by pj.dateCreated
       5,
       TimeUnit.SECONDS, // Makes the above wait time in 'seconds'
       new LinkedBlockingQueue<Runnable>() // Blocking queue
-    )		
+    )
 
     // Raise an event to say we are ready.
     notify('jobs:job_runner_ready')