Merge branch 'release/1.16.0'

.circleci/config.yml

@@ -3,23 +3,28 @@ parameters:
   java-tag:
     type: string
     default: "17.0.3"
+  run_against_develop_core:
+    type: boolean
+    default: false
+
 orbs:
   build-tools: circleci/[email protected]
+  slack: circleci/[email protected]
 executors:
   unit_tests_executor:
     docker: # run the steps with Docker
       - image: cimg/openjdk:<< pipeline.parameters.java-tag >>
 
   machine_integration_test_exec:
     machine: # run the steps with Ubuntu VM
-      image: ubuntu-2204:2022.10.1
+      image: ubuntu-2204:2023.10.1
     environment:
       PGHOST: 127.0.0.1
     resource_class: medium
 
   toil_wes_test_executor:
     machine: # run the steps with Ubuntu VM
-      image: ubuntu-2204:2022.10.1
+      image: ubuntu-2204:2023.10.1
     resource_class: medium
 
 common_filters: &common_filters
@@ -30,52 +35,70 @@ common_filters: &common_filters
       ignore:
         - gh-pages
 
+slack_context: &slack_context
+  context:
+    - oicr-slack
+
+common_jobs: &common_jobs
+    jobs:
+        - build:
+            <<: *common_filters
+            <<: *slack_context
+        - unit-tests:
+            <<: *common_filters
+            <<: *slack_context
+            requires:
+              - build
+        - integration-tests:
+            matrix:
+              parameters:
+                testing_profile: [ "singularity-tests", "bitbucket-tests" ]
+            <<: *common_filters
+            <<: *slack_context
+            requires:
+              - build
+        - non-confidential-tests:
+            <<: *common_filters
+            <<: *slack_context
+            requires:
+              - build
+        - confidential-workflow-tests:
+            <<: *common_filters
+            <<: *slack_context
+            requires:
+              - build
+        - confidential-tool-tests:
+            <<: *common_filters
+            <<: *slack_context
+            requires:
+              - build
+        - wes-toil-test:
+            <<: *common_filters
+            requires:
+              - build
+        - sonar-cloud:
+            <<: *common_filters
+            requires:
+              - unit-tests
+              - integration-tests
+              - non-confidential-tests
+              - confidential-workflow-tests
+              - confidential-tool-tests
+              - wes-toil-test
+            context:
+              - sonarcloud
+
 workflows:
   version: 2
+
   everything:
-    jobs:
-      - build:
-          <<: *common_filters
-
-      - unit-tests:
-          <<: *common_filters
-          requires:
-            - build
-      - integration-tests:
-          matrix:
-            parameters:
-              testing_profile: ["singularity-tests", "bitbucket-tests"]
-          <<: *common_filters
-          requires:
-            - build
-      - non-confidential-tests:
-          <<: *common_filters
-          requires:
-            - build
-      - confidential-workflow-tests:
-          <<: *common_filters
-          requires:
-            - build
-      - confidential-tool-tests:
-          <<: *common_filters
-          requires:
-            - build
-      - wes-toil-test:
-          <<: *common_filters
-          requires:
-            - build
-      - sonar-cloud:
-          <<: *common_filters
-          requires:
-            - unit-tests
-            - integration-tests
-            - non-confidential-tests
-            - confidential-workflow-tests
-            - confidential-tool-tests
-            - wes-toil-test
-          context:
-            - sonarcloud
+    when:
+      not: << pipeline.parameters.run_against_develop_core >>
+    <<: *common_jobs
 
+  nightly:
+    when: << pipeline.parameters.run_against_develop_core >>
+    <<: *common_jobs
 
 jobs:
   unit-tests:
@@ -84,11 +107,13 @@ jobs:
       TESTING_PROFILE: unit-tests
       MAVEN_GOAL: test
     steps:
+      - restore_bash_env
       - setup_for_unit_tests
       - run_tests
       - save_test_results
       - send_coverage
       - persist_coverage
+      - notify-slack
 
   non-confidential-tests:
     executor: machine_integration_test_exec
@@ -166,9 +191,36 @@ jobs:
       - install_yq
       - install-git-secrets
       - install_confidential_test_data
+      - restore_bash_env
+      - when:
+          condition: << pipeline.parameters.run_against_develop_core >>
+          steps:
+            - run:
+                name: findDevelopSnapshot
+                command: |
+                  ./mvnw versions:update-properties -Dincludes=io.dockstore:* -DallowSnapshots
+            - run:
+                name: force update to latest snapshot versions, # review this step to see what versions were used, also see https://stackoverflow.com/questions/29020716/maven-what-does-u-update-snapshots-really-do
+                command: |
+                  ./mvnw clean install -DskipTests -U
+      - when:
+          condition:
+            not: << pipeline.parameters.run_against_develop_core >>
+          steps:
+            - run:
+                name: build
+                command: |
+                  ./mvnw -B clean install -DskipTests
+
       - run:
-          name: build
-          command: ./mvnw -B clean install -DskipTests
+          name: Check imports are sorted
+          command: |
+            if [[ $(git diff --name-only | grep "\.java$") != '' ]]; then
+              echo "At least one Java file didn't have its imports sorted"
+              git diff --stat
+              exit 1
+            fi
+
 
         # Running scan must occur after build
       - run_git_secrets_scan
@@ -183,6 +235,8 @@ jobs:
           root: .
           paths:
             - .
+      - notify-slack
+
   sonar-cloud:
     docker: # run the steps with Docker
       - image: cimg/openjdk:<< pipeline.parameters.java-tag >>
@@ -222,6 +276,15 @@ commands:
             sudo NEEDRESTART_MODE=a apt install openssl
             bash scripts/decrypt.sh
 
+  restore_bash_env:
+    steps:
+      - attach_workspace:
+          at: .
+      - run:
+          name: Restore bash environment from bash.env
+          command: !
+            cat bash.env >> $BASH_ENV || true
+
   send_coverage:
     steps:
       - run:
@@ -251,16 +314,20 @@ commands:
 
 
   check_pom_files:
-    steps:
-      - run:
-          name: check generated flattened POMs match checked-in files.
-          command: |
-            # need better solution, but leaving this as-is fouls up the release since the Maven release plugin doesn't know about the flattened poms
-            if [[ -z "${CIRCLE_TAG}" ]]; then
-              scripts/check_poms.sh
-            fi
-          environment:
-            TESTING_PROFILE: automated-review
+        steps:
+          - when:
+              condition:
+                not: << pipeline.parameters.run_against_develop_core >>
+              steps:
+                - run:
+                    name: check generated flattened POMs match checked-in files.
+                    command: |
+                      # need better solution, but leaving this as-is fouls up the release since the Maven release plugin doesn't know about the flattened poms
+                      if [[ -z "${CIRCLE_TAG}" ]]; then
+                        scripts/check_poms.sh
+                      fi
+                    environment:
+                      TESTING_PROFILE: automated-review
 
   run_git_secrets_scan:
     steps:
@@ -475,10 +542,12 @@ commands:
   setup_and_run_integration_tests:
     steps:
       - setup_for_integration_tests
+      - restore_bash_env
       - run_tests
       - save_test_results
       - send_coverage
       - persist_coverage
+      - notify-slack
 
   persist_coverage:
     steps:
@@ -491,3 +560,13 @@ commands:
           root: .
           paths:
             - coverage
+
+  notify-slack:
+    steps:
+      - when:
+          condition: << pipeline.parameters.run_against_develop_core >>
+          steps:
+            - slack/notify:
+                channel: $slack_id
+                event: fail
+                template: basic_fail_1

.github/workflows/deploy_snapshot.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

.github/workflows/mvnw.yml

@@ -23,7 +23,7 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: 'temurin'
       # Step that does that actual cache save and restore
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -33,9 +33,10 @@ jobs:
         
       - if: "!contains(github.ref, 'dependabot')"
         name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
-          languages: java           
+          languages: java
+          queries: security-and-quality
       - name: Build with mvnw
         run: ./mvnw clean install
 
@@ -44,4 +45,4 @@ jobs:
 
       - if: "!contains(github.ref, 'dependabot')"
         name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2            
+        uses: github/codeql-action/analyze@v3            

README.md

@@ -30,7 +30,7 @@ The following section is useful for users of Dockstore (e.g. those that want to
 launch tools). 
 
 After registering at [dockstore.org](https://dockstore.org), you will be able to download the Dockstore 
-CLI at https://dockstore.org/onboarding
+CLI at [onboarding](https://dockstore.org/onboarding). The Dockstore command line script should be installed in a location in your path.
 
 ### Configuration File
 
@@ -41,6 +41,8 @@ token = <your generated by the dockstore site>
 server-url = https://www.dockstore.org/api
 ```
 
+For developers, if you are working with a custom-built or updated Dockstore client you will need to update the jar in: `~/.dockstore/config/self-installs`.
+
 ### File Provisioning
 
 By default, cwltool reads input files from the local filesystem. Dockstore also adds support for additional file systems
@@ -67,9 +69,6 @@ cache-dir = /home/<user>/.dockstore/cache   #set this to determine where input f
 
 [dockstore-file-s3-plugin]
 endpoint = #set this to point at a non AWS S3 endpoint
-
-[dockstore-file-icgc-storage-client-plugin]
-client = /media/large_volume/icgc-storage-client-1.0.23/bin/icgc-storage-client
 ```
 
 Additional plugins can be created by taking one of the repos in [plugins](https://github.com/dockstore) as a model and