Merge pull request #89 from Plant-Food-Research-Open/dev

Candidate for 2.0.0

.devcontainer/devcontainer.json

@@ -10,15 +10,7 @@
         "vscode": {
             // Set *default* container specific settings.json values on container create.
             "settings": {
-                "python.defaultInterpreterPath": "/opt/conda/bin/python",
-                "python.linting.enabled": true,
-                "python.linting.pylintEnabled": true,
-                "python.formatting.autopep8Path": "/opt/conda/bin/autopep8",
-                "python.formatting.yapfPath": "/opt/conda/bin/yapf",
-                "python.linting.flake8Path": "/opt/conda/bin/flake8",
-                "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
-                "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
-                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+                "python.defaultInterpreterPath": "/opt/conda/bin/python"
             },
 
             // Add the IDs of extensions you want installed when the container is created.

.editorconfig

@@ -18,17 +18,32 @@ end_of_line = unset
 insert_final_newline = unset
 trim_trailing_whitespace = unset
 indent_style = unset
-indent_size = unset
+[/subworkflows/nf-core/**]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
+
+# These files are edited and tested upstream in pfr/modules
+[/modules/pfr/**]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
+[/subworkflows/pfr/**]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
 
 [/assets/email*]
 indent_size = unset
 
-# ignore Readme
-[README.md]
-indent_style = unset
-
-# ignore python
-[*.{py}]
+# ignore python and markdown
+[*.{py,md}]
 indent_style = unset
 
 # ignore perl

.github/CONTRIBUTING.md

@@ -23,8 +23,11 @@ If you're not used to this workflow with git, you can start with some [docs from
 
 ## Tests
 
-You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to
-receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir <OUTDIR>`.
+You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command:
+
+```bash
+nf-test test --profile debug,test,docker --verbose
+```
 
 When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests.
 Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then.
@@ -81,7 +84,7 @@ Once there, use `nf-core schema build` to add to `nextflow_schema.json`.
 
 Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels.
 
-The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block.
+The process resources can be passed on to the tool dynamically within the process with the `${task.cpus}` and `${task.memory}` variables in the `script:` block.
 
 ### Naming schemes
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,7 +16,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/plant-food-r
 - [ ] This comment contains a description of changes (with reason).
 - [ ] If you've fixed a bug or added code that should be tested, add tests!
 - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/plant-food-research-open/assemblyqc/tree/master/.github/CONTRIBUTING.md)
-- [ ] Make sure your code lints (`nf-core lint`) and (`pre-commit run --all`)
+- [ ] Make sure your code lints (`nf-core lint`).
 - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
 - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.

.github/workflows/branch.yml

@@ -19,7 +19,7 @@ jobs:
       # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
       - name: Post PR comment
         if: failure()
-        uses: mshick/add-pr-comment@v2
+        uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2
         with:
           message: |
             ## This PR is against the `master` branch :x:

.github/workflows/ci.yml

@@ -25,19 +25,32 @@ jobs:
       matrix:
         NXF_VER:
           - "23.04.0"
-        RUN_CONFIG:
-          - "-profile test,docker"
-          - "-profile docker -c ./tests/stub/stub.config -stub"
-          - "-profile docker -params-file ./tests/invalid/params.json"
+        TEST_PARAMS:
+          - minimal
+          - invalid
+          - stub
+        include:
+          - OPTION_STUB: ""
+          - OPTION_STUB: "-stub"
+            TEST_PARAMS: stub
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v4
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
 
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
         with:
           version: ${{ matrix.NXF_VER }}
 
+      - name: Disk space cleanup
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+
       - name: Run pipeline with test data
         run: |
-          nextflow run ${GITHUB_WORKSPACE} --outdir ./results ${{ matrix.RUN_CONFIG }}
+          nextflow run \
+            ${GITHUB_WORKSPACE} \
+            --outdir ./results \
+            -profile docker \
+            -params-file \
+            ./tests/${{ matrix.TEST_PARAMS }}/params.json \
+            ${{ matrix.OPTION_STUB }}

.github/workflows/clean-up.yml

@@ -10,7 +10,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9
         with:
           stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days."
           stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful."

.github/workflows/download_pipeline.yml

@@ -6,9 +6,16 @@ name: Test successful pipeline download with 'nf-core download'
 #  - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev.
 on:
   workflow_dispatch:
+    inputs:
+      testbranch:
+        description: "The specific branch you wish to utilize for the test execution of nf-core download."
+        required: true
+        default: "dev"
   pull_request:
     types:
       - opened
+      - edited
+      - synchronize
     branches:
       - master
   pull_request_target:
@@ -20,16 +27,19 @@ env:
 
 jobs:
   download:
-    runs-on: ["self-hosted"]
+    runs-on: ubuntu-latest
     steps:
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
 
-      - uses: actions/setup-python@v5
+      - name: Disk space cleanup
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           architecture: "x64"
-      - uses: eWaterCycle/setup-singularity@v7
+      - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7
         with:
           singularity-version: 3.8.3
 
@@ -42,13 +52,13 @@ jobs:
         run: |
           echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV}
           echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV}
-          echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV}
+          echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV}
 
       - name: Download the pipeline
         env:
           NXF_SINGULARITY_CACHEDIR: ./
         run: |
-          nf-core download  ${{ env.REPO_LOWERCASE }} \
+          nf-core download ${{ env.REPO_LOWERCASE }} \
           --revision ${{ env.REPO_BRANCH }} \
           --outdir ./${{ env.REPOTITLE_LOWERCASE }} \
           --compress "none" \
@@ -60,8 +70,17 @@ jobs:
       - name: Inspect download
         run: tree ./${{ env.REPOTITLE_LOWERCASE }}
 
-      - name: Run the downloaded pipeline
+      - name: Run the downloaded pipeline (stub)
+        id: stub_run_pipeline
+        continue-on-error: true
         env:
           NXF_SINGULARITY_CACHEDIR: ./
           NXF_SINGULARITY_HOME_MOUNT: true
         run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results
+      - name: Run the downloaded pipeline (stub run not supported)
+        id: run_pipeline
+        if: ${{ job.steps.stub_run_pipeline.status == failure() }}
+        env:
+          NXF_SINGULARITY_CACHEDIR: ./
+          NXF_SINGULARITY_HOME_MOUNT: true
+        run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results

.github/workflows/fix-linting.yml

@@ -4,7 +4,7 @@ on:
     types: [created]
 
 jobs:
-  deploy:
+  fix-linting:
     # Only run if comment is on a PR with the main repo, and if it contains the magic keywords
     if: >
       contains(github.event.comment.html_url, '/pull/') &&
@@ -13,36 +13,77 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Use the @nf-core-bot token to check out so we can push later
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
         with:
           token: ${{ secrets.nf_core_bot_auth_token }}
 
+      # indication that the linting is being fixed
+      - name: React on comment
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: eyes
+
       # Action runs on the issue comment, so we don't get the PR by default
       # Use the gh cli to check out the PR
       - name: Checkout Pull Request
         run: gh pr checkout ${{ github.event.issue.number }}
         env:
           GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
 
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+      # Install and run pre-commit
+      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
-          python-version: 3.11
-          cache: "pip"
+          python-version: "3.12"
 
       - name: Install pre-commit
         run: pip install pre-commit
 
       - name: Run pre-commit
-        run: pre-commit run --all-files || echo "status=fail" >> $GITHUB_ENV
+        id: pre-commit
+        run: pre-commit run --all-files
+        continue-on-error: true
+
+      # indication that the linting has finished
+      - name: react if linting finished succesfully
+        if: steps.pre-commit.outcome == 'success'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: "+1"
 
       - name: Commit & push changes
-        if: env.status == 'fail'
+        id: commit-and-push
+        if: steps.pre-commit.outcome == 'failure'
         run: |
           git config user.email "[email protected]"
           git config user.name "nf-core-bot"
           git config push.default upstream
           git add .
           git status
-          git commit -m "[automated] Fix linting with pre-commit"
+          git commit -m "[automated] Fix code linting"
           git push
+
+      - name: react if linting errors were fixed
+        id: react-if-fixed
+        if: steps.commit-and-push.outcome == 'success'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: hooray
+
+      - name: react if linting errors were not fixed
+        if: steps.commit-and-push.outcome == 'failure'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: confused
+
+      - name: react if linting errors were not fixed
+        if: steps.commit-and-push.outcome  == 'failure'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually.
+            See [CI log](https://github.com/plant-food-research-open/assemblyqc/actions/runs/${{ github.run_id }}) for more details.