diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml new file mode 100644 index 00000000000..4dd6df48919 --- /dev/null +++ b/.github/actions/nf-test-action/action.yml @@ -0,0 +1,83 @@ +name: "nf-test Action" +description: "Runs nf-test with common setup steps" +inputs: + path: + description: "Path to test" + required: true + profile: + description: "Profile to use" + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: "0.9.2" + install-pdiff: true + + - name: Setup apptainer + if: contains(inputs.profile, 'singularity') + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: contains(inputs.profile, 'singularity') + shell: bash + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Conda setup + if: ${{inputs.profile == 'conda'}} + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 + with: + auto-update-conda: true + conda-solver: libmamba + conda-remove-defaults: true + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' + shell: bash + run: | + nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + + - name: Run nf-test + shell: bash + env: + SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + run: | + NFT_WORKDIR=~ \ + nf-test test \ + --profile=${{ inputs.profile }} \ + --tap=test.tap \ + --verbose \ + ${{ inputs.path }} + - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 + if: ${{ inputs.path != '' }} + with: + path: >- + test.tap + + - name: Clean up + if: always() + shell: bash + run: | + sudo rm -rf /home/ubuntu/tests/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c22ed82bed..0ff73656067 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 # FIXME Flip this off once we get to less than a couple hundred. Adding # this so it will only run against changed files. It'll make it much @@ -86,8 +88,7 @@ jobs: with: fetch-depth: 2 # To retrieve the preceding commit. - # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented - - uses: mirpedrol/paths-filter@main + - uses: dorny/paths-filter@v3 id: filter with: filters: "tests/config/pytest_modules.yml" @@ -135,6 +136,7 @@ jobs: - name: debug run: | + echo ${{ steps.filter.outputs.components }} echo ${{ steps.outputs.outputs.modules }} echo ${{ steps.outputs.outputs.subworkflows }} @@ -441,6 +443,7 @@ jobs: miniconda-version: "latest" channels: conda-forge,bioconda python-version: ${{ matrix.python-version }} + conda-remove-defaults: true - name: Conda setup run: | @@ -485,7 +488,7 @@ jobs: !${{ github.workspace }}/.singularity nf-test: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: nf-test needs: [nf-test-changes] if: ( needs.nf-test-changes.outputs.paths != '[]' ) @@ -670,233 +673,47 @@ jobs: env: NXF_ANSI_LOG: false - NFTEST_VER: "0.9.0" - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip-pdiff - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pdiff - restore-keys: | - ${{ runner.os }}-pip-pdiff - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pdiff cryptography - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,bioconda - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Set up secrets - - name: Set up nextflow secrets - # TODO Only run if the tag includes `sentieon` - if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null - run: | - nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") - - # Test the module - - name: Run nf-test - if: ${{ matrix.path != '' }} - env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" - run: | - # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker - if [ "${{ matrix.profile }}" == "docker" ]; then - PROFILE="docker_self_hosted" - else - PROFILE=${{ matrix.profile }} - fi - - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }} \ - --tap=test.tap \ - --verbose \ - ${{ matrix.path }} - - - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 - if: ${{ matrix.path != '' }} + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action with: - path: >- - test.tap - - - name: Clean up - if: always() - run: | - sudo rm -rf /home/ubuntu/tests/ + path: ${{ matrix.path }} + profile: ${{ matrix.profile }} nf-test-gpu: - runs-on: ${{ github.event.inputs.runners || 'gpu' }} - name: nf-test with GPUs + runs-on: "gpu" + name: nf-test-gpu needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) + if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') ) strategy: fail-fast: false matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] - profile: [conda, docker_self_hosted, singularity] include: - path: modules/nf-core/parabricks/applybqsr + profile: [docker_self_hosted, singularity] - path: modules/nf-core/parabricks/fq2bam - # - path: modules/nf-core/parabricks/fq2bammeth excluded - tackle in different PR - exclude: - - profile: conda - path: modules/nf-core/parabricks/fq2bam - - profile: conda - path: modules/nf-core/parabricks/fq2bammeth - - profile: conda - path: modules/nf-core/parabricks/applybqsr + profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/fq2bammeth + profile: [docker_self_hosted, singularity] env: NXF_ANSI_LOG: false - NFTEST_VER: "0.9.0" - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip-pdiff - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pdiff - restore-keys: | - ${{ runner.os }}-pip-pdiff - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pdiff cryptography - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,bioconda - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Set up secrets - - name: Set up nextflow secrets - # TODO Only run if the tag includes `sentieon` - if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null - run: | - nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") - # Test the module - - name: Run nf-test - if: ${{ matrix.path != '' }} + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" - run: | - # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker - if [ "${{ matrix.profile }}" == "docker" ]; then - PROFILE="docker_self_hosted" - else - PROFILE=${{ matrix.profile }} - fi - - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }},gpu \ - --tap=test.tap \ - --verbose \ - ${{ matrix.path }} - - - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 - if: ${{ matrix.path != '' }} with: - path: >- - test.tap - - - name: Clean up - if: always() - run: | - sudo rm -rf /home/ubuntu/tests/ + path: ${{ matrix.path }} + profile: ${{ matrix.profile }},gpu confirm-pass: runs-on: ubuntu-latest @@ -910,6 +727,7 @@ jobs: pytest, nf-test-changes, nf-test, + nf-test-gpu, ] if: always() steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d9bf144771..b97fd79f3a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: prettier additional_dependencies: - - prettier@3.2.5 + - prettier@3.3.3 - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.29.4 hooks: diff --git a/modules/nf-core/fastqc/meta.yml b/modules/nf-core/fastqc/meta.yml index 4827da7af24..2b2e62b8aea 100644 --- a/modules/nf-core/fastqc/meta.yml +++ b/modules/nf-core/fastqc/meta.yml @@ -11,6 +11,7 @@ tools: FastQC gives general quality metrics about your reads. It provides information about the quality score distribution across your reads, the per base sequence content (%A/C/G/T). + You get information about adapter contamination and other overrepresented sequences. homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index de10acc9fb2..c86fb134d75 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -25,11 +25,6 @@ if ("$PROFILE" == "singularity") { podman.enabled = true podman.userEmulation = true podman.runOptions = "--runtime crun --platform linux/x86_64 --systemd=always" -} else if ("$PROFILE" == "gpu") { - docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' - apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' - singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' - use_gpu = true } else { docker.enabled = true docker.userEmulation = false diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 1bdf57d23c1..f651ff4d89d 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -43,6 +43,12 @@ profiles { docker.fixOwnership = true docker.runOptions = '--platform=linux/amd64' } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + apptainer.runOptions = '--nv' + singularity.runOptions = '--nv' + use_gpu = true + } } docker.registry = 'quay.io'