Rebuild 100k sample #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rebuild 100k sample | |
| on: | |
| # cron job once a week on Mondays at 12:42 UTC | |
| schedule: | |
| - cron: '42 12 * * 1' | |
| # Manually triggered using GitHub's UI | |
| workflow_dispatch: | |
| jobs: | |
| gisaid: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: nextstrain/.github/actions/setup-nextstrain-cli@master | |
| with: | |
| python-version: "3.10" | |
| - name: Launch GISAID build | |
| run: | | |
| set -x | |
| declare -a config | |
| config+=(slack_token=$SLACK_TOKEN) | |
| nextstrain build \ | |
| --aws-batch \ | |
| --detach \ | |
| --cpus 16 \ | |
| --memory 31GiB \ | |
| . \ | |
| upload \ | |
| --configfile nextstrain_profiles/100k/config-gisaid.yaml \ | |
| --config "${config[@]}" \ | |
| --set-threads tree=8 \ | |
| |& tee build-launch-gisaid.log | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} | |
| - name: Launch open build | |
| run: | | |
| set -x | |
| declare -a config | |
| config+=(slack_token=$SLACK_TOKEN) | |
| nextstrain build \ | |
| --aws-batch \ | |
| --detach \ | |
| --cpus 16 \ | |
| --memory 31GiB \ | |
| . \ | |
| upload \ | |
| --configfile nextstrain_profiles/100k/config-open.yaml \ | |
| --config "${config[@]}" \ | |
| --set-threads tree=8 \ | |
| |& tee build-launch-open.log | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} | |
| - name: Build info | |
| run: | | |
| echo "--> 100k samples for GISAID + Open data rebuilding (using separate AWS jobs)" | |
| echo | |
| echo "--> When completed, the following files will be updated:" | |
| echo "s3://nextstrain-data/files/ncov/open/100k/metadata.tsv.xz" | |
| echo "s3://nextstrain-data/files/ncov/open/100k/sequences.fasta.xz" | |
| echo "s3://nextstrain-ncov-private/100k/metadata.tsv.xz" | |
| echo "s3://nextstrain-ncov-private/100k/sequences.fasta.xz" | |
| echo | |
| echo "--> You can attach to the GISAID AWS job via:" | |
| tail -n1 build-launch-gisaid.log | |
| echo | |
| echo "--> You can attach to the Open AWS job via:" | |
| tail -n1 build-launch-open.log | |
| echo | |
| JOBID=$( tail -n1 build-launch-gisaid.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) | |
| echo "--> View the GISAID job in the AWS console via" | |
| echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" | |
| echo | |
| JOBID=$( tail -n1 build-launch-open.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' ) | |
| echo "--> View the Open job in the AWS console via" | |
| echo " https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}" | |
| echo |