From b6658787f4ca6a0317fff3e6a8f0956843a1b225 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 1 Jun 2023 13:30:16 -0400 Subject: [PATCH 1/7] use micromamba --- .github/workflows/test.yml | 45 ++++++++++++++------------------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a0010f..707943e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ on: defaults: run: - shell: bash -l {0} + shell: bash -el {0} jobs: build: @@ -24,17 +24,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/conda_pkgs_dir - /usr/share/miniconda/envs/roman-data-workshop-env - key: conda-${{ runner.os }}-${{ hashFiles('00_install/environment.yml') }}-${{ github.sha }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-micromamba@v1 with: - activate-environment: roman-data-workshop-env + environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml - use-only-tar-bz2: true + cache-environment: true + cache-downloads: true + init-shell: bash - run: python ./00_install/verify_install.py - run: conda env export > roman-data-workshop-env-${{ github.sha }}.yml - uses: actions/upload-artifact@v3.1.0 @@ -52,18 +48,13 @@ jobs: with: path: data/ key: data-${{ hashFiles('data/download.py') }} - - uses: actions/cache@v3 + - uses: conda-incubator/setup-micromamba@v1 with: - path: | - ~/conda_pkgs_dir - /usr/share/miniconda/envs/roman-data-workshop-env - key: conda-${{ runner.os }}-${{ hashFiles('00_install/environment.yml') }}-${{ github.sha }} - if: steps.data_cache.outputs.cache-hit != 'true' - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: roman-data-workshop-env + environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml - use-only-tar-bz2: true + cache-environment: true + cache-downloads: true + init-shell: bash if: steps.data_cache.outputs.cache-hit != 'true' - run: python data/download.py if: steps.data_cache.outputs.cache-hit != 'true' @@ -78,17 +69,13 @@ jobs: CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 steps: - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/conda_pkgs_dir - /usr/share/miniconda/envs/roman-data-workshop-env - key: conda-${{ runner.os }}-${{ hashFiles('00_install/environment.yml') }}-${{ github.sha }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-micromamba@v1 with: - activate-environment: roman-data-workshop-env + environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml - use-only-tar-bz2: true + cache-environment: true + cache-downloads: true + init-shell: bash - run: echo "::set-output name=crds_context::$(crds list --operational-context)" id: crds_context - uses: actions/cache@v3 From 084d9b35bf9792cb4478f600d3f59e348b416849 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 1 Jun 2023 13:31:01 -0400 Subject: [PATCH 2/7] use GITHUB_OUTPUT --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 707943e..c81b579 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: cache-environment: true cache-downloads: true init-shell: bash - - run: echo "::set-output name=crds_context::$(crds list --operational-context)" + - run: echo "name=crds_context::$(crds list --operational-context)" >> $GITHUB_OUTPUT id: crds_context - uses: actions/cache@v3 with: From 677fa8f194a73f411f9ca0cef696cd50ab03f653 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 1 Jun 2023 13:32:43 -0400 Subject: [PATCH 3/7] fix action name --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c81b579..9bb91c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: conda-incubator/setup-micromamba@v1 + - uses: mamba-org/setup-micromamba@v1 with: environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml @@ -48,7 +48,7 @@ jobs: with: path: data/ key: data-${{ hashFiles('data/download.py') }} - - uses: conda-incubator/setup-micromamba@v1 + - uses: mamba-org/setup-micromamba@v1 with: environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml @@ -69,7 +69,7 @@ jobs: CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 steps: - uses: actions/checkout@v3 - - uses: conda-incubator/setup-micromamba@v1 + - uses: mamba-org/setup-micromamba@v1 with: environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml From f1e8ef23aaf971660fd3704c3f2d46917c1a5ae4 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 1 Jun 2023 13:48:40 -0400 Subject: [PATCH 4/7] remove -e --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9bb91c9..7ced990 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ on: defaults: run: - shell: bash -el {0} + shell: bash -l {0} jobs: build: From 47f80d564bb4db75d250c2b426ec12c4457cde09 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 1 Jun 2023 13:49:50 -0400 Subject: [PATCH 5/7] use workflow concurrency --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ced990..a9f117d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,10 @@ on: - '**/*.yaml' - '**/*.py' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + defaults: run: shell: bash -l {0} From d0d9f3957919474704cd6359a3b72b5f586d44dc Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Thu, 1 Feb 2024 16:15:40 -0500 Subject: [PATCH 6/7] add conda to environment --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9f117d..de13c03 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,8 @@ jobs: with: environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml + create-args: >- + conda cache-environment: true cache-downloads: true init-shell: bash @@ -56,6 +58,8 @@ jobs: with: environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml + create-args: >- + conda cache-environment: true cache-downloads: true init-shell: bash @@ -77,6 +81,8 @@ jobs: with: environment-name: roman-data-workshop-env environment-file: 00_install/environment.yml + create-args: >- + conda cache-environment: true cache-downloads: true init-shell: bash From 5521c0e8e3bcb2e86631c919ea1596ac682e3df5 Mon Sep 17 00:00:00 2001 From: zacharyburnett Date: Thu, 1 Feb 2024 16:24:52 -0500 Subject: [PATCH 7/7] cache based on date --- .github/workflows/test.yml | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de13c03..d6c3ac6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,10 +20,19 @@ concurrency: defaults: run: - shell: bash -l {0} + shell: micromamba-shell {0} jobs: + date: + runs-on: ubuntu-latest + steps: + - run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT + id: date + shell: bash + outputs: + date: ${{ steps.date.outputs.date }} build: + needs: [ date ] name: build environment runs-on: ubuntu-latest steps: @@ -34,9 +43,12 @@ jobs: environment-file: 00_install/environment.yml create-args: >- conda - cache-environment: true cache-downloads: true - init-shell: bash + cache-downloads-key: downloads-${{ needs.date.outputs.date }} + cache-environment: true + cache-environment-key: environment-${{ needs.date.outputs.date }} + init-shell: none + generate-run-shell: true - run: python ./00_install/verify_install.py - run: conda env export > roman-data-workshop-env-${{ github.sha }}.yml - uses: actions/upload-artifact@v3.1.0 @@ -44,8 +56,8 @@ jobs: name: roman-data-workshop-env-${{ github.sha }}.yml path: roman-data-workshop-env-${{ github.sha }}.yml data: + needs: [ date, build ] name: download data - needs: [ build ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -60,15 +72,18 @@ jobs: environment-file: 00_install/environment.yml create-args: >- conda - cache-environment: true cache-downloads: true - init-shell: bash + cache-downloads-key: downloads-${{ needs.date.outputs.date }} + cache-environment: true + cache-environment-key: environment-${{ needs.date.outputs.date }} + init-shell: none + generate-run-shell: true if: steps.data_cache.outputs.cache-hit != 'true' - run: python data/download.py if: steps.data_cache.outputs.cache-hit != 'true' test: + needs: [ date, build, data ] name: run notebooks - needs: [ build, data ] runs-on: ubuntu-latest env: CRDS_SERVER_URL: https://roman-crds-test.stsci.edu @@ -83,9 +98,12 @@ jobs: environment-file: 00_install/environment.yml create-args: >- conda - cache-environment: true cache-downloads: true - init-shell: bash + cache-downloads-key: downloads-${{ needs.date.outputs.date }} + cache-environment: true + cache-environment-key: environment-${{ needs.date.outputs.date }} + init-shell: none + generate-run-shell: true - run: echo "name=crds_context::$(crds list --operational-context)" >> $GITHUB_OUTPUT id: crds_context - uses: actions/cache@v3