Build Recipes #399
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Recipes | |
| on: | |
| push: | |
| branches: [ main ] | |
| tags: | |
| - '*' | |
| pull_request: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| schedule: | |
| # Twice per week, on Monday and Thursday | |
| - cron: "0 0 * * 1,4" | |
| permissions: | |
| contents: write | |
| env: | |
| # Increase this value to reset cache if environment.yml has not changed | |
| CONDA_CACHE_NUMBER: 0 | |
| # Increase this value to reset cache if emscripten_version has not changed | |
| EMSDK_CACHE_FOLDER: 'emsdk-cache' | |
| EMSDK_CACHE_NUMBER: 0 | |
| CCACHE_DIR: /tmp/.ccache | |
| CCACHE_CACHE_NUMBER: 0 | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install Emscripten ccache | |
| run: | | |
| # FIXME: Installing ccache using `emsdk install ccache-git-emscripten-64bit` doesn't work well in conda env: | |
| # https://stackoverflow.com/questions/71340058/conda-does-not-look-for-libpthread-and-libpthread-nonshared-at-the-right-place-w | |
| git clone https://github.com/juj/ccache -b emscripten --depth 1 | |
| cd ccache | |
| cmake . | |
| make ccache | |
| export PATH=$(pwd):$PATH | |
| cd .. | |
| which ccache | |
| - name: Cache conda | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/conda_pkgs_dir | |
| key: | |
| ${{ runner.os }}-conda-${{ env.CONDA_CACHE_NUMBER }}-${{ | |
| hashFiles('environment.yml') }} | |
| - name: Setup Mambaforge | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| activate-environment: pyodide-env | |
| use-mamba: true | |
| - name: Get Date | |
| id: get-date | |
| run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT | |
| - name: Cache Conda env | |
| id: conda-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CONDA }}/envs | |
| key: | |
| conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('environment.yml') }}-${{ env.CONDA_CACHE_NUMBER }} | |
| - name: Update environment if cache miss | |
| run: | |
| mamba env update -n pyodide-env -f environment.yml | |
| if: steps.conda-cache.outputs.cache-hit != 'true' | |
| # ref. https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh#L28-L53 | |
| - name: Free unused disk space | |
| run: | | |
| # delete packages | |
| sudo apt-get remove -y '^dotnet-.*' || true | |
| sudo apt-get remove -y '^llvm-.*' || true | |
| sudo apt-get remove -y '^temurin-.*' || true | |
| sudo apt-get remove -y '^mysql-server-core-.*' || true | |
| sudo apt-get remove -y '^postgresql-.*' || true | |
| sudo apt-get remove -y azure-cli google-chrome-stable google-cloud-cli firefox powershell microsoft-edge-stable mono-devel || true | |
| sudo apt-get autoremove -y | |
| sudo apt-get clean | |
| # delete directories | |
| sudo rm -rf /usr/share/dotnet/ | |
| sudo rm -rf /usr/share/swift/ | |
| sudo rm -rf /usr/local/graalvm/ | |
| sudo rm -rf /usr/local/.ghcup/ | |
| sudo rm -rf /usr/local/share/powershell | |
| sudo rm -rf /usr/local/share/chromium | |
| sudo rm -rf /usr/local/lib/android | |
| - name: Install latest pyodide-build and build dependencies | |
| run : | | |
| which python | |
| python -m pip install pyodide-build | |
| pyodide xbuildenv install --url http://pyodide-cache.s3-website-us-east-1.amazonaws.com/xbuildenv/dev/xbuildenv.tar.bz2 | |
| - name: Check emscripten version | |
| run: | | |
| echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV | |
| - name: Cache emsdk | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{env.EMSDK_CACHE_FOLDER}} | |
| key: ${{env.EMSDK_CACHE_NUMBER}}-${{env.EMSCRIPTEN_VERSION}}-${{ runner.os }} | |
| - uses: mymindstorm/setup-emsdk@v14 | |
| with: | |
| version: ${{ env.EMSCRIPTEN_VERSION }} | |
| actions-cache-folder: ${{env.EMSDK_CACHE_FOLDER}} | |
| - name: Set ccache suffix | |
| run: | | |
| # This step makes the cache key in the main branch and PRs different. | |
| # main branch run a full build, while PRs run a partial build | |
| # so we keep the ccache cache separate | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| echo CCACHE_SUFFIX="-pr" >> $GITHUB_ENV | |
| fi | |
| - name: Cache ccache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: ${{ env.CCACHE_DIR }}-${{ env.CCACHE_CACHE_NUMBER }}-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }}${{ env.CCACHE_SUFFIX }} | |
| restore-keys: | | |
| ${{ env.CCACHE_DIR }}-${{ env.CCACHE_CACHE_NUMBER }}-${{ env.EMSCRIPTEN_VERSION }}-${{ runner.os }} | |
| - name: Calculate recipes to build (pull_request) | |
| if: github.event_name == 'pull_request' | |
| id: calculate_recipes_pr | |
| run: | | |
| export CHANGED_RECIPES=$(python ./tools/calc_diff.py \ | |
| --base origin/${{ github.event.pull_request.base.ref }} \ | |
| --target ${{ github.sha }}) | |
| echo "Changed recipes: $CHANGED_RECIPES" | |
| # If there are no changed recipes, we build only core packages sets | |
| if [ -z "$CHANGED_RECIPES" ]; then | |
| echo "recipes=tag:core" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "recipes=$CHANGED_RECIPES,tag:core" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Build recipes (full) | |
| if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || contains(github.event.head_commit.message, '[full build]') }} | |
| run: | | |
| ccache -z | |
| export _EMCC_CACHE=1 | |
| export PIP_CONSTRAINT=$(pwd)/tools/constraints.txt | |
| pyodide build-recipes "*" --install --install-dir=./repodata | |
| ccache -s | |
| - name: Build recipes (changed only) | |
| if: github.event_name == 'pull_request' && !contains(github.event.head_commit.message, '[full build]') | |
| run: | | |
| ccache -z | |
| export _EMCC_CACHE=1 | |
| export PIP_CONSTRAINT=$(pwd)/tools/constraints.txt | |
| pyodide build-recipes ${{ steps.calculate_recipes_pr.outputs.recipes }} --install --install-dir=./repodata | |
| ccache -s | |
| - name: Store artifacts build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: repodata | |
| path: ./repodata/ | |
| retention-days: 15 | |
| - name: Compress build artifacts | |
| run: | | |
| tar -czvf packages.tar.gz -C repodata . | |
| - name: Release | |
| uses: softprops/action-gh-release@v2 | |
| if: startsWith(github.ref, 'refs/tags/') | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| files: | | |
| ./packages.tar.gz | |
| ./repodata/pyodide-lock.json | |
| test: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| needs: [build] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| test-config: [ | |
| {runner: selenium, runtime: chrome, runtime-version: 126 }, | |
| ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Cache conda | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/conda_pkgs_dir | |
| key: | |
| ${{ runner.os }}-conda-${{ env.CONDA_CACHE_NUMBER }}-${{ | |
| hashFiles('environment.yml') }} | |
| - name: Setup Mambaforge | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| activate-environment: pyodide-env | |
| use-mamba: true | |
| - name: Get Date | |
| id: get-date | |
| run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT | |
| - name: Cache Conda env | |
| id: conda-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CONDA }}/envs | |
| key: | |
| conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('environment.yml') }}-${{ env.CONDA_CACHE_NUMBER }} | |
| - name: Update environment if cache miss | |
| run: | |
| mamba env update -n pyodide-env -f environment.yml | |
| if: steps.conda-cache.outputs.cache-hit != 'true' | |
| - name: Download latest Pyodide | |
| run: | | |
| wget http://pyodide-cache.s3-website-us-east-1.amazonaws.com/xbuildenv/dev/pyodide-core.tar.bz2 | |
| tar -xvf pyodide-core.tar.bz2 | |
| mv pyodide dist | |
| - uses: pyodide/pyodide-actions/install-browser@v2 | |
| with: | |
| runner: ${{ matrix.test-config.runner }} | |
| browser: ${{ matrix.test-config.runtime }} | |
| browser-version: ${{ matrix.test-config.runtime-version }} | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: repodata | |
| path: ./repodata/ | |
| - name: Copy repodata | |
| run: | | |
| cp ./repodata/* ./dist/ | |
| - name: Install test dependencies | |
| run: | | |
| which python | |
| echo y | python -m pip install -r requirements.txt | |
| - name: Run tests | |
| run: | | |
| pytest -v \ | |
| --dist-dir=./dist/ \ | |
| --runner=${{ matrix.test-config.runner }} \ | |
| --rt ${{ matrix.test-config.runtime }} \ | |
| packages | |
| release: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| needs: [build] | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') | |
| environment: deploy | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Mambaforge | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| activate-environment: pyodide-env | |
| channels: conda-forge | |
| use-mamba: true | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: repodata | |
| path: ./repodata/ | |
| - name: Install anaconda-client | |
| run: | | |
| mamba install -c defaults anaconda-client -y | |
| - name: Upload wheels | |
| run: | | |
| # Anaconda denies packages with long descriptions, so set summary to null | |
| anaconda -t ${{ secrets.ANACONDA_API_TOKEN }} upload --force ./repodata/*.whl --summary "" |