From dd9c46e0a0ffc8601d4f03d7de7c4e8d5f020820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 15:52:59 +0200 Subject: [PATCH 01/15] Small updates --- .github/workflows/build.yaml | 19 ++++++++++--------- conda.recipe/meta.yaml | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 867122f..0c0ae22 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,15 +2,15 @@ name: packages on: push: tags: - - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+a[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+b[0-9]+' - - '[0-9]+.[0-9]+.[0-9]+rc[0-9]+' + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+a[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+b[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" jobs: conda_build: name: Build Conda Packages - runs-on: 'ubuntu-latest' + runs-on: "ubuntu-latest" defaults: run: shell: bash -l {0} @@ -20,16 +20,16 @@ jobs: PYTHON_VERSION: "3.10" CHANS: "-c pyviz" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Fetch unshallow run: git fetch --prune --tags --unshallow -f - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: ${{ env.PYTHON_VERSION }} - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - python-version: "3.10" + python-version: ${{ env.PYTHON_VERSION }} - name: Set output id: vars run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} @@ -46,9 +46,10 @@ jobs: - name: conda main upload if: (!(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main + pip_build: name: Build PyPI Packages - runs-on: 'ubuntu-latest' + runs-on: "ubuntu-latest" defaults: run: shell: bash -l {0} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 1641191..645aeef 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -9,7 +9,7 @@ source: build: noarch: python - script: python setup.py install --single-version-externally-managed --record=record.txt + script: {{ PYTHON }} -m pip install -vv {{ sdata['name'] }}-{{ VERSION }}-py3-none-any.whl entry_points: {% for group,epoints in sdata.get("entry_points", {}).items() %} {% for entry_point in epoints %} From 349bcb379358ad2a9abc910b0e0af60be883391a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 15:59:22 +0200 Subject: [PATCH 02/15] Add conda build script --- conda.recipe/meta.yaml | 6 +++--- scripts/conda-build.sh | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100755 scripts/conda-build.sh diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 645aeef..cbff074 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,15 +1,15 @@ -{% set sdata = load_setup_py_data() %} +{% set sdata = load_setup_py_data(setup_file="../setup.py", from_recipe_dir=True) %} package: name: intake-dremio version: {{ sdata['version'] }} source: - path: .. + url: ../dist/{{ sdata['name'].replace('-', '_') }}-{{ VERSION }}-py3-none-any.whl build: noarch: python - script: {{ PYTHON }} -m pip install -vv {{ sdata['name'] }}-{{ VERSION }}-py3-none-any.whl + script: {{ PYTHON }} -m pip install -vv {{ sdata['name'].replace('-', '_') }}-{{ VERSION }}-py3-none-any.whl entry_points: {% for group,epoints in sdata.get("entry_points", {}).items() %} {% for entry_point in epoints %} diff --git a/scripts/conda-build.sh b/scripts/conda-build.sh new file mode 100755 index 0000000..2eb0bf8 --- /dev/null +++ b/scripts/conda-build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +for file in dist/*.whl dist/*.tar.bz2; do + if [ -e "$file" ]; then + echo "dist folder already contains $(basename "$file"). Please delete it before running this script." + exit 1 + fi +done + +export SETUPTOOLS_ENABLE_FEATURES=legacy-editable +git diff --exit-code +python -m build -w . + +VERSION=$(find dist -name "*.whl" -exec basename {} \; | cut -d- -f2) +export VERSION + +conda build conda.recipe --no-anaconda-upload --no-verify + +mv "$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2" dist From 35b2fb99a3c2abe9ce3bc4abb4295cb33e54a8eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 16:38:09 +0200 Subject: [PATCH 03/15] Update to work with PyArrow --- conda.recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index cbff074..65f6aba 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -26,6 +26,7 @@ requirements: {% for dep in sdata.get('install_requires',{}) %} - {{ dep }} {% endfor %} + - libarrow-flight test: imports: - intake_dremio From 07dd082c9bfe3476b173f627b4c78d95b98ff0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 16:45:22 +0200 Subject: [PATCH 04/15] Update build script --- scripts/conda-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/conda-build.sh b/scripts/conda-build.sh index 2eb0bf8..b145faf 100755 --- a/scripts/conda-build.sh +++ b/scripts/conda-build.sh @@ -2,6 +2,8 @@ set -euxo pipefail +PACKAGE="intake-dremio" + for file in dist/*.whl dist/*.tar.bz2; do if [ -e "$file" ]; then echo "dist folder already contains $(basename "$file"). Please delete it before running this script." From 3cacae1bb66955b5e962bae5f7ab65c64248f231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 16:57:40 +0200 Subject: [PATCH 05/15] DEV: Update github action --- .github/workflows/build.yaml | 260 ++++++++++++++++++++++++++--------- 1 file changed, 196 insertions(+), 64 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0c0ae22..027e591 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,87 +7,219 @@ on: - "[0-9]+.[0-9]+.[0-9]+b[0-9]+" - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" +defaults: + run: + shell: bash -l {0} + +env: + PYTHON_VERSION: "3.10" + PACKAGE: "intake-dremio" + jobs: + waiting_room: + name: Waiting Room + runs-on: ubuntu-latest + needs: [conda_build, pip_install] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + # environment: + # name: publish + steps: + - run: echo "All builds have finished, have been approved, and ready to publish" + conda_build: - name: Build Conda Packages + name: Build Conda runs-on: "ubuntu-latest" - defaults: - run: - shell: bash -l {0} - env: - CHANS_DEV: "-c conda-forge" - PKG_TEST_PYTHON: "--test-python=py310" - PYTHON_VERSION: "3.10" - CHANS: "-c pyviz" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: "100" - name: Fetch unshallow run: git fetch --prune --tags --unshallow -f - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - uses: conda-incubator/setup-miniconda@v3 with: - auto-update-conda: true - python-version: ${{ env.PYTHON_VERSION }} - - name: Set output - id: vars - run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} + miniconda-version: "latest" - name: conda setup run: | - conda config --set always_yes True - conda install -c pyviz "pyctdev>=0.5" - doit ecosystem_setup + conda install -y conda-build python-build - name: conda build - run: doit package_build $CHANS_DEV $PKG_TEST_PYTHON --test-group=all - - name: conda dev upload - if: (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')) - run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev - - name: conda main upload - if: (!(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) - run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main + run: | + source ./scripts/conda-build.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: conda + path: dist/*.tar.bz2 + if-no-files-found: error + + # conda_publish: + # name: Publish Conda + # runs-on: ubuntu-latest + # needs: [conda_build, waiting_room] + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: conda + # path: dist/ + # - name: Set environment variables + # run: | + # echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + # echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV + # - uses: conda-incubator/setup-miniconda@v3 + # with: + # miniconda-version: "latest" + # - name: conda setup + # run: | + # conda install -y anaconda-client + # - name: conda dev upload + # if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') + # run: | + # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE + # - name: conda main upload + # if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) + # run: | + # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE pip_build: - name: Build PyPI Packages + name: Build PyPI runs-on: "ubuntu-latest" - defaults: - run: - shell: bash -l {0} - env: - CHANS_DEV: "-c pyviz/label/dev -c conda-forge" - PKG_TEST_PYTHON: "--test-python=py310" - PYTHON_VERSION: "3.10" - CHANS: "-c pyviz" - PYPI: "https://upload.pypi.org/legacy/" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + fetch-depth: "100" - name: Fetch unshallow run: git fetch --prune --tags --unshallow -f - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - uses: conda-incubator/setup-miniconda@v3 - with: - miniconda-version: "latest" - - name: conda setup - run: | - conda config --set always_yes True - conda install -c pyviz "pyctdev>=0.5" - doit ecosystem_setup - doit env_create $CHANS_DEV --python=$PYTHON_VERSION - - name: env setup - run: | - eval "$(conda shell.bash hook)" - conda activate test-environment - doit develop_install $CHANS_DEV -o tests - doit pip_on_conda - - name: pip build - run: | - eval "$(conda shell.bash hook)" - conda activate test-environment - doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit_deploy - - name: pip upload + python-version: ${{ env.PYTHON_VERSION }} + - name: Install build run: | - eval "$(conda shell.bash hook)" - conda activate test-environment - doit ecosystem=pip package_upload -u ${{ secrets.PPU }} -p ${{ secrets.PPP }} -r $PYPI + python -m pip install build + - name: Build package + run: python -m build . + - uses: actions/upload-artifact@v4 + if: always() + with: + name: pip + path: dist/ + if-no-files-found: error + + # pip_install: + # name: Install PyPI + # runs-on: "ubuntu-latest" + # needs: [pip_build] + # steps: + # - uses: actions/setup-python@v5 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # - uses: actions/download-artifact@v4 + # with: + # name: pip + # path: dist/ + # - name: Install package + # run: python -m pip install dist/*.whl + # - name: Test package + # run: python -c "import $PACKAGE; print($PACKAGE.__version__)" + # + # pip_publish: + # name: Publish PyPI + # runs-on: ubuntu-latest + # needs: [pip_build, waiting_room] + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: pip + # path: dist/ + # - name: Publish to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # user: ${{ secrets.PPU }} + # password: ${{ secrets.PPP }} + # repository-url: "https://upload.pypi.org/legacy/" + + +# jobs: +# conda_build: +# name: Build Conda Packages +# runs-on: "ubuntu-latest" +# defaults: +# run: +# shell: bash -l {0} +# env: +# CHANS_DEV: "-c conda-forge" +# PKG_TEST_PYTHON: "--test-python=py310" +# PYTHON_VERSION: "3.10" +# CHANS: "-c pyviz" +# steps: +# - uses: actions/checkout@v3 +# - name: Fetch unshallow +# run: git fetch --prune --tags --unshallow -f +# - uses: actions/setup-python@v5 +# with: +# python-version: ${{ env.PYTHON_VERSION }} +# - uses: conda-incubator/setup-miniconda@v3 +# with: +# auto-update-conda: true +# python-version: ${{ env.PYTHON_VERSION }} +# - name: Set output +# id: vars +# run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} +# - name: conda setup +# run: | +# conda config --set always_yes True +# conda install -c pyviz "pyctdev>=0.5" +# doit ecosystem_setup +# - name: conda build +# run: doit package_build $CHANS_DEV $PKG_TEST_PYTHON --test-group=all +# - name: conda dev upload +# if: (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')) +# run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev +# - name: conda main upload +# if: (!(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) +# run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main +# +# pip_build: +# name: Build PyPI Packages +# runs-on: "ubuntu-latest" +# defaults: +# run: +# shell: bash -l {0} +# env: +# CHANS_DEV: "-c pyviz/label/dev -c conda-forge" +# PKG_TEST_PYTHON: "--test-python=py310" +# PYTHON_VERSION: "3.10" +# CHANS: "-c pyviz" +# PYPI: "https://upload.pypi.org/legacy/" +# steps: +# - uses: actions/checkout@v2 +# - name: Fetch unshallow +# run: git fetch --prune --tags --unshallow -f +# - uses: actions/setup-python@v5 +# with: +# python-version: ${{ matrix.python-version }} +# - uses: conda-incubator/setup-miniconda@v3 +# with: +# miniconda-version: "latest" +# - name: conda setup +# run: | +# conda config --set always_yes True +# conda install -c pyviz "pyctdev>=0.5" +# doit ecosystem_setup +# doit env_create $CHANS_DEV --python=$PYTHON_VERSION +# - name: env setup +# run: | +# eval "$(conda shell.bash hook)" +# conda activate test-environment +# doit develop_install $CHANS_DEV -o tests +# doit pip_on_conda +# - name: pip build +# run: | +# eval "$(conda shell.bash hook)" +# conda activate test-environment +# doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit_deploy +# - name: pip upload +# run: | +# eval "$(conda shell.bash hook)" +# conda activate test-environment +# doit ecosystem=pip package_upload -u ${{ secrets.PPU }} -p ${{ secrets.PPP }} -r $PYPI From 60bf74c9ebfff885f70886e1255c946098f05a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:12:01 +0200 Subject: [PATCH 06/15] Add workflow-dispatch for build --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 027e591..9ec6b13 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -6,6 +6,8 @@ on: - "[0-9]+.[0-9]+.[0-9]+a[0-9]+" - "[0-9]+.[0-9]+.[0-9]+b[0-9]+" - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" + # Dry-run only + workflow_dispatch: defaults: run: From 1658e72f59288b1cfba19d9d8586411ef82a76ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:18:14 +0200 Subject: [PATCH 07/15] Add pip_install step --- .github/workflows/build.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9ec6b13..4fa14d5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -106,22 +106,22 @@ jobs: path: dist/ if-no-files-found: error - # pip_install: - # name: Install PyPI - # runs-on: "ubuntu-latest" - # needs: [pip_build] - # steps: - # - uses: actions/setup-python@v5 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # - uses: actions/download-artifact@v4 - # with: - # name: pip - # path: dist/ - # - name: Install package - # run: python -m pip install dist/*.whl - # - name: Test package - # run: python -c "import $PACKAGE; print($PACKAGE.__version__)" + pip_install: + name: Install PyPI + runs-on: "ubuntu-latest" + needs: [pip_build] + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + - name: Install package + run: python -m pip install dist/*.whl + - name: Test package + run: python -c "import $PACKAGE; print($PACKAGE.__version__)" # # pip_publish: # name: Publish PyPI From 1329cd8ff654af1194424f62fe5d3062361a2869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:22:53 +0200 Subject: [PATCH 08/15] Remove fetch-depth --- .github/workflows/build.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4fa14d5..c400e50 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,8 +33,6 @@ jobs: runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v4 - with: - fetch-depth: "100" - name: Fetch unshallow run: git fetch --prune --tags --unshallow -f - uses: conda-incubator/setup-miniconda@v3 @@ -87,8 +85,6 @@ jobs: runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v4 - with: - fetch-depth: "100" - name: Fetch unshallow run: git fetch --prune --tags --unshallow -f - uses: actions/setup-python@v5 From 3b3ec8d02007a01db7d7749b11e4fb52154133c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:25:54 +0200 Subject: [PATCH 09/15] Updates --- .github/workflows/build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c400e50..842abbb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -38,6 +38,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" + python-version: "3.10" - name: conda setup run: | conda install -y conda-build python-build @@ -117,6 +118,8 @@ jobs: - name: Install package run: python -m pip install dist/*.whl - name: Test package + env: + PACKAGE: "intake_dremio" run: python -c "import $PACKAGE; print($PACKAGE.__version__)" # # pip_publish: From 2dba1ec42a2d9471c3ade9a649a82ae757a7f5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:36:23 +0200 Subject: [PATCH 10/15] Pin pyarrow for conda --- conda.recipe/meta.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 65f6aba..6f089d4 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -24,9 +24,14 @@ requirements: run: - python {{ sdata['python_requires'] }} {% for dep in sdata.get('install_requires',{}) %} + {% if dep.startswith("pyarrow") %} + # Conda-forge split up pyarrow packages (libarrow-flight), default does not + # setup.py lower pins + - {{ dep }} ,<16 + {% else %} - {{ dep }} + {% endif %} {% endfor %} - - libarrow-flight test: imports: - intake_dremio From 238725929d22ededee43b0efd758c97bb2bb2c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:50:45 +0200 Subject: [PATCH 11/15] Try publish step --- .github/workflows/build.yaml | 94 ++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 842abbb..ce4bc39 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,9 +22,7 @@ jobs: name: Waiting Room runs-on: ubuntu-latest needs: [conda_build, pip_install] - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') - # environment: - # name: publish + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - run: echo "All builds have finished, have been approved, and ready to publish" @@ -52,34 +50,34 @@ jobs: path: dist/*.tar.bz2 if-no-files-found: error - # conda_publish: - # name: Publish Conda - # runs-on: ubuntu-latest - # needs: [conda_build, waiting_room] - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') - # steps: - # - uses: actions/download-artifact@v4 - # with: - # name: conda - # path: dist/ - # - name: Set environment variables - # run: | - # echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - # echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV - # - uses: conda-incubator/setup-miniconda@v3 - # with: - # miniconda-version: "latest" - # - name: conda setup - # run: | - # conda install -y anaconda-client - # - name: conda dev upload - # if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') - # run: | - # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE - # - name: conda main upload - # if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) - # run: | - # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + conda_publish: + name: Publish Conda + runs-on: ubuntu-latest + needs: [conda_build, waiting_room] + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v4 + with: + name: conda + path: dist/ + - name: Set environment variables + run: | + echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV + - uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: "latest" + - name: conda setup + run: | + conda install -y anaconda-client + # - name: conda dev upload + # if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') + # run: | + # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE + # - name: conda main upload + # if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) + # run: | + # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE pip_build: name: Build PyPI @@ -121,23 +119,23 @@ jobs: env: PACKAGE: "intake_dremio" run: python -c "import $PACKAGE; print($PACKAGE.__version__)" - # - # pip_publish: - # name: Publish PyPI - # runs-on: ubuntu-latest - # needs: [pip_build, waiting_room] - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') - # steps: - # - uses: actions/download-artifact@v4 - # with: - # name: pip - # path: dist/ - # - name: Publish to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # user: ${{ secrets.PPU }} - # password: ${{ secrets.PPP }} - # repository-url: "https://upload.pypi.org/legacy/" + + pip_publish: + name: Publish PyPI + runs-on: ubuntu-latest + needs: [pip_build, waiting_room] + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - uses: actions/download-artifact@v4 + with: + name: pip + path: dist/ + # - name: Publish to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # user: ${{ secrets.PPU }} + # password: ${{ secrets.PPP }} + # repository-url: "https://upload.pypi.org/legacy/" # jobs: From 29a656210272ac38d800f793e607cd40612075fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:56:47 +0200 Subject: [PATCH 12/15] Uncomment --- .github/workflows/build.yaml | 120 +++++------------------------------ 1 file changed, 17 insertions(+), 103 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ce4bc39..8dbea46 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,7 +22,7 @@ jobs: name: Waiting Room runs-on: ubuntu-latest needs: [conda_build, pip_install] - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - run: echo "All builds have finished, have been approved, and ready to publish" @@ -54,7 +54,7 @@ jobs: name: Publish Conda runs-on: ubuntu-latest needs: [conda_build, waiting_room] - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v4 with: @@ -70,14 +70,14 @@ jobs: - name: conda setup run: | conda install -y anaconda-client - # - name: conda dev upload - # if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') - # run: | - # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE - # - name: conda main upload - # if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) - # run: | - # anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + - name: conda dev upload + if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') + run: | + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE + - name: conda main upload + if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) + run: | + anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE pip_build: name: Build PyPI @@ -124,101 +124,15 @@ jobs: name: Publish PyPI runs-on: ubuntu-latest needs: [pip_build, waiting_room] - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v4 with: name: pip path: dist/ - # - name: Publish to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # user: ${{ secrets.PPU }} - # password: ${{ secrets.PPP }} - # repository-url: "https://upload.pypi.org/legacy/" - - -# jobs: -# conda_build: -# name: Build Conda Packages -# runs-on: "ubuntu-latest" -# defaults: -# run: -# shell: bash -l {0} -# env: -# CHANS_DEV: "-c conda-forge" -# PKG_TEST_PYTHON: "--test-python=py310" -# PYTHON_VERSION: "3.10" -# CHANS: "-c pyviz" -# steps: -# - uses: actions/checkout@v3 -# - name: Fetch unshallow -# run: git fetch --prune --tags --unshallow -f -# - uses: actions/setup-python@v5 -# with: -# python-version: ${{ env.PYTHON_VERSION }} -# - uses: conda-incubator/setup-miniconda@v3 -# with: -# auto-update-conda: true -# python-version: ${{ env.PYTHON_VERSION }} -# - name: Set output -# id: vars -# run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} -# - name: conda setup -# run: | -# conda config --set always_yes True -# conda install -c pyviz "pyctdev>=0.5" -# doit ecosystem_setup -# - name: conda build -# run: doit package_build $CHANS_DEV $PKG_TEST_PYTHON --test-group=all -# - name: conda dev upload -# if: (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')) -# run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev -# - name: conda main upload -# if: (!(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) -# run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main -# -# pip_build: -# name: Build PyPI Packages -# runs-on: "ubuntu-latest" -# defaults: -# run: -# shell: bash -l {0} -# env: -# CHANS_DEV: "-c pyviz/label/dev -c conda-forge" -# PKG_TEST_PYTHON: "--test-python=py310" -# PYTHON_VERSION: "3.10" -# CHANS: "-c pyviz" -# PYPI: "https://upload.pypi.org/legacy/" -# steps: -# - uses: actions/checkout@v2 -# - name: Fetch unshallow -# run: git fetch --prune --tags --unshallow -f -# - uses: actions/setup-python@v5 -# with: -# python-version: ${{ matrix.python-version }} -# - uses: conda-incubator/setup-miniconda@v3 -# with: -# miniconda-version: "latest" -# - name: conda setup -# run: | -# conda config --set always_yes True -# conda install -c pyviz "pyctdev>=0.5" -# doit ecosystem_setup -# doit env_create $CHANS_DEV --python=$PYTHON_VERSION -# - name: env setup -# run: | -# eval "$(conda shell.bash hook)" -# conda activate test-environment -# doit develop_install $CHANS_DEV -o tests -# doit pip_on_conda -# - name: pip build -# run: | -# eval "$(conda shell.bash hook)" -# conda activate test-environment -# doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit_deploy -# - name: pip upload -# run: | -# eval "$(conda shell.bash hook)" -# conda activate test-environment -# doit ecosystem=pip package_upload -u ${{ secrets.PPU }} -p ${{ secrets.PPP }} -r $PYPI + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: ${{ secrets.PPU }} + password: ${{ secrets.PPP }} + repository-url: "https://upload.pypi.org/legacy/" From b439844ccafaf68642fbe73e47c244c63c3aa667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 17:59:43 +0200 Subject: [PATCH 13/15] Remove legacy --- scripts/conda-build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/conda-build.sh b/scripts/conda-build.sh index b145faf..f6c93a2 100755 --- a/scripts/conda-build.sh +++ b/scripts/conda-build.sh @@ -11,7 +11,6 @@ for file in dist/*.whl dist/*.tar.bz2; do fi done -export SETUPTOOLS_ENABLE_FEATURES=legacy-editable git diff --exit-code python -m build -w . From c2dd3308ec011fe8265690673357b5ce911a5001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Wed, 12 Jun 2024 18:13:19 +0200 Subject: [PATCH 14/15] Small update --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8dbea46..16370a0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,7 +36,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" - python-version: "3.10" + python-version: ${{ env.PYTHON_VERSION }} - name: conda setup run: | conda install -y conda-build python-build From 249caff4d0e8b4a6436c668da71242202e26c84f Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Wed, 12 Jun 2024 19:04:29 +0200 Subject: [PATCH 15/15] Apply suggestions from code review --- conda.recipe/meta.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 6f089d4..cbff074 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -24,13 +24,7 @@ requirements: run: - python {{ sdata['python_requires'] }} {% for dep in sdata.get('install_requires',{}) %} - {% if dep.startswith("pyarrow") %} - # Conda-forge split up pyarrow packages (libarrow-flight), default does not - # setup.py lower pins - - {{ dep }} ,<16 - {% else %} - {{ dep }} - {% endif %} {% endfor %} test: imports: