Update build workflow to seperate build and publish jobs

.github/workflows/build.yaml

@@ -2,91 +2,226 @@ name: packages
 on:
   push:
     tags:
-    - '[0-9]+.[0-9]+.[0-9]+'
-    - '[0-9]+.[0-9]+.[0-9]+a[0-9]+'
-    - '[0-9]+.[0-9]+.[0-9]+b[0-9]+'
-    - '[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
+      - "[0-9]+.[0-9]+.[0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+a[0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+b[0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
+  # Dry-run only
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+env:
+  PYTHON_VERSION: "3.10"
+  PACKAGE: "intake-dremio"
 
 jobs:
+  waiting_room:
+    name: Waiting Room
+    runs-on: ubuntu-latest
+    needs: [conda_build, pip_install]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    # environment:
+    #   name: publish
+    steps:
+      - run: echo "All builds have finished, have been approved, and ready to publish"
+
   conda_build:
-    name: Build Conda Packages
-    runs-on: 'ubuntu-latest'
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      CHANS_DEV: "-c conda-forge"
-      PKG_TEST_PYTHON: "--test-python=py310"
-      PYTHON_VERSION: "3.10"
-      CHANS: "-c pyviz"
+    name: Build Conda
+    runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "100"
       - name: Fetch unshallow
         run: git fetch --prune --tags --unshallow -f
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
       - uses: conda-incubator/setup-miniconda@v3
         with:
-          auto-update-conda: true
-          python-version: "3.10"
-      - name: Set output
-        id: vars
-        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+          miniconda-version: "latest"
       - name: conda setup
         run: |
-          conda config --set always_yes True
-          conda install -c pyviz "pyctdev>=0.5"
-          doit ecosystem_setup
+          conda install -y conda-build python-build
       - name: conda build
-        run: doit package_build $CHANS_DEV $PKG_TEST_PYTHON --test-group=all
-      - name: conda dev upload
-        if: (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))
-        run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev
-      - name: conda main upload
-        if: (!(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main
+        run: |
+          source ./scripts/conda-build.sh
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: conda
+          path: dist/*.tar.bz2
+          if-no-files-found: error
+
+  # conda_publish:
+  #   name: Publish Conda
+  #   runs-on: ubuntu-latest
+  #   needs: [conda_build, waiting_room]
+  #   if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+  #   steps:
+  #     - uses: actions/download-artifact@v4
+  #       with:
+  #         name: conda
+  #         path: dist/
+  #     - name: Set environment variables
+  #       run: |
+  #         echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+  #         echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV
+  #     - uses: conda-incubator/setup-miniconda@v3
+  #       with:
+  #         miniconda-version: "latest"
+  #     - name: conda setup
+  #       run: |
+  #         conda install -y anaconda-client
+  #     - name: conda dev upload
+  #       if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')
+  #       run: |
+  #         anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE
+  #     - name: conda main upload
+  #       if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')))
+  #       run: |
+  #         anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE
+
   pip_build:
-    name: Build PyPI Packages
-    runs-on: 'ubuntu-latest'
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      CHANS_DEV: "-c pyviz/label/dev  -c conda-forge"
-      PKG_TEST_PYTHON: "--test-python=py310"
-      PYTHON_VERSION: "3.10"
-      CHANS: "-c pyviz"
-      PYPI: "https://upload.pypi.org/legacy/"
+    name: Build PyPI
+    runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "100"
       - name: Fetch unshallow
         run: git fetch --prune --tags --unshallow -f
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
-      - uses: conda-incubator/setup-miniconda@v3
-        with:
-          miniconda-version: "latest"
-      - name: conda setup
-        run: |
-          conda config --set always_yes True
-          conda install -c pyviz "pyctdev>=0.5"
-          doit ecosystem_setup
-          doit env_create $CHANS_DEV --python=$PYTHON_VERSION
-      - name: env setup
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install build
         run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit develop_install $CHANS_DEV -o tests
-          doit pip_on_conda
-      - name: pip build
-        run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit_deploy
-      - name: pip upload
-        run: |
-          eval "$(conda shell.bash hook)"
-          conda activate test-environment
-          doit ecosystem=pip package_upload -u ${{ secrets.PPU }} -p ${{ secrets.PPP }} -r $PYPI
+          python -m pip install build
+      - name: Build package
+        run: python -m build .
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pip
+          path: dist/
+          if-no-files-found: error
+
+  pip_install:
+    name: Install PyPI
+    runs-on: "ubuntu-latest"
+    needs: [pip_build]
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: pip
+          path: dist/
+      - name: Install package
+        run: python -m pip install dist/*.whl
+      - name: Test package
+        run: python -c "import $PACKAGE; print($PACKAGE.__version__)"
+  #
+  # pip_publish:
+  #   name: Publish PyPI
+  #   runs-on: ubuntu-latest
+  #   needs: [pip_build, waiting_room]
+  #   if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+  #   steps:
+  #     - uses: actions/download-artifact@v4
+  #       with:
+  #         name: pip
+  #         path: dist/
+  #     - name: Publish to PyPI
+  #       uses: pypa/gh-action-pypi-publish@release/v1
+  #       with:
+  #         user: ${{ secrets.PPU }}
+  #         password: ${{ secrets.PPP }}
+  #         repository-url: "https://upload.pypi.org/legacy/"
+
+
+# jobs:
+#   conda_build:
+#     name: Build Conda Packages
+#     runs-on: "ubuntu-latest"
+#     defaults:
+#       run:
+#         shell: bash -l {0}
+#     env:
+#       CHANS_DEV: "-c conda-forge"
+#       PKG_TEST_PYTHON: "--test-python=py310"
+#       PYTHON_VERSION: "3.10"
+#       CHANS: "-c pyviz"
+#     steps:
+#       - uses: actions/checkout@v3
+#       - name: Fetch unshallow
+#         run: git fetch --prune --tags --unshallow -f
+#       - uses: actions/setup-python@v5
+#         with:
+#           python-version: ${{ env.PYTHON_VERSION }}
+#       - uses: conda-incubator/setup-miniconda@v3
+#         with:
+#           auto-update-conda: true
+#           python-version: ${{ env.PYTHON_VERSION }}
+#       - name: Set output
+#         id: vars
+#         run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+#       - name: conda setup
+#         run: |
+#           conda config --set always_yes True
+#           conda install -c pyviz "pyctdev>=0.5"
+#           doit ecosystem_setup
+#       - name: conda build
+#         run: doit package_build $CHANS_DEV $PKG_TEST_PYTHON --test-group=all
+#       - name: conda dev upload
+#         if: (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))
+#         run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev
+#       - name: conda main upload
+#         if: (!(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
+#         run: doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main
+#
+#   pip_build:
+#     name: Build PyPI Packages
+#     runs-on: "ubuntu-latest"
+#     defaults:
+#       run:
+#         shell: bash -l {0}
+#     env:
+#       CHANS_DEV: "-c pyviz/label/dev  -c conda-forge"
+#       PKG_TEST_PYTHON: "--test-python=py310"
+#       PYTHON_VERSION: "3.10"
+#       CHANS: "-c pyviz"
+#       PYPI: "https://upload.pypi.org/legacy/"
+#     steps:
+#       - uses: actions/checkout@v2
+#       - name: Fetch unshallow
+#         run: git fetch --prune --tags --unshallow -f
+#       - uses: actions/setup-python@v5
+#         with:
+#           python-version: ${{ matrix.python-version }}
+#       - uses: conda-incubator/setup-miniconda@v3
+#         with:
+#           miniconda-version: "latest"
+#       - name: conda setup
+#         run: |
+#           conda config --set always_yes True
+#           conda install -c pyviz "pyctdev>=0.5"
+#           doit ecosystem_setup
+#           doit env_create $CHANS_DEV --python=$PYTHON_VERSION
+#       - name: env setup
+#         run: |
+#           eval "$(conda shell.bash hook)"
+#           conda activate test-environment
+#           doit develop_install $CHANS_DEV -o tests
+#           doit pip_on_conda
+#       - name: pip build
+#         run: |
+#           eval "$(conda shell.bash hook)"
+#           conda activate test-environment
+#           doit ecosystem=pip package_build $PKG_TEST_PYTHON --test-group=unit_deploy
+#       - name: pip upload
+#         run: |
+#           eval "$(conda shell.bash hook)"
+#           conda activate test-environment
+#           doit ecosystem=pip package_upload -u ${{ secrets.PPU }} -p ${{ secrets.PPP }} -r $PYPI

conda.recipe/meta.yaml

@@ -1,15 +1,15 @@
-{% set sdata = load_setup_py_data() %}
+{% set sdata = load_setup_py_data(setup_file="../setup.py", from_recipe_dir=True) %}
 
 package:
   name: intake-dremio
   version: {{ sdata['version'] }}
 
 source:
-  path: ..
+  url: ../dist/{{ sdata['name'].replace('-', '_') }}-{{ VERSION }}-py3-none-any.whl
 
 build:
   noarch: python
-  script: python setup.py install --single-version-externally-managed --record=record.txt
+  script: {{ PYTHON }} -m pip install -vv {{ sdata['name'].replace('-', '_') }}-{{ VERSION }}-py3-none-any.whl
   entry_points:
     {% for group,epoints in sdata.get("entry_points", {}).items() %}
       {% for entry_point in epoints %}
@@ -26,6 +26,7 @@ requirements:
     {% for dep in sdata.get('install_requires',{}) %}
     - {{ dep }}
     {% endfor %}
+    - libarrow-flight
 test:
   imports:
     - intake_dremio

scripts/conda-build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+PACKAGE="intake-dremio"
+
+for file in dist/*.whl dist/*.tar.bz2; do
+    if [ -e "$file" ]; then
+        echo "dist folder already contains $(basename "$file"). Please delete it before running this script."
+        exit 1
+    fi
+done
+
+export SETUPTOOLS_ENABLE_FEATURES=legacy-editable
+git diff --exit-code
+python -m build -w .
+
+VERSION=$(find dist -name "*.whl" -exec basename {} \; | cut -d- -f2)
+export VERSION
+
+conda build conda.recipe --no-anaconda-upload --no-verify
+
+mv "$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2" dist