Fix publish (#8)

Optimize Pipeline and fix publish
rapiddweller · Nov 9, 2024 · 2750fc8 · 2750fc8
1 parent 5212504
commit 2750fc8
Show file tree

Hide file tree

Showing 2 changed files with 102 additions and 77 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -57,8 +57,6 @@ jobs:
         with:
           fetch-depth: 0
           fetch-tags: true
-      - name: Install uv
-        uses: astral-sh/setup-uv@v3
       - name: Cache python venv
         uses: actions/cache@v4
         with:
@@ -82,17 +80,8 @@ jobs:
         shell: bash
 
   build:
-    name: Build for ${{ matrix.os }} ${{ matrix.arch }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
-        arch: [ x64, arm64 ]
-        exclude:
-          - os: windows-latest
-            arch: arm64  # Windows GitHub runners do not support ARM64 yet
-          - os: ubuntu-latest
-            arch: arm64  # Ubuntu GitHub runners do not support ARM64 yet
+    name: Build
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -105,7 +94,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-          architecture: ${{ matrix.arch }}
 
       - name: Install uv Package Manager
         uses: astral-sh/setup-uv@v3
@@ -123,7 +111,6 @@ jobs:
       - name: Upload Build Artifact
         uses: actions/upload-artifact@v4
         with:
-          name: dist-files-${{ matrix.os }}-${{ matrix.arch }}
           path: dist/*
 
   unit_test:
@@ -238,72 +225,16 @@ jobs:
           uv run pytest tests_ce/integration_tests/ -n auto
         shell: bash
 
-  publish_dev:
-    if: github.ref == 'refs/heads/development'
-    runs-on: ubuntu-latest
-    needs: [ build, unit_test, functional_test, integration_test ]
-    environment:
-      name: testpypi
-      url: https://test.pypi.org/p/datamimic_ce
-    permissions:
-      id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
-      contents: read
-    steps:
-      - name: Download All Build Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: dist
-
-      - name: Publish distribution to TestPyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
-          repository-url: https://test.pypi.org/legacy/
-          packages: 'dist/**'
-
-  release:
-    if: startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    needs: [ build, unit_test, functional_test, integration_test ]
-    environment:
-      name: pypi
-      url: https://pypi.org/p/datamimic_ce
-    permissions:
-      id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
-      contents: read
-    steps:
-      - name: Download Build Artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: 'dist-files-*'
-          path: dist
-
-      - name: Publish distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
-          packages: 'dist/*'
-
-  trigger_ee_dev:
-    if: github.ref == 'refs/heads/development'
-    runs-on: ubuntu-latest
-    needs:
-      - publish_dev
-    steps:
-      - name: trigger EE development pipelines
-        run: curl -X POST --fail -F token=$EE_TOKEN -F ref=development https://gitlab.dwellerlab.com/api/v4/projects/214/trigger/pipeline;
-        env:
-          EE_TOKEN: ${{ secrets.EE_TOKEN }}
-        shell: bash
-
   e2e_test:
     name: E2E Test on ${{ matrix.os }} ${{ matrix.arch }}
+    if: github.ref == 'refs/heads/development' || startsWith(github.ref, 'refs/tags/')
     runs-on: ${{ matrix.os }}
-    needs: build
+    needs: [ build, unit_test, functional_test, integration_test ]
     strategy:
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
         arch: [ x64, arm64 ]
+        python-version: [ '3.11', '3.12' ]
         exclude:
           - os: windows-latest
             arch: arm64  # Exclude Windows ARM64 as it's not supported
@@ -313,20 +244,37 @@ jobs:
       - name: Set Up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: ${{ matrix.python-version }}
           architecture: ${{ matrix.arch }}
 
       - name: Install uv Package Manager
         run: |
           python -m pip install --upgrade pip
           pip install uv
 
-      - name: Download Build Artifact
+      - name: Download All Build Artifacts
         uses: actions/download-artifact@v4
         with:
-          name: dist-files-${{ matrix.os }}-${{ matrix.arch }}
           path: dist
 
+      - name: Move *.whl and *.tar.gz to dist directory (Unix)
+        if: runner.os != 'Windows'
+        run: |
+          mv dist/artifact/*.whl dist
+          mv dist/artifact/*.tar.gz dist
+          rm -rf dist/artifact
+          rm -rf dist/setup
+
+      - name: Move *.whl and *.tar.gz to dist directory (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          Move-Item dist\artifact\*.whl dist
+          Move-Item dist\artifact\*.tar.gz dist
+          Remove-Item -Path dist\artifact -Recurse -Force
+          Remove-Item -Path dist\setup -Recurse -Force
+        shell: pwsh
+
+
       - name: List Contents of dist Directory
         run: |
           if [ "$RUNNER_OS" == "Windows" ]; then
@@ -369,3 +317,77 @@ jobs:
 
       - name: Run the demo-datetime Demo
         run: datamimic run ./my_demos/demo-datetime/datamimic.xml
+
+  publish_dev:
+    if: github.ref == 'refs/heads/development'
+    runs-on: ubuntu-latest
+    needs: [ e2e_test ]
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/datamimic_ce
+    permissions:
+      id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+      contents: read
+    steps:
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Download All Build Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+
+      - name: Move *.whl and *.tar.gz to dist directory
+        run: |
+          mv dist/artifact/*.whl dist
+          mv dist/artifact/*.tar.gz dist
+          rm -rf dist/artifact
+          rm -rf dist/setup
+
+
+      - name: List Contents of dist Directory
+        run: ls -R dist
+
+      - name: Publish distribution to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  release:
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    needs: [ e2e_test ]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/datamimic_ce
+    permissions:
+      id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+      contents: read
+    steps:
+      - name: Download All Build Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+
+      - name: Move *.whl and *.tar.gz to dist directory
+        run: |
+          mv dist/artifact/*.whl dist
+          mv dist/artifact/*.tar.gz dist
+          rm -rf dist/artifact
+          rm -rf dist/setup
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+
+  trigger_ee_dev:
+    if: github.ref == 'refs/heads/development'
+    runs-on: ubuntu-latest
+    needs: [ publish_dev ]
+    steps:
+      - name: trigger EE development pipelines
+        run: curl -X POST --fail -F token=$EE_TOKEN -F ref=development https://gitlab.dwellerlab.com/api/v4/projects/214/trigger/pipeline;
+        env:
+          EE_TOKEN: ${{ secrets.EE_TOKEN }}
+        shell: bash
diff --git a/pyproject.toml b/pyproject.toml
@@ -144,3 +144,6 @@ typeCheckingMode = "off"
 
 [tool.pytest.ini_options]
 markers = ["run: Custom marker for running specific tests"]
+
+[tool.pytest]
+parallel = true