From 2750fc88fcafe83737bfc6fa31c882dcf361835a Mon Sep 17 00:00:00 2001 From: Alexander Kell Date: Sat, 9 Nov 2024 15:14:41 +0700 Subject: [PATCH] Fix publish (#8) Optimize Pipeline and fix publish --- .github/workflows/main.yml | 176 +++++++++++++++++++++---------------- pyproject.toml | 3 + 2 files changed, 102 insertions(+), 77 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8bd0be8..3e6dd37 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,8 +57,6 @@ jobs: with: fetch-depth: 0 fetch-tags: true - - name: Install uv - uses: astral-sh/setup-uv@v3 - name: Cache python venv uses: actions/cache@v4 with: @@ -82,17 +80,8 @@ jobs: shell: bash build: - name: Build for ${{ matrix.os }} ${{ matrix.arch }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-latest, windows-latest, macos-latest ] - arch: [ x64, arm64 ] - exclude: - - os: windows-latest - arch: arm64 # Windows GitHub runners do not support ARM64 yet - - os: ubuntu-latest - arch: arm64 # Ubuntu GitHub runners do not support ARM64 yet + name: Build + runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -105,7 +94,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.11' - architecture: ${{ matrix.arch }} - name: Install uv Package Manager uses: astral-sh/setup-uv@v3 @@ -123,7 +111,6 @@ jobs: - name: Upload Build Artifact uses: actions/upload-artifact@v4 with: - name: dist-files-${{ matrix.os }}-${{ matrix.arch }} path: dist/* unit_test: @@ -238,72 +225,16 @@ jobs: uv run pytest tests_ce/integration_tests/ -n auto shell: bash - publish_dev: - if: github.ref == 'refs/heads/development' - runs-on: ubuntu-latest - needs: [ build, unit_test, functional_test, integration_test ] - environment: - name: testpypi - url: https://test.pypi.org/p/datamimic_ce - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ - contents: read - steps: - - name: Download All Build Artifacts - uses: actions/download-artifact@v4 - with: - path: dist - - - name: Publish distribution to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository-url: https://test.pypi.org/legacy/ - packages: 'dist/**' - - release: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - needs: [ build, unit_test, functional_test, integration_test ] - environment: - name: pypi - url: https://pypi.org/p/datamimic_ce - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ - contents: read - steps: - - name: Download Build Artifact - uses: actions/download-artifact@v4 - with: - name: 'dist-files-*' - path: dist - - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - packages: 'dist/*' - - trigger_ee_dev: - if: github.ref == 'refs/heads/development' - runs-on: ubuntu-latest - needs: - - publish_dev - steps: - - name: trigger EE development pipelines - run: curl -X POST --fail -F token=$EE_TOKEN -F ref=development https://gitlab.dwellerlab.com/api/v4/projects/214/trigger/pipeline; - env: - EE_TOKEN: ${{ secrets.EE_TOKEN }} - shell: bash - e2e_test: name: E2E Test on ${{ matrix.os }} ${{ matrix.arch }} + if: github.ref == 'refs/heads/development' || startsWith(github.ref, 'refs/tags/') runs-on: ${{ matrix.os }} - needs: build + needs: [ build, unit_test, functional_test, integration_test ] strategy: matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] arch: [ x64, arm64 ] + python-version: [ '3.11', '3.12' ] exclude: - os: windows-latest arch: arm64 # Exclude Windows ARM64 as it's not supported @@ -313,7 +244,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: ${{ matrix.python-version }} architecture: ${{ matrix.arch }} - name: Install uv Package Manager @@ -321,12 +252,29 @@ jobs: python -m pip install --upgrade pip pip install uv - - name: Download Build Artifact + - name: Download All Build Artifacts uses: actions/download-artifact@v4 with: - name: dist-files-${{ matrix.os }}-${{ matrix.arch }} path: dist + - name: Move *.whl and *.tar.gz to dist directory (Unix) + if: runner.os != 'Windows' + run: | + mv dist/artifact/*.whl dist + mv dist/artifact/*.tar.gz dist + rm -rf dist/artifact + rm -rf dist/setup + + - name: Move *.whl and *.tar.gz to dist directory (Windows) + if: runner.os == 'Windows' + run: | + Move-Item dist\artifact\*.whl dist + Move-Item dist\artifact\*.tar.gz dist + Remove-Item -Path dist\artifact -Recurse -Force + Remove-Item -Path dist\setup -Recurse -Force + shell: pwsh + + - name: List Contents of dist Directory run: | if [ "$RUNNER_OS" == "Windows" ]; then @@ -369,3 +317,77 @@ jobs: - name: Run the demo-datetime Demo run: datamimic run ./my_demos/demo-datetime/datamimic.xml + + publish_dev: + if: github.ref == 'refs/heads/development' + runs-on: ubuntu-latest + needs: [ e2e_test ] + environment: + name: testpypi + url: https://test.pypi.org/p/datamimic_ce + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ + contents: read + steps: + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Download All Build Artifacts + uses: actions/download-artifact@v4 + with: + path: dist + + - name: Move *.whl and *.tar.gz to dist directory + run: | + mv dist/artifact/*.whl dist + mv dist/artifact/*.tar.gz dist + rm -rf dist/artifact + rm -rf dist/setup + + + - name: List Contents of dist Directory + run: ls -R dist + + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + release: + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + needs: [ e2e_test ] + environment: + name: pypi + url: https://pypi.org/p/datamimic_ce + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ + contents: read + steps: + - name: Download All Build Artifacts + uses: actions/download-artifact@v4 + with: + path: dist + + - name: Move *.whl and *.tar.gz to dist directory + run: | + mv dist/artifact/*.whl dist + mv dist/artifact/*.tar.gz dist + rm -rf dist/artifact + rm -rf dist/setup + + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + + trigger_ee_dev: + if: github.ref == 'refs/heads/development' + runs-on: ubuntu-latest + needs: [ publish_dev ] + steps: + - name: trigger EE development pipelines + run: curl -X POST --fail -F token=$EE_TOKEN -F ref=development https://gitlab.dwellerlab.com/api/v4/projects/214/trigger/pipeline; + env: + EE_TOKEN: ${{ secrets.EE_TOKEN }} + shell: bash \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 0bc0b22..c9fb14f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,3 +144,6 @@ typeCheckingMode = "off" [tool.pytest.ini_options] markers = ["run: Custom marker for running specific tests"] + +[tool.pytest] +parallel = true