is running Github Datamimic CE CI on rapiddweller/datamimic #39
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Github Datamimic CE CI | |
| run-name: ${{ github.run_actor }} is running ${{ github.workflow }} on ${{ github.repository }} | |
| on: | |
| push: | |
| branches: | |
| - development | |
| tags: | |
| - '[0-9]+.[0-9]+.[0-9]+' | |
| pull_request: | |
| branches: | |
| - development | |
| env: | |
| DATAMIMIC_LIB_ENVIRONMENT: lib_staging | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Export latest tag | |
| run: | | |
| export RD_PROJECT_LATEST_TAG=${GITHUB_REF#refs/*/} | |
| echo "RD_PROJECT_LATEST_TAG=$RD_PROJECT_LATEST_TAG" | |
| echo "RD_PROJECT_LATEST_TAG=$RD_PROJECT_LATEST_TAG" >> build.env | |
| - name: Export lib version without patch | |
| run: | | |
| export LIB_VERSION_WITHOUT_PATCH=$(echo "$RD_PROJECT_LATEST_TAG" | sed 's/\.[^.]*$//') | |
| echo "LIB_VERSION_WITHOUT_PATCH=$LIB_VERSION_WITHOUT_PATCH" | |
| echo "LIB_VERSION_WITHOUT_PATCH=$LIB_VERSION_WITHOUT_PATCH" >> build.env | |
| - name: Upload build.env | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: setup | |
| path: build.env | |
| if-no-files-found: error | |
| retention-days: 1 | |
| install_dependencies: | |
| runs-on: ubuntu-latest | |
| container: python:3.11-slim-buster | |
| needs: setup | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Install dependencies os packages | |
| run: | | |
| apt-get update | |
| apt-get install -y libpq-dev curl build-essential git | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Cache python venv | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Check version | |
| run: | | |
| uv --version | |
| echo testdb=${DB_HOST}:${DB_PORT}/${DB_DATABASE} with user=${DB_USER} password=${DB_PASSWORD} | |
| echo MINIO_CONSUMER_HOST=${MINIO_CONSUMER_HOST} with MINIO_CONSUMER_ACCESS_KEY=${MINIO_CONSUMER_ACCESS_KEY} MINIO_CONSUMER_SECRET_KEY=${MINIO_CONSUMER_SECRET_KEY} | |
| echo redis=${REDIS_HOST}:${REDIS_PORT} with password=${REDIS_PASSWORD} | |
| echo logger_redis=${LOGGER_REDIS_HOST}:${LOGGER_REDIS_PORT} with password=${LOGGER_REDIS_PASSWORD} | |
| shell: bash | |
| - name: Create virtual env | |
| run: | | |
| uv venv .venv --python 3.11 | |
| . .venv/bin/activate | |
| - name: Install dependencies | |
| run: | | |
| uv sync --all-extras | |
| shell: bash | |
| build: | |
| name: Build for ${{ matrix.os }} ${{ matrix.arch }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest, macos-latest ] | |
| arch: [ x64, arm64 ] | |
| exclude: | |
| - os: windows-latest | |
| arch: arm64 # Windows GitHub runners do not support ARM64 yet | |
| - os: ubuntu-latest | |
| arch: arm64 # Ubuntu GitHub runners do not support ARM64 yet | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| ref: ${{ github.event.pull_request.head.ref || github.ref }} | |
| - name: Set Up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| architecture: ${{ matrix.arch }} | |
| - name: Install uv Package Manager | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Install dependencies | |
| run: | | |
| uv sync | |
| shell: bash | |
| - name: Build Package | |
| run: | | |
| uv build | |
| shell: bash | |
| - name: Upload Build Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist-files-${{ matrix.os }}-${{ matrix.arch }} | |
| path: dist/* | |
| unit_test: | |
| runs-on: ubuntu-latest | |
| needs: install_dependencies | |
| container: python:3.11-slim-buster | |
| steps: | |
| - name: Setup git | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Load venv cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Run unit tests | |
| run: | | |
| . .venv/bin/activate | |
| which pytest | |
| uv run pytest tests_ce/unit_tests/ -n auto | |
| shell: bash | |
| functional_test: | |
| runs-on: ubuntu-latest | |
| needs: install_dependencies | |
| container: python:3.11-slim-buster | |
| services: | |
| postgres: | |
| image: postgres:13 | |
| env: | |
| POSTGRES_USER: datamimic | |
| POSTGRES_PASSWORD: datamimic | |
| POSTGRES_DB: rd-datamimic | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| mongo: | |
| image: mongo:4.4 | |
| env: | |
| MONGO_INITDB_ROOT_USERNAME: datamimic | |
| MONGO_INITDB_ROOT_PASSWORD: datamimic | |
| ports: | |
| - 27017:27017 | |
| options: --health-cmd "mongo --eval 'db.stats().ok'" --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: Setup git | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Load venv cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Run functional tests | |
| run: | | |
| . .venv/bin/activate | |
| uv run pytest tests_ce/functional_tests/ -n auto | |
| shell: bash | |
| integration_test: | |
| runs-on: ubuntu-latest | |
| needs: install_dependencies | |
| services: | |
| postgres: | |
| image: postgres:13 | |
| env: | |
| POSTGRES_USER: datamimic | |
| POSTGRES_PASSWORD: datamimic | |
| POSTGRES_DB: rd-datamimic | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| mongo: | |
| image: mongo:4.4 | |
| env: | |
| MONGO_INITDB_ROOT_USERNAME: datamimic | |
| MONGO_INITDB_ROOT_PASSWORD: datamimic | |
| ports: | |
| - 27017:27017 | |
| options: --health-cmd "mongo --eval 'db.stats().ok'" --health-interval 10s --health-timeout 5s --health-retries 5 | |
| container: python:3.11-slim-buster | |
| steps: | |
| - name: Setup git | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Load venv cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Run integration tests | |
| run: | | |
| . .venv/bin/activate | |
| uv run pytest tests_ce/integration_tests/ -n auto | |
| shell: bash | |
| publish_dev: | |
| if: github.ref == 'refs/heads/development' | |
| runs-on: ubuntu-latest | |
| needs: [ build, unit_test, functional_test, integration_test ] | |
| environment: | |
| name: testpypi | |
| url: https://test.pypi.org/p/datamimic_ce | |
| permissions: | |
| id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ | |
| contents: read | |
| steps: | |
| - name: Download Build Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: 'dist-files-*' | |
| path: dist | |
| - name: Publish distribution to TestPyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.TEST_PYPI_API_TOKEN }} | |
| repository-url: https://test.pypi.org/legacy/ | |
| packages: 'dist/*' | |
| release: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| needs: [ build, unit_test, functional_test, integration_test ] | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/datamimic_ce | |
| permissions: | |
| id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ | |
| contents: read | |
| steps: | |
| - name: Download Build Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: 'dist-files-*' | |
| path: dist | |
| - name: Publish distribution to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| packages: 'dist/*' | |
| trigger_ee_dev: | |
| if: github.ref == 'refs/heads/development' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - publish_dev | |
| steps: | |
| - name: trigger EE development pipelines | |
| run: curl -X POST --fail -F token=$EE_TOKEN -F ref=development https://gitlab.dwellerlab.com/api/v4/projects/214/trigger/pipeline; | |
| env: | |
| EE_TOKEN: ${{ secrets.EE_TOKEN }} | |
| shell: bash | |
| e2e_test: | |
| name: E2E Test on ${{ matrix.os }} ${{ matrix.arch }} | |
| runs-on: ${{ matrix.os }} | |
| needs: build | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, macos-latest, windows-latest ] | |
| arch: [ x64, arm64 ] | |
| exclude: | |
| - os: windows-latest | |
| arch: arm64 # Exclude Windows ARM64 as it's not supported | |
| - os: ubuntu-latest | |
| arch: arm64 # Ubuntu GitHub runners do not support ARM64 yet | |
| steps: | |
| - name: Set Up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| architecture: ${{ matrix.arch }} | |
| - name: Install uv Package Manager | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install uv | |
| - name: Download Build Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dist-files-${{ matrix.os }}-${{ matrix.arch }} | |
| path: dist | |
| - name: List Contents of dist Directory | |
| run: | | |
| if [ "$RUNNER_OS" == "Windows" ]; then | |
| dir dist | |
| else | |
| ls dist | |
| fi | |
| shell: bash | |
| - name: Install Built Package on Windows | |
| if: runner.os == 'Windows' | |
| run: | | |
| $wheel = Get-ChildItem -Path dist\*.whl | Select-Object -First 1 | |
| pip install $wheel.FullName | |
| shell: pwsh | |
| - name: Install Built Package on Unix | |
| if: runner.os != 'Windows' | |
| run: | | |
| pip install dist/*.whl | |
| shell: bash | |
| - name: Run DATAMIMIC Help Command | |
| run: datamimic --help | |
| - name: List Available Demos | |
| run: datamimic demo list | |
| - name: Create All Demos | |
| run: datamimic demo create --all --target my_demos | |
| - name: Run the demo-model Demo | |
| run: datamimic run ./my_demos/demo-model/datamimic.xml | |
| - name: Run the demo-condition Demo | |
| run: datamimic run ./my_demos/demo-condition/datamimic.xml | |
| - name: Run the demo-custom-components Demo | |
| run: datamimic run ./my_demos/demo-custom-components/datamimic.xml | |
| - name: Run the demo-datetime Demo | |
| run: datamimic run ./my_demos/demo-datetime/datamimic.xml |