is running Github Datamimic CE CI on rapiddweller/datamimic #26
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Github Datamimic CE CI | |
| run-name: ${{ github.run_actor }} is running ${{ github.workflow }} on ${{ github.repository }} | |
| on: | |
| push: | |
| branches: | |
| - development | |
| tags: | |
| - '[0-9]+.[0-9]+.[0-9]+' | |
| pull_request: | |
| branches: | |
| - development | |
| env: | |
| DATAMIMIC_LIB_ENVIRONMENT: lib_staging | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Export latest tag | |
| run: | | |
| export RD_PROJECT_LATEST_TAG=${GITHUB_REF#refs/*/} | |
| echo "RD_PROJECT_LATEST_TAG=$RD_PROJECT_LATEST_TAG" | |
| echo "RD_PROJECT_LATEST_TAG=$RD_PROJECT_LATEST_TAG" >> build.env | |
| - name: Export lib version without patch | |
| run: | | |
| export LIB_VERSION_WITHOUT_PATCH=$(echo "$RD_PROJECT_LATEST_TAG" | sed 's/\.[^.]*$//') | |
| echo "LIB_VERSION_WITHOUT_PATCH=$LIB_VERSION_WITHOUT_PATCH" | |
| echo "LIB_VERSION_WITHOUT_PATCH=$LIB_VERSION_WITHOUT_PATCH" >> build.env | |
| - name: Upload build.env | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: setup | |
| path: build.env | |
| if-no-files-found: error | |
| retention-days: 1 | |
| install_dependencies: | |
| runs-on: ubuntu-latest | |
| container: python:3.11-slim-buster | |
| needs: setup | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Install dependencies os packages | |
| run: | | |
| apt-get update | |
| apt-get install -y libpq-dev curl build-essential git | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Cache python venv | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Check version | |
| run: | | |
| uv --version | |
| echo testdb=${DB_HOST}:${DB_PORT}/${DB_DATABASE} with user=${DB_USER} password=${DB_PASSWORD} | |
| echo MINIO_CONSUMER_HOST=${MINIO_CONSUMER_HOST} with MINIO_CONSUMER_ACCESS_KEY=${MINIO_CONSUMER_ACCESS_KEY} MINIO_CONSUMER_SECRET_KEY=${MINIO_CONSUMER_SECRET_KEY} | |
| echo redis=${REDIS_HOST}:${REDIS_PORT} with password=${REDIS_PASSWORD} | |
| echo logger_redis=${LOGGER_REDIS_HOST}:${LOGGER_REDIS_PORT} with password=${LOGGER_REDIS_PASSWORD} | |
| shell: bash | |
| - name: Create virtual env | |
| run: | | |
| uv venv .venv --python 3.11 | |
| . .venv/bin/activate | |
| - name: Install dependencies | |
| run: | | |
| uv sync --all-extras | |
| shell: bash | |
| unit_test: | |
| runs-on: ubuntu-latest | |
| needs: install_dependencies | |
| container: python:3.11-slim-buster | |
| steps: | |
| - name: Setup git | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Load venv cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Run unit tests | |
| run: | | |
| . .venv/bin/activate | |
| which pytest | |
| uv run pytest tests_ce/unit_tests/ -n auto | |
| shell: bash | |
| functional_test: | |
| runs-on: ubuntu-latest | |
| needs: install_dependencies | |
| container: python:3.11-slim-buster | |
| services: | |
| postgres: | |
| image: postgres:13 | |
| env: | |
| POSTGRES_USER: datamimic | |
| POSTGRES_PASSWORD: datamimic | |
| POSTGRES_DB: rd-datamimic | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| mongo: | |
| image: mongo:4.4 | |
| env: | |
| MONGO_INITDB_ROOT_USERNAME: datamimic | |
| MONGO_INITDB_ROOT_PASSWORD: datamimic | |
| ports: | |
| - 27017:27017 | |
| options: --health-cmd "mongo --eval 'db.stats().ok'" --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - name: Setup git | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Load venv cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Run functional tests | |
| run: | | |
| . .venv/bin/activate | |
| uv run pytest tests_ce/functional_tests/ -n auto | |
| shell: bash | |
| integration_test: | |
| runs-on: ubuntu-latest | |
| services: | |
| postgres: | |
| image: postgres:13 | |
| env: | |
| POSTGRES_USER: datamimic | |
| POSTGRES_PASSWORD: datamimic | |
| POSTGRES_DB: rd-datamimic | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| mongo: | |
| image: mongo:4.4 | |
| env: | |
| MONGO_INITDB_ROOT_USERNAME: datamimic | |
| MONGO_INITDB_ROOT_PASSWORD: datamimic | |
| ports: | |
| - 27017:27017 | |
| options: --health-cmd "mongo --eval 'db.stats().ok'" --health-interval 10s --health-timeout 5s --health-retries 5 | |
| container: python:3.11-slim-buster | |
| needs: install_dependencies | |
| steps: | |
| - name: Setup git | |
| run: | | |
| apt-get update | |
| apt-get install -y git | |
| git config --global --add safe.directory $(realpath .) | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Load venv cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-python-${{ hashFiles('./pyproject.toml') }} | |
| - name: Run integration tests | |
| run: | | |
| . .venv/bin/activate | |
| uv run pytest tests_ce/integration_tests/ -n auto | |
| shell: bash | |
| publish_dev: | |
| if: github.ref == 'refs/heads/development' | |
| runs-on: ubuntu-latest | |
| needs: [ unit_test, functional_test, integration_test ] | |
| environment: | |
| name: testpypi | |
| url: https://test.pypi.org/p/datamimic_ce | |
| permissions: | |
| id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Package | |
| # IMPORTANT: Use "pip install build" instead of "uv add" to avoid updating pyproject.toml, which accidentally create dirty package version | |
| run: | | |
| uv venv .venv --python 3.11 | |
| . .venv/bin/activate | |
| which python | |
| uv pip install build | |
| python -m build | |
| shell: bash | |
| - name: Publish distribution to TestPyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: https://test.pypi.org/legacy/ | |
| # verbose: true | |
| release: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| needs: [ unit_test, functional_test, integration_test ] | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/datamimic_ce | |
| permissions: | |
| id-token: write # IMPORTANT: mandatory for trusted publishing https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Package | |
| run: | | |
| uv venv .venv --python 3.11 | |
| . .venv/bin/activate | |
| which python | |
| uv pip install build | |
| python -m build | |
| shell: bash | |
| # - name: Publish distribution to TestPyPI | |
| # uses: pypa/gh-action-pypi-publish@release/v1 | |
| trigger_ee_dev: | |
| if: github.ref == 'refs/heads/development' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - publish_dev | |
| steps: | |
| - name: trigger EE development pipelines | |
| run: curl -X POST --fail -F token=$EE_TOKEN -F ref=development https://gitlab.dwellerlab.com/api/v4/projects/214/trigger/pipeline; | |
| env: | |
| EE_TOKEN: ${{ secrets.EE_TOKEN }} | |
| shell: bash |