(HF) fix(dag): fix dag argd environment (#3810) #385
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "3. [on_cd] Deployement workflow" | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - master | |
| - production | |
| permissions: | |
| contents: write | |
| id-token: write | |
| jobs: | |
| linter: | |
| uses: ./.github/workflows/reusable_linter.yml | |
| with: | |
| NOTIF_CHANNEL_ID: ${{ vars.DEFAULT_NOTIF_CHANNEL_ID }} | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| find-test-jobs: | |
| outputs: | |
| jobs: ${{ steps.find-testable-jobs.outputs.testable_jobs }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: "Trouver les Jobs testables" | |
| id: "find-testable-jobs" | |
| run: | | |
| find jobs -type d -name "tests" -exec dirname {} \; > /tmp/list.txt | |
| folders=() | |
| for folder in $(cat /tmp/list.txt); do | |
| folders+=(\"${folder}\") | |
| done | |
| json_array="[$(IFS=,; echo "${folders[*]}")]" | |
| echo "testable_jobs=$json_array" | tee -a $GITHUB_OUTPUT | |
| test-jobs: | |
| needs: find-test-jobs | |
| if: ${{ needs.find-test-jobs.outputs.jobs != '' && toJson(fromJson(needs.find-test-jobs.outputs.jobs)) != '[]' }} | |
| uses: ./.github/workflows/reusable_job_test.yml | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: ${{ fromJSON(needs.find-test-jobs.outputs.jobs) }} | |
| with: | |
| JOB_PATH: ${{ matrix.job }} | |
| NOTIF_CHANNEL_ID: ${{ vars.DEFAULT_NOTIF_CHANNEL_ID }} | |
| test-orchestration: | |
| uses: ./.github/workflows/reusable_test_orchestration.yml | |
| with: | |
| NOTIF_CHANNEL_ID: ${{ vars.DEFAULT_NOTIF_CHANNEL_ID }} | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| composer-deploy-on-dev: | |
| if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/production' }} | |
| uses: ./.github/workflows/reusable_deploy_composer.yml | |
| needs: [test-orchestration, test-jobs, linter] | |
| with: | |
| GITHUB_ENV_NAME: "dev" | |
| ENV_SHORT_NAME: "dev" | |
| COMPOSER_DAGS_BUCKET: ${{ vars.COMPOSER_DAGS_BUCKET_DEV }} | |
| AIRFLOW_NAME: "data-composer-dev" | |
| DATA_GCP_PROJECT: "passculture-data-ehp" | |
| APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_DEV }} | |
| IS_COMPOSER: true | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| airflow-deploy-on-dev: | |
| # deploying new custom gke airflow on dev | |
| if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/production' }} | |
| uses: ./.github/workflows/reusable_deploy_composer.yml | |
| needs: [test-orchestration, test-jobs, linter] | |
| with: | |
| GITHUB_ENV_NAME: "dev" | |
| ENV_SHORT_NAME: "dev" | |
| COMPOSER_DAGS_BUCKET: ${{ vars.GKE_AIRFLOW_DAGS_BUCKET_DEV }} | |
| AIRFLOW_NAME: "gke-data-airflow-dev" | |
| DATA_GCP_PROJECT: "passculture-data-ehp" | |
| APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_DEV }} | |
| IS_COMPOSER: false | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| composer-deploy-on-stg: | |
| if: github.ref == 'refs/heads/master' | |
| uses: ./.github/workflows/reusable_deploy_composer.yml | |
| needs: [test-orchestration, test-jobs, linter] | |
| with: | |
| GITHUB_ENV_NAME: "staging" | |
| ENV_SHORT_NAME: "stg" | |
| COMPOSER_DAGS_BUCKET: ${{ vars.COMPOSER_DAGS_BUCKET_STG }} | |
| AIRFLOW_NAME: "data-composer-stg" | |
| DATA_GCP_PROJECT: "passculture-data-ehp" | |
| APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_STG }} | |
| IS_COMPOSER: true | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| composer-deploy-on-prod: | |
| if: github.ref == 'refs/heads/production' | |
| uses: ./.github/workflows/reusable_deploy_composer.yml | |
| needs: [test-orchestration, test-jobs, linter, composer-deploy-on-dev] | |
| with: | |
| GITHUB_ENV_NAME: "production" | |
| ENV_SHORT_NAME: "prod" | |
| COMPOSER_DAGS_BUCKET: ${{ vars.COMPOSER_DAGS_BUCKET_PROD }} | |
| AIRFLOW_NAME: "data-composer-prod" | |
| DATA_GCP_PROJECT: "passculture-data-prod" | |
| APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_PROD }} | |
| IS_COMPOSER: true | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| mkdocs-deploy-on-prod: | |
| if: github.ref == 'refs/heads/production' | |
| uses: ./.github/workflows/reusable_deploy_mkdocs.yml | |
| needs: [linter] | |
| with: | |
| ENV_SHORT_NAME: "prod" | |
| APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_PROD }} | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| find-build-job: | |
| outputs: | |
| has-changed: ${{ steps.container-changes.outputs.output }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: "Get changed files" | |
| id: "changed-files" | |
| uses: tj-actions/changed-files@v41 | |
| with: | |
| files: "orchestration/k8s-airflow/**" | |
| json: true | |
| dir_names: true | |
| dir_names_max_depth: 4 | |
| - name: "Container changes" | |
| id: "container-changes" | |
| run: | | |
| if [ "${{ steps.changed-files.outputs.any_changed }}" == "true" ]; then | |
| echo "output=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "output=false" >> $GITHUB_OUTPUT | |
| fi | |
| k8s-deploy-on-dev: | |
| needs: [find-build-job] | |
| if: ${{ needs.find-build-job.outputs.has-changed == 'true' && ( github.ref == 'refs/heads/master' || github.event_name == 'workflow_dispatch' )}} | |
| uses: ./.github/workflows/reusable_build_image.yml | |
| with: | |
| TARGET_ENV: "dev" | |
| DATA_GCP_PROJECT: "passculture-data-ehp" | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} |