Add support for building private dependencies in CI #35715
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - 'auto-cadence-upgrade/**' | |
| - staging | |
| - trying | |
| - 'feature/**' | |
| - 'v[0-9]+.[0-9]+' | |
| pull_request: | |
| branches: | |
| - master* | |
| - 'auto-cadence-upgrade/**' | |
| - 'feature/**' | |
| - 'v[0-9]+.[0-9]+' | |
| merge_group: | |
| branches: | |
| - master | |
| env: | |
| GO_VERSION: "1.22" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| golangci: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| dir: [./, ./integration/, ./insecure/] | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Run go generate | |
| run: go generate ./... | |
| working-directory: ${{ matrix.dir }} | |
| - name: Run golangci-lint | |
| uses: golangci/golangci-lint-action@v3 | |
| with: | |
| # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. | |
| version: v1.54 | |
| args: -v | |
| working-directory: ${{ matrix.dir }} | |
| # https://github.com/golangci/golangci-lint-action/issues/244 | |
| skip-cache: true | |
| tidy: | |
| name: Tidy | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| # This task MUST run AFTER the checkout | |
| # This task loads the ssh agent and adds the cadence deploy key into the agent | |
| # This enables our go processes to pull private cadence dependencies | |
| - name: Load cadence deploy key for private repo | |
| shell: bash | |
| run: | | |
| eval "$(ssh-agent -s)" | |
| if [[ -n "${{ secrets.CADENCE_DEPLOY_KEY }}" ]]; then | |
| echo "${{ secrets.CADENCE_DEPLOY_KEY }}" | tr -d '\r' | ssh-add - | |
| echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV | |
| echo "Deploy key added to SSH agent." | |
| # Update git config to use SSH for dependencies | |
| git config --global url."[email protected]:".insteadOf "https://github.com/" | |
| # Dynamically set GOPRIVATE environment variable | |
| echo "GOPRIVATE=github.com/onflow/*" >> $GITHUB_ENV | |
| else | |
| echo "Warning: CADENCE_DEPLOY_KEY is not set. Skipping SSH key addition." | |
| fi | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Run tidy | |
| run: make tidy | |
| - name: code sanity check | |
| run: make code-sanity-check | |
| create-dynamic-test-matrix: | |
| name: Create Dynamic Test Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Set Test Matrix | |
| id: set-test-matrix | |
| run: go run tools/test_matrix_generator/matrix.go | |
| create-insecure-dynamic-test-matrix: | |
| name: Create Dynamic Unit Test Insecure Package Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Set Test Matrix | |
| id: set-test-matrix | |
| run: go run tools/test_matrix_generator/matrix.go -c insecure | |
| create-integration-dynamic-test-matrix: | |
| name: Create Dynamic Integration Test Package Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Set Test Matrix | |
| id: set-test-matrix | |
| run: go run tools/test_matrix_generator/matrix.go -c integration | |
| unit-test: | |
| name: Unit Tests (${{ matrix.targets.name }}) | |
| needs: create-dynamic-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}} | |
| ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 | |
| runs-on: ${{ matrix.targets.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| # This task loads the ssh agent and adds the cadence deploy key into the agent | |
| # This enables our go processes to pull private cadence dependencies | |
| - name: Load cadence deploy key for private repo | |
| shell: bash | |
| run: | | |
| eval "$(ssh-agent -s)" | |
| if [[ -n "${{ secrets.CADENCE_DEPLOY_KEY }}" ]]; then | |
| echo "${{ secrets.CADENCE_DEPLOY_KEY }}" | tr -d '\r' | ssh-add - | |
| echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV | |
| echo "Deploy key added to SSH agent." | |
| # Update git config to use SSH for dependencies | |
| git config --global url."[email protected]:".insteadOf "https://github.com/" | |
| # Dynamically set GOPRIVATE environment variable | |
| echo "GOPRIVATE=github.com/onflow/*" >> $GITHUB_ENV | |
| else | |
| echo "Warning: CADENCE_DEPLOY_KEY is not set. Skipping SSH key addition." | |
| fi | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.targets.name }}) | |
| run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools | |
| - name: Run tests (${{ matrix.targets.name }}) | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v4 | |
| timeout-minutes: 1 | |
| continue-on-error: true | |
| with: | |
| file: ./coverage.txt | |
| flags: unittests | |
| name: codecov-umbrella | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| unit-test-insecure: | |
| name: Unit Tests Insecure (${{ matrix.targets.name }}) | |
| needs: create-insecure-dynamic-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targets: ${{ fromJSON(needs.create-insecure-dynamic-test-matrix.outputs.dynamic-matrix)}} | |
| ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 | |
| runs-on: ${{ matrix.targets.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| # This task MUST run AFTER the checkout | |
| # This task loads the ssh agent and adds the cadence deploy key into the agent | |
| # This enables our go processes to pull private cadence dependencies | |
| - name: Load cadence deploy key for private repo | |
| shell: bash | |
| run: | | |
| eval "$(ssh-agent -s)" | |
| if [[ -n "${{ secrets.CADENCE_DEPLOY_KEY }}" ]]; then | |
| echo "${{ secrets.CADENCE_DEPLOY_KEY }}" | tr -d '\r' | ssh-add - | |
| echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV | |
| echo "Deploy key added to SSH agent." | |
| # Update git config to use SSH for dependencies | |
| git config --global url."[email protected]:".insteadOf "https://github.com/" | |
| # Dynamically set GOPRIVATE environment variable | |
| echo "GOPRIVATE=github.com/onflow/*" >> $GITHUB_ENV | |
| else | |
| echo "Warning: CADENCE_DEPLOY_KEY is not set. Skipping SSH key addition." | |
| fi | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.targets.name }}) | |
| run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools | |
| - name: Run tests (${{ matrix.targets.name }}) | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 make -C ./insecure -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v4 | |
| timeout-minutes: 1 | |
| continue-on-error: true | |
| with: | |
| file: ./coverage.txt | |
| flags: unittests | |
| name: codecov-umbrella | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| docker-build: | |
| name: Docker Build | |
| runs-on: buildjet-16vcpu-ubuntu-2204 | |
| env: | |
| CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| with: | |
| # all tags are needed for integration tests | |
| fetch-depth: 0 | |
| # This MUST run AFTER the code is checked out | |
| # This task loads the ssh agent and adds the cadence deploy key into the agent | |
| # This enables our go processes to pull private cadence dependencies | |
| - name: Load cadence deploy key for private repo | |
| shell: bash | |
| run: | | |
| eval "$(ssh-agent -s)" | |
| if [[ -n "${{ secrets.CADENCE_DEPLOY_KEY }}" ]]; then | |
| echo "${{ secrets.CADENCE_DEPLOY_KEY }}" | tr -d '\r' | ssh-add - | |
| echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV | |
| echo "Deploy key added to SSH agent." | |
| # Update git config to use SSH for dependencies | |
| git config --global url."[email protected]:".insteadOf "https://github.com/" | |
| # Dynamically set GOPRIVATE environment variable | |
| echo "GOPRIVATE=github.com/onflow/*" >> $GITHUB_ENV | |
| else | |
| echo "Warning: CADENCE_DEPLOY_KEY is not set. Skipping SSH key addition." | |
| fi | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Docker build | |
| env: | |
| CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} | |
| run: make docker-native-build-flow docker-native-build-flow-corrupt | |
| - name: Save Docker images | |
| run: | | |
| docker save \ | |
| gcr.io/flow-container-registry/access:latest \ | |
| gcr.io/flow-container-registry/collection:latest \ | |
| gcr.io/flow-container-registry/consensus:latest \ | |
| gcr.io/flow-container-registry/execution:latest \ | |
| gcr.io/flow-container-registry/ghost:latest \ | |
| gcr.io/flow-container-registry/observer:latest \ | |
| gcr.io/flow-container-registry/verification:latest \ | |
| gcr.io/flow-container-registry/access-corrupted:latest \ | |
| gcr.io/flow-container-registry/execution-corrupted:latest \ | |
| gcr.io/flow-container-registry/verification-corrupted:latest > flow-docker-images.tar | |
| - name: Cache Docker images | |
| uses: actions/cache@v3 | |
| with: | |
| path: flow-docker-images.tar | |
| # use the workflow run id as part of the cache key to ensure these docker images will only be used for a single workflow run | |
| key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} | |
| integration-test-others: | |
| name: Integration Tests Others (${{ matrix.targets.name }}) | |
| needs: create-integration-dynamic-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| targets: ${{ fromJSON(needs.create-integration-dynamic-test-matrix.outputs.dynamic-matrix)}} | |
| ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202 | |
| runs-on: ${{ matrix.targets.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| # This MUST run AFTER the code is checked out | |
| # This task loads the ssh agent and adds the cadence deploy key into the agent | |
| # This enables our go processes to pull private cadence dependencies | |
| - name: Load cadence deploy key for private repo | |
| shell: bash | |
| run: | | |
| eval "$(ssh-agent -s)" | |
| if [[ -n "${{ secrets.CADENCE_DEPLOY_KEY }}" ]]; then | |
| echo "${{ secrets.CADENCE_DEPLOY_KEY }}" | tr -d '\r' | ssh-add - | |
| echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV | |
| echo "Deploy key added to SSH agent." | |
| # Update git config to use SSH for dependencies | |
| git config --global url."[email protected]:".insteadOf "https://github.com/" | |
| # Dynamically set GOPRIVATE environment variable | |
| echo "GOPRIVATE=github.com/onflow/*" >> $GITHUB_ENV | |
| else | |
| echo "Warning: CADENCE_DEPLOY_KEY is not set. Skipping SSH key addition." | |
| fi | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Setup tests (${{ matrix.targets.name }}) | |
| run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools | |
| - name: Run tests (${{ matrix.targets.name }}) | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 make -C ./integration -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v4 | |
| timeout-minutes: 1 | |
| continue-on-error: true | |
| with: | |
| file: ./coverage.txt | |
| flags: unittests | |
| name: codecov-umbrella | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| integration-test: | |
| name: Integration Tests | |
| needs: docker-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Access Cohort1 Integration Tests | |
| make: make -C integration access-cohort1-tests | |
| runner: buildjet-4vcpu-ubuntu-2204 | |
| - name: Access Cohort2 Integration Tests | |
| make: make -C integration access-cohort2-tests | |
| runner: ubuntu-latest | |
| - name: Access Cohort3 Integration Tests | |
| make: make -C integration access-cohort3-tests | |
| runner: ubuntu-latest | |
| # test suite has single test which is flaky and needs to be fixed - reminder here to put it back when it's fixed | |
| # - name: BFT (Framework) Integration Tests | |
| # make: make -C integration bft-framework-tests | |
| # runner: ubuntu-latest | |
| - name: BFT (Protocol) Integration Tests | |
| make: make -C integration bft-protocol-tests | |
| runner: buildjet-8vcpu-ubuntu-2204 | |
| - name: BFT (Gossipsub) Integration Tests | |
| make: make -C integration bft-gossipsub-tests | |
| runner: ubuntu-latest | |
| - name: Collection Integration Tests | |
| make: make -C integration collection-tests | |
| runner: ubuntu-latest | |
| - name: Consensus Integration Tests | |
| make: make -C integration consensus-tests | |
| runner: ubuntu-latest | |
| - name: Epoch Cohort1 Integration Tests | |
| make: make -C integration epochs-cohort1-tests | |
| runner: buildjet-8vcpu-ubuntu-2204 | |
| - name: Epoch Cohort2 Integration Tests | |
| make: make -C integration epochs-cohort2-tests | |
| runner: buildjet-4vcpu-ubuntu-2204 | |
| - name: Execution Integration Tests | |
| make: make -C integration execution-tests | |
| runner: ubuntu-latest | |
| - name: Ghost Integration Tests | |
| make: make -C integration ghost-tests | |
| runner: ubuntu-latest | |
| - name: MVP Integration Tests | |
| make: make -C integration mvp-tests | |
| runner: ubuntu-latest | |
| - name: Network Integration Tests | |
| make: make -C integration network-tests | |
| runner: ubuntu-latest | |
| - name: Verification Integration Tests | |
| make: make -C integration verification-tests | |
| runner: ubuntu-latest | |
| - name: Upgrade Integration Tests | |
| make: make -C integration upgrades-tests | |
| runner: ubuntu-latest | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v3 | |
| with: | |
| # all tags are needed for integration tests | |
| fetch-depth: 0 | |
| # This MUST run AFTER the code is checked out | |
| # This task loads the ssh agent and adds the cadence deploy key into the agent | |
| # This enables our go processes to pull private cadence dependencies | |
| - name: Load cadence deploy key for private repo | |
| shell: bash | |
| run: | | |
| eval "$(ssh-agent -s)" | |
| if [[ -n "${{ secrets.CADENCE_DEPLOY_KEY }}" ]]; then | |
| echo "${{ secrets.CADENCE_DEPLOY_KEY }}" | tr -d '\r' | ssh-add - | |
| echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV | |
| echo "Deploy key added to SSH agent." | |
| # Update git config to use SSH for dependencies | |
| git config --global url."[email protected]:".insteadOf "https://github.com/" | |
| # Dynamically set GOPRIVATE environment variable | |
| echo "GOPRIVATE=github.com/onflow/*" >> $GITHUB_ENV | |
| else | |
| echo "Warning: CADENCE_DEPLOY_KEY is not set. Skipping SSH key addition." | |
| fi | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Load cached Docker images | |
| uses: actions/cache@v3 | |
| with: | |
| path: flow-docker-images.tar | |
| # use the same cache key as the docker-build job | |
| key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }} | |
| - name: Load Docker images | |
| run: docker load -i flow-docker-images.tar | |
| - name: Run tests (${{ matrix.name }}) | |
| # TODO(rbtz): re-enable when we fix exisiting races. | |
| #env: | |
| # RACE_DETECTOR: 1 | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 35 | |
| max_attempts: 5 | |
| command: VERBOSE=1 ${{ matrix.make }} |