From 3d5df703b7feecbe6901d773d3c3821462d38acf Mon Sep 17 00:00:00 2001 From: Miao ZhiCheng Date: Thu, 29 Aug 2024 10:30:53 +0300 Subject: [PATCH] Workflow: tweak test-local-subgraph (#2013) **Problem** At some point, the sdk-core/subgraph test cases started to fail with "socket hang up" error when querying subgraph. After some research, it was discovered that it is due to the usage of an older version of the "cross-fetch" polyfill which didn't support node 20+, see [here](https://github.com/jasonkuhrt/graphql-request/issues/628) and [here](https://github.com/jasonkuhrt/graphql-request/pull/597). **Solution** The newer cross-fetch project solved this. The solution is to fix the "resolution" of the cross-fetch in this project to the latest, for now. Additionally, this PR also: a) include some minor refactoring around the test setup process, b) test against both node 18 and 20 for SDK core, and test against node 20 for subgraph. --- .../call.test-ethereum-contracts.yml | 5 +- .../workflows/call.test-local-subgraph.yml | 47 +++++++--------- .github/workflows/call.test-sdk-core.yml | 42 ++++++--------- ...subgraph-on-previous-sdk-core-versions.yml | 11 ++-- package.json | 3 +- packages/subgraph/default.nix | 5 +- packages/subgraph/docker-compose.yml | 5 +- packages/subgraph/tasks/setup-graph-node.sh | 30 +++-------- yarn.lock | 53 ++----------------- 9 files changed, 56 insertions(+), 145 deletions(-) mode change 100644 => 100755 packages/subgraph/tasks/setup-graph-node.sh diff --git a/.github/workflows/call.test-ethereum-contracts.yml b/.github/workflows/call.test-ethereum-contracts.yml index 7b3a5f2627..64d8b0b2dc 100644 --- a/.github/workflows/call.test-ethereum-contracts.yml +++ b/.github/workflows/call.test-ethereum-contracts.yml @@ -56,11 +56,10 @@ jobs: - name: Set cache key prefix id: set-cache-key-prefix run: | - node_version=${{ matrix.node-version }} if [ "${{ inputs.run-coverage-tests }}" == false ]; then - v=ethereum-contracts-test-${node_version}- + v=ethereum-contracts-${{ env.DEV_SHELL_NAME }}-test- else - v=ethereum-contracts-coverage-${node_version}- + v=ethereum-contracts-${{ env.DEV_SHELL_NAME }}-coverage- fi echo "cache_key_prefix=$v" >> "$GITHUB_OUTPUT" shell: bash diff --git a/.github/workflows/call.test-local-subgraph.yml b/.github/workflows/call.test-local-subgraph.yml index eea711bd78..cb05a6a288 100644 --- a/.github/workflows/call.test-local-subgraph.yml +++ b/.github/workflows/call.test-local-subgraph.yml @@ -4,17 +4,16 @@ on: workflow_call: env: - # FIXME: subgraph test fails with node20 - DEV_SHELL_NAME: ci-node18 + DEV_SHELL_NAME: ci-node20 + ethereum-contracts-working-directory: ./packages/ethereum-contracts + sdk-core-working-directory: ./packages/sdk-core + subgraph-working-directory: ./packages/subgraph jobs: subgraph-unit-tests: name: Run subgraph unit tests runs-on: ubuntu-latest - env: - subgraph-working-directory: ./packages/subgraph - defaults: run: shell: nix develop .#${{ env.DEV_SHELL_NAME }} -c bash -xe {0} @@ -27,7 +26,7 @@ jobs: uses: ./.github/actions/setup-workflow-base with: package-name: subgraph - cache-key-prefix: subgraph-unit-tests- + cache-key-prefix: subgraph-${{ env.DEV_SHELL_NAME }}-unit-tests- dev-shell-name: ${{ env.DEV_SHELL_NAME }} - name: Install dependencies @@ -42,7 +41,7 @@ jobs: run: | yarn lint yarn build - working-directory: ./packages/ethereum-contracts + working-directory: ${{ env.ethereum-contracts-working-directory }} - name: Run unit tests if: steps.base.outputs.cache-hit != 'true' @@ -54,10 +53,6 @@ jobs: runs-on: ubuntu-latest - env: - subgraph-working-directory: ./packages/subgraph - sdk-core-working-directory: ./packages/sdk-core - defaults: run: shell: nix develop .#${{ env.DEV_SHELL_NAME }} -c bash -xe {0} @@ -70,7 +65,7 @@ jobs: uses: ./.github/actions/setup-workflow-base with: package-name: subgraph - cache-key-prefix: subgraph-integration-tests- + cache-key-prefix: subgraph-${{ env.DEV_SHELL_NAME }}-integration-tests- dev-shell-name: ${{ env.DEV_SHELL_NAME }} - name: Install dependencies @@ -80,12 +75,7 @@ jobs: - name: Build contracts if: steps.base.outputs.cache-hit != 'true' run: yarn build - working-directory: ./packages/ethereum-contracts - - - name: Start hardhat node - if: steps.base.outputs.cache-hit != 'true' - run: ./tasks/startHardhatNode.sh start - working-directory: ${{ env.sdk-core-working-directory }} + working-directory: ${{ env.ethereum-contracts-working-directory }} - name: Build SDK core if: steps.base.outputs.cache-hit != 'true' @@ -93,28 +83,27 @@ jobs: run: yarn build working-directory: ${{ env.sdk-core-working-directory }} + - name: Start hardhat node + if: steps.base.outputs.cache-hit != 'true' + run: ./tasks/startHardhatNode.sh start + working-directory: ${{ env.sdk-core-working-directory }} + - name: Deploy framework and tokens if: steps.base.outputs.cache-hit != 'true' run: npx hardhat run dev-scripts/run-deploy-contracts-and-token.js --network localhost - working-directory: ./packages/ethereum-contracts + working-directory: ${{ env.ethereum-contracts-working-directory }} - name: Prepare files for local testing if: steps.base.outputs.cache-hit != 'true' run: yarn prepare-local working-directory: ${{ env.subgraph-working-directory }} - - name: Run setup-graph-node - if: steps.base.outputs.cache-hit != 'true' - run: | - chmod +x ./tasks/setup-graph-node.sh - ./tasks/setup-graph-node.sh - working-directory: ${{ env.subgraph-working-directory }} - - - name: Docker compose + - name: Setup graph node if: steps.base.outputs.cache-hit != 'true' run: | - docker rm subgraph_graph-node_1 || true - docker compose up & + ./tasks/setup-graph-node.sh & + # wait for it to warm up + sleep 30 working-directory: ${{ env.subgraph-working-directory }} - name: Run subgraph integration test suite diff --git a/.github/workflows/call.test-sdk-core.yml b/.github/workflows/call.test-sdk-core.yml index b6ffed2ffb..d6341c3e2b 100644 --- a/.github/workflows/call.test-sdk-core.yml +++ b/.github/workflows/call.test-sdk-core.yml @@ -29,10 +29,9 @@ jobs: id: set-matrix run: | if [ "${{ inputs.run-coverage-tests }}" == "true" ];then - echo "matrix={\"node-version\": [18]}" >> "$GITHUB_OUTPUT" + echo "matrix={\"node-version\": [20]}" >> "$GITHUB_OUTPUT" else - # FIXME: subgraph test fails with node20 - echo "matrix={\"node-version\": [18]}" >> "$GITHUB_OUTPUT" + echo "matrix={\"node-version\": [18, 20]}" >> "$GITHUB_OUTPUT" fi test-sdk-core: @@ -63,11 +62,10 @@ jobs: - name: Set cache key prefix id: set-cache-key-prefix run: | - node_version=${{ matrix.node-version }} if [ "${{ inputs.run-coverage-tests }}" == false ]; then - v=sdk-core-test-${node_version}- + v=sdk-core-${{ env.DEV_SHELL_NAME }}-test- else - v=sdk-core-coverage-${node_version}- + v=sdk-core-${{ env.DEV_SHELL_NAME }}-coverage- fi echo "cache_key_prefix=$v" >> "$GITHUB_OUTPUT" shell: bash @@ -87,19 +85,12 @@ jobs: npx tsc --version npx hardhat --version - - name: Lint and build essentials + - name: Build contracts if: steps.base.outputs.cache-hit != 'true' run: | - yarn lint yarn build working-directory: ${{ env.ethereum-contracts-working-directory }} - - name: Start hardhat node - if: steps.base.outputs.cache-hit != 'true' - run: | - ./tasks/startHardhatNode.sh start - working-directory: ${{ env.sdk-core-working-directory }} - - name: Build SDK Core if: steps.base.outputs.cache-hit != 'true' # build sdk-core because of auto linking to dependency @@ -108,7 +99,12 @@ jobs: yarn build working-directory: ${{ env.sdk-core-working-directory }} - - name: Deploy Framework and Tokens + - name: Start hardhat node + if: steps.base.outputs.cache-hit != 'true' + run: ./tasks/startHardhatNode.sh start + working-directory: ${{ env.sdk-core-working-directory }} + + - name: Deploy framework and tokens if: steps.base.outputs.cache-hit != 'true' run: npx hardhat run dev-scripts/run-deploy-contracts-and-token.js --network localhost working-directory: ${{ env.ethereum-contracts-working-directory }} @@ -118,26 +114,18 @@ jobs: run: yarn prepare-local working-directory: ${{ env.subgraph-working-directory }} - - name: Run setup-graph-node - if: steps.base.outputs.cache-hit != 'true' - run: | - chmod +x ./tasks/setup-graph-node.sh - ./tasks/setup-graph-node.sh - working-directory: ${{ env.subgraph-working-directory }} - - - name: Docker compose + - name: Setup graph node if: steps.base.outputs.cache-hit != 'true' run: | - docker rm subgraph_graph-node_1 || true - docker compose up & + ./tasks/setup-graph-node.sh & + # wait for it to warm up + sleep 30 working-directory: ${{ env.subgraph-working-directory }} - name: Build and deploy local subgraph if: steps.base.outputs.cache-hit != 'true' run: | yarn build-and-deploy-local - # artificial slow down to give the subgraph time to sync - sleep 30 working-directory: ${{ env.subgraph-working-directory }} - name: Run test suite diff --git a/.github/workflows/call.test-subgraph-on-previous-sdk-core-versions.yml b/.github/workflows/call.test-subgraph-on-previous-sdk-core-versions.yml index e13da59427..f1fd34b8af 100644 --- a/.github/workflows/call.test-subgraph-on-previous-sdk-core-versions.yml +++ b/.github/workflows/call.test-subgraph-on-previous-sdk-core-versions.yml @@ -62,16 +62,11 @@ jobs: run: yarn prepare-local working-directory: ./packages/subgraph - - name: "Run setup-graph-node" - run: | - chmod +x ./tasks/setup-graph-node.sh - ./tasks/setup-graph-node.sh - working-directory: ./packages/subgraph - - name: "Docker compose" run: | - docker rm subgraph_graph-node_1 || true - docker compose up & + ./tasks/setup-graph-node.sh & + # wait 5 seconds for it to warm up + sleep 5 working-directory: ./packages/subgraph - name: "Prepare and Deploy Local Subgraph" diff --git a/package.json b/package.json index 4fa3381846..427e96d9d2 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,8 @@ "web3": "^1.10.4" }, "resolutions": { - "mocha": "^10.2.0" + "mocha": "^10.2.0", + "cross-fetch": "^4.0.0" }, "workspaces": { "packages": [ diff --git a/packages/subgraph/default.nix b/packages/subgraph/default.nix index aa4a2cc1c1..9e528f88da 100644 --- a/packages/subgraph/default.nix +++ b/packages/subgraph/default.nix @@ -11,13 +11,16 @@ includedFiles = [ ./package.json ./config - ./matchstick.yaml ./schema.graphql ./tsconfig.json ./types ./src ./scripts + # for testing + ./docker-compose.yml + ./matchstick.yaml ./tests + ./tasks ]; }; } diff --git a/packages/subgraph/docker-compose.yml b/packages/subgraph/docker-compose.yml index 7848978088..2950fc4e79 100644 --- a/packages/subgraph/docker-compose.yml +++ b/packages/subgraph/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3" services: graph-node: image: graphprotocol/graph-node:v0.35.0 @@ -11,15 +10,13 @@ services: depends_on: - ipfs - postgres - extra_hosts: - - host.docker.internal:host-gateway environment: postgres_host: postgres postgres_user: graph-node postgres_pass: let-me-in postgres_db: graph-node ipfs: "ipfs:5001" - ethereum: "mainnet:http://host.docker.internal:8545" + ethereum: "mainnet:http://${DOCKER_HOST_IP}:8545" GRAPH_LOG: info ipfs: image: ipfs/go-ipfs:v0.4.23 diff --git a/packages/subgraph/tasks/setup-graph-node.sh b/packages/subgraph/tasks/setup-graph-node.sh old mode 100644 new mode 100755 index f7758a3d98..bb9a699b69 --- a/packages/subgraph/tasks/setup-graph-node.sh +++ b/packages/subgraph/tasks/setup-graph-node.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e +set -ex if ! which docker >/dev/null 2>&1 ; then echo "Please install 'docker' first" @@ -17,26 +17,10 @@ if ! which jq >/dev/null 2>&1; then exit 1 fi -# Create the graph-node container -docker compose up --no-start graph-node +# this creates the bridged network for the composed ervices (network id: subgraph_default) +docker compose up --no-start +# we then extract the gateway ip address and export it +DOCKER_HOST_IP=$(docker network inspect subgraph_default | jq -r '.[0].IPAM.Config[].Gateway') -# Start graph-node so we can inspect it -docker compose start graph-node - -# Identify the container ID -CONTAINER_ID=$(docker container ls | grep graph-node | cut -d' ' -f1) - -# Inspect the container to identify the host IP address -HOST_IP=$(docker inspect "$CONTAINER_ID" | jq -r .[0].NetworkSettings.Networks[].Gateway) - -echo "Host IP: $HOST_IP" - -# Inject the host IP into docker-compose.yml -sed -i -e "s/host.docker.internal/$HOST_IP/g" docker-compose.yml - -function stop_graph_node { - # Ensure graph-node is stopped - docker compose stop graph-node -} - -trap stop_graph_node EXIT +# docker compose with required variables +DOCKER_HOST_IP=$DOCKER_HOST_IP docker compose up diff --git a/yarn.lock b/yarn.lock index 5eea612513..1b89030f9c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7578,22 +7578,7 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== -cross-fetch@^2.1.0: - version "2.2.6" - resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-2.2.6.tgz#2ef0bb39a24ac034787965c457368a28730e220a" - integrity sha512-9JZz+vXCmfKUZ68zAptS7k4Nu8e2qcibe7WVZYps7sAgk5R8GYTc+T1WR0v1rlP9HxgARmOX1UTIJZFytajpNA== - dependencies: - node-fetch "^2.6.7" - whatwg-fetch "^2.0.4" - -cross-fetch@^3.1.4, cross-fetch@^3.1.5: - version "3.1.5" - resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" - integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== - dependencies: - node-fetch "2.6.7" - -cross-fetch@^4.0.0: +cross-fetch@^2.1.0, cross-fetch@^3.1.4, cross-fetch@^3.1.5, cross-fetch@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983" integrity sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g== @@ -17264,7 +17249,7 @@ string-format@^2.0.0: resolved "https://registry.yarnpkg.com/string-format/-/string-format-2.0.0.tgz#f2df2e7097440d3b65de31b6d40d54c96eaffb9b" integrity sha512-bbEs3scLeYNXLecRRuk6uJxdXUSj6le/8rNPHChIJTn2V79aXVTR1EH2OH5zLKKoz0V02fOUKZZcw01pLUShZA== -"string-width-cjs@npm:string-width@^4.2.0": +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -17282,15 +17267,6 @@ string-width@^1.0.1: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - string-width@^5.0.1, string-width@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" @@ -17382,7 +17358,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -17410,13 +17386,6 @@ strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -19954,11 +19923,6 @@ websocket@^1.0.32: utf-8-validate "^5.0.2" yaeti "^0.0.6" -whatwg-fetch@^2.0.4: - version "2.0.4" - resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz#dde6a5df315f9d39991aa17621853d720b85566f" - integrity sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng== - whatwg-mimetype@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7" @@ -20084,7 +20048,7 @@ workerpool@6.2.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343" integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -20110,15 +20074,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"