From f6b942c0db06e8e7acea6756ebe2d64a143bc49f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 25 Sep 2024 20:44:17 +0800 Subject: [PATCH 1/8] Performance history canary This PR adds a "canary" build to the performance regression CI of OpenJDK. The "canary" is a chosen revision of mmtk-core and mmtk-openjdk that is tested alongside each merged PR. The performance of the "canary" should not change unless there is an environment change. Spotting a change in the "canary" performance can help us identify environment changes that are unintended or otherwise unnoticed. TODO: Add a script to select the latest release tag as the canary instead of hard-coding a version. --- .github/workflows/perf-regression-ci.yml | 63 +++++++++++++++++------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 09cea5f6b6..2072b2c840 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -98,17 +98,18 @@ jobs: runs-on: [self-hosted, Linux, freq-scaling-off] timeout-minutes: 1440 steps: - - name: Checkout MMTk Core + # checkout latest versions + - name: Checkout MMTk Core (latest) uses: actions/checkout@v4 with: - path: mmtk-core - - name: Checkout OpenJDK Binding + path: latest/mmtk-core + - name: Checkout OpenJDK Binding (latest) uses: actions/checkout@v4 with: repository: mmtk/mmtk-openjdk - path: mmtk-openjdk - - name: Checkout OpenJDK - working-directory: mmtk-openjdk + path: latest/mmtk-openjdk + - name: Checkout OpenJDK (latest) + working-directory: latest/mmtk-openjdk run: | ./.github/scripts/ci-checkout.sh # checkout perf-kit @@ -120,21 +121,42 @@ jobs: path: ci-perf-kit token: ${{ secrets.CI_ACCESS_TOKEN }} submodules: true + # checkout canary versions. + # Currently using a release version. + # We need a script to find the latest stable version from a list of tags (`git tags -l`). + # Alternatively, we may use the latest commit in the last epoch once we stablize the idea of epoch. + - name: Checkout MMTk Core (canary) + uses: actions/checkout@v4 + with: + ref: "v0.27.0" + path: canary/mmtk-core + - name: Checkout OpenJDK Binding (canary) + uses: actions/checkout@v4 + with: + ref: "v0.27.0" + repository: mmtk/mmtk-openjdk + path: canary/mmtk-openjdk + - name: Checkout OpenJDK (canary) + working-directory: canary/mmtk-openjdk + run: | + ./.github/scripts/ci-checkout.sh # setup - - name: Overwrite MMTk core in openjdk binding - run: cp -r mmtk-core mmtk-openjdk/repos/ + - name: Setup directory structures + run: | + for BASE_DIR in . ./canary; do + pushd $BASE_DIR + # copy mmtk-core repo + cp -r mmtk-core mmtk-openjdk/repos/ + # replace dependency + ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path repos/mmtk-core + # cleanup previous build + rm -rf mmtk-openjdk/repos/openjdk/scratch + rm -rf mmtk-openjdk/repos/openjdk/build + popd + done - name: Setup Rust Toolchain + # This seems unused. run: echo "RUSTUP_TOOLCHAIN=`cat mmtk-core/rust-toolchain`" >> $GITHUB_ENV - # cleanup previosu build - - name: Cleanup previous build - run: | - rm -rf mmtk-openjdk/repos/openjdk/scratch - rm -rf mmtk-openjdk/repos/openjdk/build - - name: Setup - run: | - ./ci-perf-kit/scripts/history-run-setup.sh - sed -i 's/^mmtk[[:space:]]=/#ci:mmtk=/g' mmtk-openjdk/mmtk/Cargo.toml - sed -i 's/^#[[:space:]]mmtk/mmtk/g' mmtk-openjdk/mmtk/Cargo.toml - id: branch # we cannot use env vars in action input (the deploy step). So put the env var to this step's outputs. run: echo "branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT @@ -145,7 +167,10 @@ jobs: export RESULT_REPO_BRANCH=${{ env.RESULT_REPO_BRANCH }} export RESULT_REPO_ACCESS_TOKEN=${{ secrets.CI_ACCESS_TOKEN }} export FROM_DATE=2020-07-10 - ./ci-perf-kit/scripts/openjdk-history-run.sh ./mmtk-openjdk ./reports/${{ steps.branch.outputs.branch_name }} + ./ci-perf-kit/scripts/openjdk-history-run.sh \ + ./latest/mmtk-openjdk \ + ./canary/mmtk-openjdk \ + ./reports/${{ steps.branch.outputs.branch_name }} # deploy - name: Deploy to Github Page if: ${{ env.DEPLOY == 'true' }} From d7d55a1e16f2c569413eff056ea323aa472dc240 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 14 Oct 2024 17:47:51 +0800 Subject: [PATCH 2/8] Fix typo and update canary version --- .github/workflows/perf-regression-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 2072b2c840..7936c8efc5 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -128,12 +128,12 @@ jobs: - name: Checkout MMTk Core (canary) uses: actions/checkout@v4 with: - ref: "v0.27.0" + ref: "v0.28.0" path: canary/mmtk-core - name: Checkout OpenJDK Binding (canary) uses: actions/checkout@v4 with: - ref: "v0.27.0" + ref: "v0.28.0" repository: mmtk/mmtk-openjdk path: canary/mmtk-openjdk - name: Checkout OpenJDK (canary) @@ -143,7 +143,7 @@ jobs: # setup - name: Setup directory structures run: | - for BASE_DIR in . ./canary; do + for BASE_DIR in ./latest ./canary; do pushd $BASE_DIR # copy mmtk-core repo cp -r mmtk-core mmtk-openjdk/repos/ From 9d76f12bd809bd6a6a2c305dfea600595d9faf94 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 14 Oct 2024 17:59:57 +0800 Subject: [PATCH 3/8] Fix mmtk-core repo path --- .github/workflows/perf-regression-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 7936c8efc5..4969861996 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -148,7 +148,7 @@ jobs: # copy mmtk-core repo cp -r mmtk-core mmtk-openjdk/repos/ # replace dependency - ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path repos/mmtk-core + ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path ../repos/mmtk-core # cleanup previous build rm -rf mmtk-openjdk/repos/openjdk/scratch rm -rf mmtk-openjdk/repos/openjdk/build From 7ff02eda377118c9fb92a77fb7e4c2580af7ad3d Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 15 Oct 2024 12:07:58 +0800 Subject: [PATCH 4/8] Fix mmtk-core path again. --- .github/workflows/perf-regression-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 4969861996..4e4d46a59a 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -145,10 +145,11 @@ jobs: run: | for BASE_DIR in ./latest ./canary; do pushd $BASE_DIR - # copy mmtk-core repo - cp -r mmtk-core mmtk-openjdk/repos/ # replace dependency - ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path ../repos/mmtk-core + # Note that ci-replace-mmtk-dep.sh will apply `realpath()` to the `--mmtk-core-path` option. + # so we specify the relative path from the PWD to the mmtk-core repo. + ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml \ + --mmtk-core-path mmtk-core # cleanup previous build rm -rf mmtk-openjdk/repos/openjdk/scratch rm -rf mmtk-openjdk/repos/openjdk/build From 3c29d9be1689a796dd362fd91a48435546d9fcca Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 18 Oct 2024 11:41:22 +0800 Subject: [PATCH 5/8] Not setting RUSTUP_TOOLCHAIN env var. The latest and the canary version may use different toolchain, and will be selected automatically when compiling them. --- .github/workflows/perf-regression-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 4e4d46a59a..ffc0cccf45 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -155,9 +155,6 @@ jobs: rm -rf mmtk-openjdk/repos/openjdk/build popd done - - name: Setup Rust Toolchain - # This seems unused. - run: echo "RUSTUP_TOOLCHAIN=`cat mmtk-core/rust-toolchain`" >> $GITHUB_ENV - id: branch # we cannot use env vars in action input (the deploy step). So put the env var to this step's outputs. run: echo "branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT From 4d3c5235a9217ce2cb008ef000d81e679fd44482 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 18 Oct 2024 14:42:35 +0800 Subject: [PATCH 6/8] Minor changes Extract canary version to an environment variable. Do not use secret when checking out ci-perf-kit because it is a public repo now. Add a name to the "branch" action. --- .github/workflows/perf-regression-ci.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index ffc0cccf45..842b914ff4 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -97,6 +97,12 @@ jobs: openjdk-perf-regression: runs-on: [self-hosted, Linux, freq-scaling-off] timeout-minutes: 1440 + env: + # This version will be used as the canary version, and will be used to checkout both + # `mmtk-core` and `mmtk-openjdk`. We choose one release version for this purpose. We may + # change to another release version if necessary, or introduce a mechanism to dynamically + # choose the canary in the future. + CANARY_VERSION: "v0.28.0" steps: # checkout latest versions - name: Checkout MMTk Core (latest) @@ -119,21 +125,17 @@ jobs: repository: mmtk/ci-perf-kit ref: "0.8.0" path: ci-perf-kit - token: ${{ secrets.CI_ACCESS_TOKEN }} submodules: true # checkout canary versions. - # Currently using a release version. - # We need a script to find the latest stable version from a list of tags (`git tags -l`). - # Alternatively, we may use the latest commit in the last epoch once we stablize the idea of epoch. - name: Checkout MMTk Core (canary) uses: actions/checkout@v4 with: - ref: "v0.28.0" + ref: $CANARY_VERSION path: canary/mmtk-core - name: Checkout OpenJDK Binding (canary) uses: actions/checkout@v4 with: - ref: "v0.28.0" + ref: $CANARY_VERSION repository: mmtk/mmtk-openjdk path: canary/mmtk-openjdk - name: Checkout OpenJDK (canary) @@ -155,7 +157,8 @@ jobs: rm -rf mmtk-openjdk/repos/openjdk/build popd done - - id: branch + - name: Setup branch name + id: branch # we cannot use env vars in action input (the deploy step). So put the env var to this step's outputs. run: echo "branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT # run From 569e357864d357bc5b42ae4d096e380f1e35277a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 18 Oct 2024 15:07:52 +0800 Subject: [PATCH 7/8] Fix env var usage --- .github/workflows/perf-regression-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 842b914ff4..b54792cec7 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -130,12 +130,12 @@ jobs: - name: Checkout MMTk Core (canary) uses: actions/checkout@v4 with: - ref: $CANARY_VERSION + ref: ${{ env.CANARY_VERSION }} path: canary/mmtk-core - name: Checkout OpenJDK Binding (canary) uses: actions/checkout@v4 with: - ref: $CANARY_VERSION + ref: ${{ env.CANARY_VERSION }} repository: mmtk/mmtk-openjdk path: canary/mmtk-openjdk - name: Checkout OpenJDK (canary) From 4061cc577d438d3fad3c04e4653edd68b2e8a18e Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 26 Sep 2024 13:17:09 +0800 Subject: [PATCH 8/8] Testing regression history canary --- .github/workflows/perf-regression-ci.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index b54792cec7..ad99816ab7 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -8,13 +8,13 @@ on: # READ BEFORE ENABLING THE TRIGGER BELOW # This trigger is only used when testing the scripts in a branch, and should be commented out in other cases. # If this trigger is used, please change the following env: RESULT_REPO_BRANCH -> 'test' (MUST), DEPLOY -> 'false' (optional) - # pull_request: - # branches: - # - master + pull_request: + branches: + - master env: # The branch to save run data and plot graph from. Use 'self-hosted' for master, use 'test' or anything else for testing in a branch. - RESULT_REPO_BRANCH: 'self-hosted' + RESULT_REPO_BRANCH: 'test' # Whether we deploy the generated page. Set to true for master. DEPLOY: true # Directories in ci-perf-kit that will be uploaded as artifacts. The dirs can be found in ci-perf-kit/scripts/common.sh @@ -24,6 +24,7 @@ env: jobs: # JikesRVM jikesrvm-perf-regression: + if: false # Disable this to test openjdk regression, only. runs-on: [self-hosted, Linux, freq-scaling-off] timeout-minutes: 1440 steps: @@ -114,6 +115,12 @@ jobs: with: repository: mmtk/mmtk-openjdk path: latest/mmtk-openjdk + - name: Modify latest version toolchain + working-directory: latest/mmtk-openjdk + run: | + cat mmtk/rust-toolchain + echo "stable" > mmtk/rust-toolchain + cat mmtk/rust-toolchain - name: Checkout OpenJDK (latest) working-directory: latest/mmtk-openjdk run: | @@ -122,8 +129,8 @@ jobs: - name: Checkout Perf Kit uses: actions/checkout@v4 with: - repository: mmtk/ci-perf-kit - ref: "0.8.0" + repository: wks/ci-perf-kit + ref: "feature/history-canary" path: ci-perf-kit submodules: true # checkout canary versions. @@ -168,6 +175,7 @@ jobs: export RESULT_REPO_BRANCH=${{ env.RESULT_REPO_BRANCH }} export RESULT_REPO_ACCESS_TOKEN=${{ secrets.CI_ACCESS_TOKEN }} export FROM_DATE=2020-07-10 + export OPENJDK_HISTORY_RUN_TEST_FAST=1 ./ci-perf-kit/scripts/openjdk-history-run.sh \ ./latest/mmtk-openjdk \ ./canary/mmtk-openjdk \ @@ -196,6 +204,7 @@ jobs: if-no-files-found: error openjdk-mutator-perf: + if: false # Disable this to test openjdk regression, only. runs-on: [self-hosted, Linux, freq-scaling-off] timeout-minutes: 1440 steps: