Testing the history canary feature (don't merge)

.github/workflows/perf-regression-ci.yml

@@ -8,13 +8,13 @@ on:
   # READ BEFORE ENABLING THE TRIGGER BELOW
   # This trigger is only used when testing the scripts in a branch, and should be commented out in other cases.
   # If this trigger is used, please change the following env: RESULT_REPO_BRANCH -> 'test' (MUST), DEPLOY -> 'false' (optional)
-  # pull_request:
-  #   branches:
-  #     - master
+  pull_request:
+    branches:
+      - master
 
 env:
   # The branch to save run data and plot graph from. Use 'self-hosted' for master, use 'test' or anything else for testing in a branch.
-  RESULT_REPO_BRANCH: 'self-hosted'
+  RESULT_REPO_BRANCH: 'test'
   # Whether we deploy the generated page. Set to true for master.
   DEPLOY: true
   # Directories in ci-perf-kit that will be uploaded as artifacts. The dirs can be found in ci-perf-kit/scripts/common.sh
@@ -24,6 +24,7 @@ env:
 jobs:
   # JikesRVM
   jikesrvm-perf-regression:
+    if: false # Disable this to test openjdk regression, only.
     runs-on: [self-hosted, Linux, freq-scaling-off]
     timeout-minutes: 1440
     steps:
@@ -97,45 +98,74 @@ jobs:
   openjdk-perf-regression:
     runs-on: [self-hosted, Linux, freq-scaling-off]
     timeout-minutes: 1440
+    env:
+      # This version will be used as the canary version, and will be used to checkout both
+      # `mmtk-core` and `mmtk-openjdk`.  We choose one release version for this purpose. We may
+      # change to another release version if necessary, or introduce a mechanism to dynamically
+      # choose the canary in the future.
+      CANARY_VERSION: "v0.28.0"
     steps:
-      - name: Checkout MMTk Core
+      # checkout latest versions
+      - name: Checkout MMTk Core (latest)
         uses: actions/checkout@v4
         with:
-          path: mmtk-core
-      - name: Checkout OpenJDK Binding
+          path: latest/mmtk-core
+      - name: Checkout OpenJDK Binding (latest)
         uses: actions/checkout@v4
         with:
           repository: mmtk/mmtk-openjdk
-          path: mmtk-openjdk
-      - name: Checkout OpenJDK
-        working-directory: mmtk-openjdk
+          path: latest/mmtk-openjdk
+      - name: Modify latest version toolchain
+        working-directory: latest/mmtk-openjdk
+        run: |
+          cat mmtk/rust-toolchain
+          echo "stable" > mmtk/rust-toolchain
+          cat mmtk/rust-toolchain
+      - name: Checkout OpenJDK (latest)
+        working-directory: latest/mmtk-openjdk
         run: |
           ./.github/scripts/ci-checkout.sh
       # checkout perf-kit
       - name: Checkout Perf Kit
         uses: actions/checkout@v4
         with:
-          repository: mmtk/ci-perf-kit
-          ref: "0.8.0"
+          repository: wks/ci-perf-kit
+          ref: "feature/history-canary"
           path: ci-perf-kit
-          token: ${{ secrets.CI_ACCESS_TOKEN }}
           submodules: true
-      # setup
-      - name: Overwrite MMTk core in openjdk binding
-        run: cp -r mmtk-core mmtk-openjdk/repos/
-      - name: Setup Rust Toolchain
-        run: echo "RUSTUP_TOOLCHAIN=`cat mmtk-core/rust-toolchain`" >> $GITHUB_ENV
-      # cleanup previosu build
-      - name: Cleanup previous build
+      # checkout canary versions.
+      - name: Checkout MMTk Core (canary)
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CANARY_VERSION }}
+          path: canary/mmtk-core
+      - name: Checkout OpenJDK Binding (canary)
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CANARY_VERSION }}
+          repository: mmtk/mmtk-openjdk
+          path: canary/mmtk-openjdk
+      - name: Checkout OpenJDK (canary)
+        working-directory: canary/mmtk-openjdk
         run: |
-          rm -rf mmtk-openjdk/repos/openjdk/scratch
-          rm -rf mmtk-openjdk/repos/openjdk/build
-      - name: Setup
+          ./.github/scripts/ci-checkout.sh
+      # setup
+      - name: Setup directory structures
         run: |
-          ./ci-perf-kit/scripts/history-run-setup.sh
-          sed -i 's/^mmtk[[:space:]]=/#ci:mmtk=/g' mmtk-openjdk/mmtk/Cargo.toml
-          sed -i 's/^#[[:space:]]mmtk/mmtk/g' mmtk-openjdk/mmtk/Cargo.toml
-      - id: branch
+          for BASE_DIR in ./latest ./canary; do
+            pushd $BASE_DIR
+              # replace dependency
+              # Note that ci-replace-mmtk-dep.sh will apply `realpath()` to the `--mmtk-core-path` option.
+              # so we specify the relative path from the PWD to the mmtk-core repo.
+              ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml \
+                --mmtk-core-path mmtk-core
+              # cleanup previous build
+              rm -rf mmtk-openjdk/repos/openjdk/scratch
+              rm -rf mmtk-openjdk/repos/openjdk/build
+            popd
+          done
+      - name: Setup branch name
+        id: branch
         # we cannot use env vars in action input (the deploy step). So put the env var to this step's outputs.
         run: echo "branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT
       # run
@@ -145,7 +175,11 @@ jobs:
           export RESULT_REPO_BRANCH=${{ env.RESULT_REPO_BRANCH }}
           export RESULT_REPO_ACCESS_TOKEN=${{ secrets.CI_ACCESS_TOKEN }}
           export FROM_DATE=2020-07-10
-          ./ci-perf-kit/scripts/openjdk-history-run.sh ./mmtk-openjdk ./reports/${{ steps.branch.outputs.branch_name }}
+          export OPENJDK_HISTORY_RUN_TEST_FAST=1
+          ./ci-perf-kit/scripts/openjdk-history-run.sh \
+            ./latest/mmtk-openjdk \
+            ./canary/mmtk-openjdk \
+            ./reports/${{ steps.branch.outputs.branch_name }}
       # deploy
       - name: Deploy to Github Page
         if: ${{ env.DEPLOY == 'true' }}
@@ -170,6 +204,7 @@ jobs:
           if-no-files-found: error
 
   openjdk-mutator-perf:
+    if: false # Disable this to test openjdk regression, only.
     runs-on: [self-hosted, Linux, freq-scaling-off]
     timeout-minutes: 1440
     steps: