ci: microbenchmarks + GH actions testing [wip] [dnm]

.github/actions/microbenchmark-build/action.yml

@@ -0,0 +1,65 @@
+name: "Microbenchmark Build"
+description: "Builds microbenchmarks for a given package and revision."
+inputs:
+  ref:
+    description: "head or base"
+    required: true
+  pkg:
+    description: "test package to build"
+    required: true
+outputs:
+  merge_base:
+    description: "merge base"
+    value: ${{ steps.determine-merge-base.outputs.merge_base }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Copy build script
+      run: cp build/github/microbenchmarks-build.sh ${RUNNER_TEMP}/
+      shell: bash
+
+    - run: ./build/github/get-engflow-keys.sh
+      shell: bash
+
+    - name: Checkout code with limited depth
+      if: inputs.ref == 'base'
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 15
+
+    - name: Determine merge base
+      id: determine-merge-base
+      if: inputs.ref == 'base'
+      run: |
+        set -e
+        MERGE_BASE=$(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})
+        if [ -z "$MERGE_BASE" ]; then
+          echo "Error: Merge base could not be found within the last 15 commits." >&2
+          exit 1
+        fi
+        echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
+      shell: bash
+
+    - name: Checkout build commit
+      if: inputs.ref == 'base'
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ steps.determine-merge-base.outputs.merge_base }}
+
+    - name: Build tests
+      run: ${RUNNER_TEMP}/microbenchmarks-build.sh
+      shell: bash
+      env:
+        TEST_PKG: ${{ inputs.pkg }}
+
+    - name: Checkout Head # required for post job cleanup (if still on the base ref)
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      if: always() && inputs.ref == 'base'
+
+    - name: Clean up
+      run: ./build/github/cleanup-engflow-keys.sh
+      shell: bash
+      if: always()

.github/actions/microbenchmark-run/action.yml

@@ -0,0 +1,41 @@
+name: "Microbenchmark Run"
+description: "Runs microbenchmarks."
+inputs:
+  base:
+    description: "Base SHA"
+    required: true
+  pkg:
+    description: "Test packages required for the run"
+    required: true
+  group:
+    description: "Runner group"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout Head
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+    - run: ./build/github/get-engflow-keys.sh
+      shell: bash
+
+    - name: Unique Build ID
+      run: echo "BUILD_ID=${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_ENV
+      shell: bash
+
+    - name: Run benchmarks
+      run: build/github/microbenchmarks-run.sh
+      shell: bash
+      env:
+        BASE_SHA: ${{ inputs.base }}
+        HEAD_SHA: ${{ github.event.pull_request.head.sha || github.ref }}
+        TEST_PACKAGES: ${{ inputs.pkg }}
+        GROUP: ${{ inputs.group }}
+
+    - name: Clean up
+      run: ./build/github/cleanup-engflow-keys.sh
+      shell: bash
+      if: always()

.github/workflows/code-cover-gen.yml

@@ -6,6 +6,7 @@ on:
 
 jobs:
   code-cover-gen:
+    if: false  # This will always skip this job
     runs-on: ubuntu-latest
     env:
       PR: ${{ github.event.pull_request.number }}

.github/workflows/github-actions-essential-ci.yml

@@ -53,6 +53,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   acceptance:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_big_runner_group]
     timeout-minutes: 30
     steps:
@@ -74,6 +75,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   check_generated_code:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:
@@ -89,6 +91,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   docker_image_amd64:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:
@@ -110,6 +113,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   examples_orms:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_big_runner_group]
     timeout-minutes: 40
     steps:
@@ -131,6 +135,7 @@ jobs:
         run: ./cockroach/build/github/cleanup-engflow-keys.sh
         if: always()
   lint:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_big_runner_group]
     timeout-minutes: 60
     steps:
@@ -155,6 +160,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   local_roachtest:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_big_runner_group]
     timeout-minutes: 60
     steps:
@@ -176,6 +182,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   local_roachtest_fips:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group_fips]
     timeout-minutes: 60
     steps:
@@ -197,6 +204,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   linux_amd64_build:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:
@@ -218,6 +226,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   linux_amd64_fips_build:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:
@@ -239,6 +248,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   linux_arm64_build:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:
@@ -260,6 +270,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   macos_amd64_build:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:
@@ -281,6 +292,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   macos_arm64_build:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:
@@ -302,6 +314,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   unit_tests:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 60
     steps:
@@ -325,6 +338,7 @@ jobs:
         run: ./build/github/cleanup-engflow-keys.sh
         if: always()
   windows_build:
+    if: false  # This will always skip this job
     runs-on: [self-hosted, basic_runner_group]
     timeout-minutes: 30
     steps:

.github/workflows/microbenchmarks-ci.yaml

@@ -0,0 +1,102 @@
+name: Microbenchmarks CI
+on:
+  pull_request:
+    types: [ opened, reopened, synchronize ]
+    branches: [ master ]
+
+env:
+  BUCKET: "cockroach-microbench-ci"
+  HEAD: ${{ github.event.pull_request.head.sha || github.ref }}
+
+jobs:
+  base:
+    name: Build merge base binaries
+    runs-on: [self-hosted, basic_runner_group]
+    timeout-minutes: 30
+    outputs:
+      merge_base: ${{ steps.build.outputs.merge_base }}
+    steps:
+      - name: Checkout Head
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD }}
+      - name: Run Microbenchmark Build
+        id: build
+        uses: ./.github/actions/microbenchmark-build
+        with:
+          ref: base
+          pkg: "pkg/sql/tests"
+
+  head:
+    name: Build head binaries
+    runs-on: [self-hosted, basic_runner_group]
+    timeout-minutes: 30
+    outputs:
+      merge_base: ${{ steps.build.outputs.merge_base }}
+    steps:
+      - name: Checkout Head
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD }}
+      - name: Run Microbenchmark Build
+        id: build
+        uses: ./.github/actions/microbenchmark-build
+        with:
+          ref: head
+          pkg: "pkg/sql/tests"
+
+  run-group-1:
+    runs-on: [self-hosted, basic_runner_group]
+    timeout-minutes: 30
+    needs: [base, head]
+    steps:
+      - name: Checkout Head
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD }}
+      - name: Run Microbenchmarks
+        uses: ./.github/actions/microbenchmark-run
+        with:
+          base:  ${{ needs.base.outputs.merge_base }}
+          pkg: "pkg/sql/tests"
+          group: 1
+
+  run-group-2:
+    runs-on: [self-hosted, basic_runner_group]
+    timeout-minutes: 30
+    needs: [base, head]
+    steps:
+      - name: Checkout Head
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.HEAD }}
+      - name: Run Microbenchmarks
+        uses: ./.github/actions/microbenchmark-run
+        with:
+          base:  ${{ needs.base.outputs.merge_base }}
+          pkg: "pkg/sql/tests"
+          group: 2
+  compare:
+    runs-on: [self-hosted, basic_runner_group]
+    timeout-minutes: 30
+    needs: [base, run-group-1, run-group-2]
+    steps:
+      - name: Checkout Head
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+      - run: ./build/github/get-engflow-keys.sh
+
+      - name: Unique Build ID
+        run: echo "BUILD_ID=${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_ENV
+
+      - name: Compare benchmarks
+        run: build/github/microbenchmarks-compare.sh
+        env:
+          BASE_SHA: ${{ needs.base.outputs.merge_base }}
+          HEAD_SHA: ${{ env.HEAD }}
+
+      - name: Clean up
+        run: ./build/github/cleanup-engflow-keys.sh
+        if: always()

build/github/microbenchmarks-build.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -euxo pipefail
+
+pkg_last=$(basename "${TEST_PKG}")
+pkg_bin=$(echo "${TEST_PKG}" | tr '/' '_')
+
+build_sha=$(git rev-parse HEAD)
+storage_bucket="$BUCKET"
+output_url="gs://${storage_bucket}/builds/${build_sha}/bin/${pkg_bin}"
+
+# Disable parallel uploads, as it requires specific permissions
+gcloud config set storage/parallel_composite_upload_enabled False
+
+if gcloud storage ls "${output_url}" &>/dev/null; then
+  echo "Build for $build_sha already exists. Skipping..."
+  exit 0
+fi
+
+bazel build "//${TEST_PKG}:tests_test" \
+  --jobs 100 \
+  --crdb_test_off \
+  --bes_keywords integration-test-artifact-build \
+  --config=crosslinux \
+  --remote_download_minimal \
+  $(./build/github/engflow-args.sh)
+
+# Copy the binary to a GCS cache
+bazel_bin=$(bazel info bazel-bin --config=crosslinux)
+gcloud storage cp "${bazel_bin}/pkg/sql/tests/${pkg_last}_test_/${pkg_last}_test" "${output_url}"