diff --git a/.github/actions/microbenchmark-build/action.yml b/.github/actions/microbenchmark-build/action.yml
new file mode 100644
index 000000000000..783625658cfa
--- /dev/null
+++ b/.github/actions/microbenchmark-build/action.yml
@@ -0,0 +1,65 @@
+name: "Microbenchmark Build"
+description: "Builds microbenchmarks for a given package and revision."
+inputs:
+ ref:
+ description: "head or base"
+ required: true
+ pkg:
+ description: "test package to build"
+ required: true
+outputs:
+ merge_base:
+ description: "merge base"
+ value: ${{ steps.determine-merge-base.outputs.merge_base }}
+
+runs:
+ using: "composite"
+ steps:
+ - name: Copy build script
+ run: cp build/github/microbenchmarks/build.sh ${RUNNER_TEMP}/
+ shell: bash
+
+ - run: ./build/github/get-engflow-keys.sh
+ shell: bash
+
+ - name: Checkout code with limited depth
+ if: inputs.ref == 'base'
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 15
+
+ - name: Determine merge base
+ id: determine-merge-base
+ if: inputs.ref == 'base'
+ run: |
+ set -e
+ MERGE_BASE=$(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})
+ if [ -z "$MERGE_BASE" ]; then
+ echo "Error: Merge base could not be found within the last 15 commits." >&2
+ exit 1
+ fi
+ echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
+ shell: bash
+
+ - name: Checkout build commit
+ if: inputs.ref == 'base'
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ steps.determine-merge-base.outputs.merge_base }}
+
+ - name: Build tests
+ run: ${RUNNER_TEMP}/build.sh
+ shell: bash
+ env:
+ TEST_PKG: ${{ inputs.pkg }}
+
+ - name: Checkout Head # required for post job cleanup (if still on the base ref)
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.ref }}
+ if: always() && inputs.ref == 'base'
+
+ - name: Clean up
+ run: ./build/github/cleanup-engflow-keys.sh
+ shell: bash
+ if: always()
diff --git a/.github/actions/microbenchmark-run/action.yml b/.github/actions/microbenchmark-run/action.yml
new file mode 100644
index 000000000000..1ace5c37f32c
--- /dev/null
+++ b/.github/actions/microbenchmark-run/action.yml
@@ -0,0 +1,41 @@
+name: "Microbenchmark Run"
+description: "Runs microbenchmarks."
+inputs:
+ base:
+ description: "Base SHA"
+ required: true
+ pkg:
+ description: "Test packages required for the run"
+ required: true
+ group:
+ description: "Runner group"
+ required: true
+
+runs:
+ using: "composite"
+ steps:
+ - name: Checkout Head
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+ - run: ./build/github/get-engflow-keys.sh
+ shell: bash
+
+ - name: Unique Build ID
+ run: echo "BUILD_ID=${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_ENV
+ shell: bash
+
+ - name: Run benchmarks
+ run: build/github/microbenchmarks/run.sh
+ shell: bash
+ env:
+ BASE_SHA: ${{ inputs.base }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha || github.ref }}
+ TEST_PACKAGES: ${{ inputs.pkg }}
+ GROUP: ${{ inputs.group }}
+
+ - name: Clean up
+ run: ./build/github/cleanup-engflow-keys.sh
+ shell: bash
+ if: always()
diff --git a/.github/workflows/code-cover-gen.yml b/.github/workflows/code-cover-gen.yml
index f078ca56f985..68767b5f815f 100644
--- a/.github/workflows/code-cover-gen.yml
+++ b/.github/workflows/code-cover-gen.yml
@@ -6,6 +6,7 @@ on:
jobs:
code-cover-gen:
+ if: false # This will always skip this job
runs-on: ubuntu-latest
env:
PR: ${{ github.event.pull_request.number }}
diff --git a/.github/workflows/github-actions-essential-ci.yml b/.github/workflows/github-actions-essential-ci.yml
index ab18508a8c9b..705754a10e66 100644
--- a/.github/workflows/github-actions-essential-ci.yml
+++ b/.github/workflows/github-actions-essential-ci.yml
@@ -53,6 +53,7 @@ concurrency:
cancel-in-progress: true
jobs:
acceptance:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_big_runner_group]
timeout-minutes: 30
steps:
@@ -74,6 +75,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
check_generated_code:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
@@ -89,6 +91,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
docker_image_amd64:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
@@ -110,6 +113,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
examples_orms:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_big_runner_group]
timeout-minutes: 40
steps:
@@ -131,6 +135,7 @@ jobs:
run: ./cockroach/build/github/cleanup-engflow-keys.sh
if: always()
lint:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_big_runner_group]
timeout-minutes: 60
steps:
@@ -155,6 +160,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
local_roachtest:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_big_runner_group]
timeout-minutes: 60
steps:
@@ -176,6 +182,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
local_roachtest_fips:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group_fips]
timeout-minutes: 60
steps:
@@ -197,6 +204,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
linux_amd64_build:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
@@ -218,6 +226,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
linux_amd64_fips_build:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
@@ -239,6 +248,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
linux_arm64_build:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
@@ -260,6 +270,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
macos_amd64_build:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
@@ -281,6 +292,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
macos_arm64_build:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
@@ -302,6 +314,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
unit_tests:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 60
steps:
@@ -325,6 +338,7 @@ jobs:
run: ./build/github/cleanup-engflow-keys.sh
if: always()
windows_build:
+ if: false # This will always skip this job
runs-on: [self-hosted, basic_runner_group]
timeout-minutes: 30
steps:
diff --git a/.github/workflows/microbenchmarks-ci.yaml b/.github/workflows/microbenchmarks-ci.yaml
new file mode 100644
index 000000000000..d78f76f117e7
--- /dev/null
+++ b/.github/workflows/microbenchmarks-ci.yaml
@@ -0,0 +1,128 @@
+name: Microbenchmarks CI
+on:
+ pull_request:
+ types: [ opened, reopened, synchronize ]
+ branches: [ master ]
+
+env:
+ HEAD: ${{ github.event.pull_request.head.sha || github.ref }}
+ BUCKET: "cockroach-microbench-ci"
+ PACKAGE: "pkg/sql/tests"
+
+jobs:
+ base:
+ name: build base
+ runs-on: [self-hosted, basic_runner_group]
+ timeout-minutes: 30
+ outputs:
+ merge_base: ${{ steps.build.outputs.merge_base }}
+ steps:
+ - name: Checkout Head
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ env.HEAD }}
+ - name: Build
+ id: build
+ uses: ./.github/actions/microbenchmark-build
+ with:
+ ref: base
+ pkg: ${{ env.PACKAGE }}
+
+ head:
+ name: build head
+ runs-on: [self-hosted, basic_runner_group]
+ timeout-minutes: 30
+ outputs:
+ merge_base: ${{ steps.build.outputs.merge_base }}
+ steps:
+ - name: Checkout Head
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ env.HEAD }}
+ - name: Build
+ id: build
+ uses: ./.github/actions/microbenchmark-build
+ with:
+ ref: head
+ pkg: ${{ env.PACKAGE }}
+
+ run-group-1:
+ runs-on: [self-hosted, basic_big_runner_group]
+ timeout-minutes: 30
+ needs: [base, head]
+ steps:
+ - name: Checkout Head
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ env.HEAD }}
+ - name: Run
+ uses: ./.github/actions/microbenchmark-run
+ with:
+ base: ${{ needs.base.outputs.merge_base }}
+ pkg: ${{ env.PACKAGE }}
+ group: 1
+
+ run-group-2:
+ runs-on: [self-hosted, basic_big_runner_group]
+ timeout-minutes: 30
+ needs: [base, head]
+ steps:
+ - name: Checkout Head
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ env.HEAD }}
+ - name: Run
+ uses: ./.github/actions/microbenchmark-run
+ with:
+ base: ${{ needs.base.outputs.merge_base }}
+ pkg: ${{ env.PACKAGE }}
+ group: 2
+
+ compare:
+ runs-on: [self-hosted, basic_runner_group]
+ timeout-minutes: 30
+ needs: [base, run-group-1, run-group-2]
+ steps:
+ - name: Checkout Head
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+ - run: ./build/github/get-engflow-keys.sh
+
+ - name: Unique Build ID
+ run: echo "BUILD_ID=${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_ENV
+
+ - name: Compare and Post
+ run: build/github/microbenchmarks/compare.sh
+ env:
+ BASE_SHA: ${{ needs.base.outputs.merge_base }}
+ HEAD_SHA: ${{ env.HEAD }}
+
+ - name: Clean up
+ run: ./build/github/cleanup-engflow-keys.sh
+ if: always()
+
+ post:
+ runs-on: [ self-hosted, basic_runner_group ]
+ timeout-minutes: 30
+ needs: [ compare ]
+ steps:
+ - name: Checkout Head
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+ - run: ./build/github/get-engflow-keys.sh
+
+ - name: Unique Build ID
+ run: echo "BUILD_ID=${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_ENV
+
+ - name: Compare and Post
+ run: build/github/microbenchmarks/post.sh
+ env:
+ HEAD_SHA: ${{ env.HEAD }}
+
+ - name: Clean up
+ run: ./build/github/cleanup-engflow-keys.sh
+ if: always()
diff --git a/build/github/microbenchmarks/build.sh b/build/github/microbenchmarks/build.sh
new file mode 100755
index 000000000000..fd6564e52257
--- /dev/null
+++ b/build/github/microbenchmarks/build.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -euxo pipefail
+
+pkg_last=$(basename "${TEST_PKG}")
+pkg_bin=$(echo "${TEST_PKG}" | tr '/' '_')
+
+build_sha=$(git rev-parse HEAD)
+storage_bucket="$BUCKET"
+output_url="gs://${storage_bucket}/builds/${build_sha}/bin/${pkg_bin}"
+
+# Disable parallel uploads, as it requires specific permissions
+gcloud config set storage/parallel_composite_upload_enabled False
+
+if gcloud storage ls "${output_url}" &>/dev/null; then
+ echo "Build for $build_sha already exists. Skipping..."
+ exit 0
+fi
+
+bazel build "//${TEST_PKG}:tests_test" \
+ --jobs 100 \
+ --crdb_test_off \
+ --bes_keywords integration-test-artifact-build \
+ --config=crosslinux \
+ --remote_download_minimal \
+ $(./build/github/engflow-args.sh)
+
+# Copy the binary to a GCS cache
+bazel_bin=$(bazel info bazel-bin --config=crosslinux)
+gcloud storage cp "${bazel_bin}/pkg/sql/tests/${pkg_last}_test_/${pkg_last}_test" "${output_url}"
diff --git a/build/github/microbenchmarks/compare.sh b/build/github/microbenchmarks/compare.sh
new file mode 100755
index 000000000000..84d9d32c61d4
--- /dev/null
+++ b/build/github/microbenchmarks/compare.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -euxo pipefail
+
+working_dir=$(mktemp -d)
+output_dir=$(mktemp -d)
+storage_bucket="$BUCKET"
+shas=("$BASE_SHA" "$HEAD_SHA")
+
+# Disable parallel uploads, as it requires specific permissions
+gcloud config set storage/parallel_composite_upload_enabled False
+
+# Retrieve outputs from the runs
+for sha in "${shas[@]}"; do
+ mkdir -p "${working_dir}/${sha}/artifacts"
+ gcloud storage cp -r "gs://${storage_bucket}/artifacts/${sha}/${BUILD_ID}/*" "${working_dir}/${sha}/artifacts/"
+done
+
+# Build microbenchmark CI utility
+# This will build the repository's base version
+bazel build --config=crosslinux $(./build/github/engflow-args.sh) \
+ --jobs 100 \
+ --bes_keywords integration-test-artifact-build \
+ //pkg/cmd/microbench-ci \
+
+bazel_bin=$(bazel info bazel-bin --config=crosslinux)
+microbench_ci_bin=$bazel_bin/pkg/cmd/microbench-ci/microbench-ci_
+
+# Compare the microbenchmarks
+$microbench_ci_bin/microbench-ci compare \
+ --working-dir="$working_dir" \
+ --summary="$output_dir/summary.json" \
+ --github-summary="$output_dir/summary.md" \
+ --build-id="$BUILD_ID" \
+ --old="$BASE_SHA" \
+ --new="$HEAD_SHA"
+
+cat "$output_dir/summary.md" > "$GITHUB_STEP_SUMMARY"
+
+# Copy comparison results to GCS
+gcloud storage cp -r "$output_dir/*" "gs://${storage_bucket}/results/${HEAD_SHA}/${BUILD_ID}"
diff --git a/build/github/microbenchmarks/post.sh b/build/github/microbenchmarks/post.sh
new file mode 100755
index 000000000000..f357117b9ceb
--- /dev/null
+++ b/build/github/microbenchmarks/post.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -euxo pipefail
+
+storage_bucket="$BUCKET"
+
+# Build microbenchmark CI utility
+# This will build the repository's base version
+bazel build --config=crosslinux $(./build/github/engflow-args.sh) \
+ --jobs 100 \
+ --bes_keywords integration-test-artifact-build \
+ //pkg/cmd/microbench-ci \
+
+bazel_bin=$(bazel info bazel-bin --config=crosslinux)
+microbench_ci_bin=$bazel_bin/pkg/cmd/microbench-ci/microbench-ci_
+
+# Grab the summary from the previous step
+working_dir=$(mktemp -d)
+gcloud storage cp "gs://${storage_bucket}/results/${HEAD_SHA}/${BUILD_ID}/summary.md" "$working_dir/summary.md"
+
+cat "$working_dir/summary.md"
+
+# Compare the microbenchmarks
+#$microbench_ci_bin/microbench-ci post
+
+
diff --git a/build/github/microbenchmarks/run.sh b/build/github/microbenchmarks/run.sh
new file mode 100755
index 000000000000..683d7885c772
--- /dev/null
+++ b/build/github/microbenchmarks/run.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -euxo pipefail
+
+working_dir=$(mktemp -d)
+temp_dir=$(mktemp -d)
+storage_bucket="$BUCKET"
+shas=("$BASE_SHA" "$HEAD_SHA")
+
+# Disable parallel uploads, as it requires specific permissions
+gcloud config set storage/parallel_composite_upload_enabled False
+
+# Retrieve required binaries from the base and head builds
+for pkg in "${TEST_PACKAGES[@]}"; do
+ for sha in "${shas[@]}"; do
+ pkg_bin=$(echo "${pkg}" | tr '/' '_')
+ url="gs://${storage_bucket}/builds/${sha}/bin/${pkg_bin}"
+ dest="$working_dir/$sha/bin/"
+ mkdir -p "$dest"
+ gcloud storage cp "${url}" "$dest/${pkg_bin}"
+ chmod +x "$dest/${pkg_bin}"
+ done
+done
+
+# Build microbenchmark CI utility (HEAD version)
+bazel build --config=crosslinux $(./build/github/engflow-args.sh) \
+ --jobs 100 \
+ --bes_keywords integration-test-artifact-build \
+ //pkg/cmd/microbench-ci \
+
+bazel_bin=$(bazel info bazel-bin --config=crosslinux)
+microbench_ci_bin=$bazel_bin/pkg/cmd/microbench-ci/microbench-ci_
+
+# Run the microbenchmarks
+$microbench_ci_bin/microbench-ci run --group="$GROUP" --working-dir="$working_dir" --old="$BASE_SHA" --new="$HEAD_SHA"
+
+# Copy benchmark results to GCS
+curl -H "Metadata-Flavor: Google" \
+ http://metadata.google.internal/computeMetadata/v1/instance/machine-type | awk -F'/' '{print $NF}' > "$temp_dir/machine_type.txt"
+for sha in "${shas[@]}"; do
+ gcloud storage cp -n "$temp_dir/machine_type.txt" "gs://${storage_bucket}/artifacts/${sha}/${BUILD_ID}/machine_type.txt"
+ gcloud storage cp -r "${working_dir}/${sha}/artifacts/*" "gs://${storage_bucket}/artifacts/${sha}/${BUILD_ID}/"
+done
diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
index 3169ffcc1f59..59c43ac45e72 100644
--- a/pkg/BUILD.bazel
+++ b/pkg/BUILD.bazel
@@ -1151,6 +1151,8 @@ GO_TARGETS = [
"//pkg/cmd/label-merged-pr:label-merged-pr",
"//pkg/cmd/label-merged-pr:label-merged-pr_lib",
"//pkg/cmd/label-merged-pr:label-merged-pr_test",
+ "//pkg/cmd/microbench-ci:microbench-ci",
+ "//pkg/cmd/microbench-ci:microbench-ci_lib",
"//pkg/cmd/mirror/go:go_lib",
"//pkg/cmd/mirror/go:go_test",
"//pkg/cmd/mirror/go:mirror",
@@ -1182,6 +1184,7 @@ GO_TARGETS = [
"//pkg/cmd/roachprod-microbench/cluster:cluster_test",
"//pkg/cmd/roachprod-microbench/google:google",
"//pkg/cmd/roachprod-microbench/model:model",
+ "//pkg/cmd/roachprod-microbench/parser:parser",
"//pkg/cmd/roachprod-microbench/util:util",
"//pkg/cmd/roachprod-microbench/util:util_test",
"//pkg/cmd/roachprod-microbench:roachprod-microbench",
@@ -2321,6 +2324,7 @@ GO_TARGETS = [
"//pkg/storage:storage",
"//pkg/storage:storage_test",
"//pkg/testutils/bazelcodecover:bazelcodecover",
+ "//pkg/testutils/benchmark:benchmark",
"//pkg/testutils/colcontainerutils:colcontainerutils",
"//pkg/testutils/datapathutils:datapathutils",
"//pkg/testutils/diagutils:diagutils",
diff --git a/pkg/cmd/dev/build.go b/pkg/cmd/dev/build.go
index 1ee15fbd44a4..edd97045a4a4 100644
--- a/pkg/cmd/dev/build.go
+++ b/pkg/cmd/dev/build.go
@@ -92,6 +92,7 @@ var buildTargetMapping = map[string]string{
"geos": geosTarget,
"langgen": "//pkg/sql/opt/optgen/cmd/langgen:langgen",
"libgeos": geosTarget,
+ "microbench-ci": "//pkg/cmd/microbench-ci:microbench-ci",
"optgen": "//pkg/sql/opt/optgen/cmd/optgen:optgen",
"optfmt": "//pkg/sql/opt/optgen/cmd/optfmt:optfmt",
"reduce": "//pkg/cmd/reduce:reduce",
diff --git a/pkg/cmd/microbench-ci/BUILD.bazel b/pkg/cmd/microbench-ci/BUILD.bazel
new file mode 100644
index 000000000000..f56aa8ee422d
--- /dev/null
+++ b/pkg/cmd/microbench-ci/BUILD.bazel
@@ -0,0 +1,35 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "microbench-ci_lib",
+ srcs = [
+ "benchmark.go",
+ "compare.go",
+ "github.go",
+ "main.go",
+ "report.go",
+ "run.go",
+ ],
+ embedsrcs = ["template/github_summary.md"],
+ importpath = "github.com/cockroachdb/cockroach/pkg/cmd/microbench-ci",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//pkg/cmd/roachprod-microbench/model",
+ "//pkg/cmd/roachprod-microbench/parser",
+ "@com_github_cockroachdb_errors//:errors",
+ "@com_github_google_go_github_v61//github",
+ "@com_github_spf13_cobra//:cobra",
+ "@in_gopkg_yaml_v2//:yaml_v2",
+ "@org_golang_x_exp//maps",
+ "@org_golang_x_oauth2//:oauth2",
+ "@org_golang_x_perf//benchfmt",
+ "@org_golang_x_perf//benchmath",
+ "@org_golang_x_perf//benchunit",
+ ],
+)
+
+go_binary(
+ name = "microbench-ci",
+ embed = [":microbench-ci_lib"],
+ visibility = ["//visibility:public"],
+)
diff --git a/pkg/cmd/microbench-ci/benchmark.go b/pkg/cmd/microbench-ci/benchmark.go
new file mode 100644
index 000000000000..e6920b85f0b2
--- /dev/null
+++ b/pkg/cmd/microbench-ci/benchmark.go
@@ -0,0 +1,134 @@
+package main
+
+import (
+ "fmt"
+ "hash/fnv"
+ "os"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/cockroachdb/errors"
+ yaml "gopkg.in/yaml.v2"
+)
+
+type (
+ Benchmark struct {
+ DisplayName string `yaml:"display_name"`
+ Package string `yaml:"package"`
+ Labels []string `yaml:"labels"`
+ Name string
+ RunnerGroup int `yaml:"runner_group"`
+ MeasureCount int `yaml:"measure_count"`
+ Measure Sample `yaml:"measure"`
+ CPUProfile Sample `yaml:"cpu_profile"`
+ MemProfile Sample `yaml:"mem_profile"`
+ MutexProfile Sample `yaml:"mutex_profile"`
+ Thresholds map[string]float64 `yaml:"thresholds"`
+ }
+ BenchmarkIterationType uint8
+ Benchmarks []Benchmark
+
+ Sample struct {
+ Iterations int `yaml:"iterations"`
+ }
+)
+
+const (
+ Measure BenchmarkIterationType = iota
+ ProfileCPU
+ ProfileMemory
+ ProfileMutex
+)
+
+var sanitizeRe = regexp.MustCompile(`\W+`)
+
+func (b *Benchmark) sanitizedPackageName() string {
+ return strings.ReplaceAll(b.Package, "/", "_")
+}
+
+func (b *Benchmark) sanitizedName() string {
+ return sanitizeRe.ReplaceAllString(strings.TrimPrefix(b.Name, "Benchmark"), "_")
+}
+
+func (b *Benchmark) binaryName() string {
+ return b.sanitizedPackageName()
+}
+
+func (b *Benchmark) rawLog() string {
+ return fmt.Sprintf("raw_%s.log", b.sanitizedName())
+}
+
+func (b *Benchmark) cleanLog() string {
+ return fmt.Sprintf("cleaned_%s.log", b.sanitizedName())
+}
+
+func (b *Benchmark) cpuProfile() string {
+ return fmt.Sprintf("cpu_%s.out", b.sanitizedName())
+}
+
+func (b *Benchmark) memProfile() string {
+ return fmt.Sprintf("mem_%s.out", b.sanitizedName())
+}
+
+func (b *Benchmark) mutexProfile() string {
+ return fmt.Sprintf("mutex_%s.out", b.sanitizedName())
+}
+
+func (b *Benchmark) packageHash() string {
+ h := fnv.New32a()
+ _, _ = h.Write([]byte("./" + b.Package))
+ u := h.Sum32()
+ return strconv.Itoa(int(u))
+}
+
+// args returns the arguments to pass to `go test` for the given iteration type.
+func (b *Benchmark) args(outputDir string, iterType BenchmarkIterationType) []string {
+ args := []string{
+ "-test.run", "^$",
+ "-test.bench", b.Name,
+ "-test.benchmem",
+ "-test.count", "1",
+ "-test.outputdir", outputDir,
+ "-test.benchtime", fmt.Sprintf("%dx", b.Measure.Iterations),
+ }
+
+ var sample Sample
+ switch iterType {
+ case Measure:
+ sample = b.Measure
+ case ProfileCPU:
+ sample = b.CPUProfile
+ args = append(args, "-test.cpuprofile", b.cpuProfile())
+ case ProfileMemory:
+ sample = b.MemProfile
+ args = append(args, "-test.memprofile", b.memProfile())
+ case ProfileMutex:
+ sample = b.MemProfile
+ args = append(args, "-test.mutexprofile", b.mutexProfile())
+ default:
+ panic("unknown iteration type")
+ }
+ // If the specified sample has no iterations, use the default iterations.
+ if sample.Iterations == 0 {
+ sample.Iterations = b.Measure.Iterations
+ }
+ args = append(args, "-test.benchtime", fmt.Sprintf("%dx", sample.Iterations))
+ return args
+}
+
+// loadBenchmarkConfig loads the benchmark configurations from the given file.
+func loadBenchmarkConfig(path string) ([]Benchmark, error) {
+ var c = struct {
+ Benchmarks []Benchmark `yaml:"benchmarks"`
+ }{}
+ fileContent, err := os.ReadFile(path)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to read config file %s", path)
+ }
+ err = yaml.UnmarshalStrict(fileContent, &c)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to unmarshal config file %s", path)
+ }
+ return c.Benchmarks, nil
+}
diff --git a/pkg/cmd/microbench-ci/compare.go b/pkg/cmd/microbench-ci/compare.go
new file mode 100644
index 000000000000..786dd197b178
--- /dev/null
+++ b/pkg/cmd/microbench-ci/compare.go
@@ -0,0 +1,113 @@
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "math"
+ "os"
+ "path"
+
+ "github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench/model"
+ "github.com/cockroachdb/errors"
+ "golang.org/x/exp/maps"
+ "golang.org/x/perf/benchfmt"
+)
+
+type (
+ CompareResult struct {
+ Benchmark *Benchmark
+ MetricMap model.MetricMap
+ EntryName string
+ }
+ CompareResults []*CompareResult
+
+ Status int
+)
+
+const (
+ NoChange Status = iota
+ Better
+ Worse
+ Regression
+)
+
+// status returns the status of a metric in the comparison.
+func (c *CompareResult) status(metricName string) Status {
+ entry := c.MetricMap[metricName]
+ if entry == nil {
+ return NoChange
+ }
+ cc := entry.ComputeComparison(c.EntryName, string(Old), string(New))
+ b := c.Benchmark
+ status := NoChange
+ threshold := b.Thresholds[metricName] * 100.0
+ if cc.Delta*float64(entry.Better) > 0 {
+ status = Better
+ } else if cc.Delta*float64(entry.Better) < 0 {
+ status = Worse
+ if math.Abs(cc.Delta) >= threshold {
+ status = Regression
+ }
+ }
+ return status
+}
+
+// regressed returns true if any metric in the comparison has regressed.
+func (c *CompareResult) regressed() bool {
+ for metric := range c.MetricMap {
+ status := c.status(metric)
+ if status == Regression {
+ return true
+ }
+ }
+ return false
+}
+
+// compare compares the metrics of a benchmark between two revisions.
+func (b *Benchmark) compare() (*CompareResult, error) {
+ builder := model.NewBuilder()
+ compareResult := CompareResult{Benchmark: b}
+ for _, revision := range []Revision{Old, New} {
+ data, err := os.ReadFile(path.Join(suite.artifactsDir(revision), b.cleanLog()))
+ if err != nil {
+ return nil, err
+ }
+ reader := benchfmt.NewReader(bytes.NewReader(data), b.cleanLog())
+ err = builder.AddMetrics(string(revision), "", reader)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ metricMap := builder.ComputeMetricMap()
+ if len(metricMap) == 0 {
+ return nil, errors.Newf("no metrics found for benchmark %s", b.Name)
+ }
+ entry := metricMap[maps.Keys(metricMap)[0]]
+ if len(entry.BenchmarkEntries) == 0 {
+ return nil, errors.Newf("no benchmark entries found for benchmark %s", b.Name)
+ }
+ entries := len(maps.Keys(entry.BenchmarkEntries))
+ if entries != 1 {
+ hint := fmt.Sprintf("please ensure the benchmark name %q atches a single benchmark", b.Name)
+ return nil, errors.Newf("unexpected multiple benchmark entries (%s): %v",
+ hint, maps.Keys(entry.BenchmarkEntries))
+ }
+
+ compareResult.EntryName = maps.Keys(entry.BenchmarkEntries)[0]
+ compareResult.MetricMap = metricMap
+ return &compareResult, nil
+}
+
+// compareBenchmarks compares the metrics of all benchmarks between two revisions.
+func (b Benchmarks) compareBenchmarks() (CompareResults, error) {
+ compareResults := make(CompareResults, 0, len(b))
+ for _, benchmark := range b {
+ compareResult, err := benchmark.compare()
+ if err != nil {
+ return nil, err
+ }
+ compareResults = append(compareResults, compareResult)
+ }
+ return compareResults, nil
+}
diff --git a/pkg/cmd/microbench-ci/config/pull-request-suite.yml b/pkg/cmd/microbench-ci/config/pull-request-suite.yml
new file mode 100644
index 000000000000..bcaf89b730a5
--- /dev/null
+++ b/pkg/cmd/microbench-ci/config/pull-request-suite.yml
@@ -0,0 +1,39 @@
+benchmarks:
+ - display_name: Sysbench
+ labels: ["SQL", "3node", "oltp_read_write"]
+ name: "BenchmarkSysbench/SQL/3node/oltp_read_write"
+ package: "pkg/sql/tests"
+ runner_group: 1
+ measure_count: 10
+ measure:
+ iterations: 250
+ thresholds:
+ "sec/op": .05
+ "B/op": .03
+ "allocs/op": .03
+
+ - display_name: Sysbench
+ labels: ["KV", "1node", "local", "oltp_read_only"]
+ name: "BenchmarkSysbench/KV/1node_local/oltp_read_only"
+ package: "pkg/sql/tests"
+ runner_group: 2
+ measure_count: 10
+ measure:
+ iterations: 800
+ thresholds:
+ "sec/op": .03
+ "B/op": .02
+ "allocs/op": .02
+
+ - display_name: Sysbench
+ labels: ["KV", "1node", "local", "oltp_write_only"]
+ name: "BenchmarkSysbench/KV/1node_local/oltp_write_only"
+ package: "pkg/sql/tests"
+ runner_group: 2
+ measure_count: 10
+ measure:
+ iterations: 800
+ thresholds:
+ "sec/op": .03
+ "B/op": .02
+ "allocs/op": .02
diff --git a/pkg/cmd/microbench-ci/github.go b/pkg/cmd/microbench-ci/github.go
new file mode 100644
index 000000000000..60a6b9ac7cb0
--- /dev/null
+++ b/pkg/cmd/microbench-ci/github.go
@@ -0,0 +1,74 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strings"
+
+ "github.com/cockroachdb/errors"
+ "github.com/google/go-github/v61/github"
+ "golang.org/x/oauth2"
+)
+
+// CommentTag is used to identify an existing comment.
+const CommentTag = ""
+
+// post posts or updates a comment on a GitHub PR, with the given summary text,
+// that has the CommentTag marker in it.
+func post(summaryText, githubRepository string, prNumber int) error {
+ token := os.Getenv("GITHUB_TOKEN")
+ if token == "" {
+ return errors.New("GITHUB_TOKEN is not set, this command is meant to be run in a GitHub Action")
+ }
+
+ repoInfo := strings.Split(githubRepository, "/")
+ if len(repoInfo) != 2 {
+ return errors.New("invalid GitHub repository flag")
+ }
+ owner := repoInfo[0]
+ repo := repoInfo[1]
+
+ ctx := context.Background()
+ ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
+ tc := oauth2.NewClient(ctx, ts)
+ client := github.NewClient(tc)
+
+ // Check for an existing comment
+ comments, _, err := client.Issues.ListComments(ctx, owner, repo, prNumber, nil)
+ if err != nil {
+ return err
+ }
+ var existingComment *github.IssueComment
+ for _, comment := range comments {
+ if strings.Contains(comment.GetBody(), CommentTag) {
+ existingComment = comment
+ break
+ }
+ }
+
+ commentBody := github.String(fmt.Sprintf("%s\n%s", summaryText, CommentTag))
+ if existingComment != nil {
+ // Update the existing comment
+ existingComment.Body = commentBody
+ _, _, err = client.Issues.EditComment(ctx, owner, repo, existingComment.GetID(), existingComment)
+ if err != nil {
+ return err
+ }
+ return nil
+ }
+
+ // Create a new comment
+ _, _, err = client.Issues.CreateComment(ctx, owner, repo, prNumber, &github.IssueComment{
+ Body: commentBody,
+ })
+ if err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/pkg/cmd/microbench-ci/main.go b/pkg/cmd/microbench-ci/main.go
new file mode 100644
index 000000000000..c766edc10b0e
--- /dev/null
+++ b/pkg/cmd/microbench-ci/main.go
@@ -0,0 +1,208 @@
+package main
+
+import (
+ "fmt"
+ "log"
+ "os"
+ "path"
+
+ "github.com/spf13/cobra"
+)
+
+type (
+ Config struct {
+ WorkingDir string
+ BenchmarkConfigPath string
+ SummaryPath string
+ GitHubSummaryPath string
+ BuildID string
+ Old string
+ New string
+ Group int
+ }
+
+ Suite struct {
+ Benchmarks Benchmarks
+ Revisions Revisions
+ }
+
+ Revisions map[Revision]string
+ Revision string
+)
+
+const (
+ Old Revision = "old"
+ New Revision = "new"
+)
+
+// Bucket is the GCS bucket where artifacts are stored.
+const Bucket = "cockroach-microbench-ci"
+
+var (
+ config *Config
+ suite *Suite
+)
+
+func defaultConfig() *Config {
+ return &Config{
+ WorkingDir: "microbench-ci-artifacts",
+ BenchmarkConfigPath: "pkg/cmd/microbench-ci/config/pull-request-suite.yml",
+ SummaryPath: "summary.json",
+ GitHubSummaryPath: "github-summary.md",
+ }
+}
+
+func (c *Config) loadSuite() error {
+ suite = &Suite{
+ Revisions: Revisions{
+ Old: config.Old,
+ New: config.New,
+ },
+ }
+ benchmarks, err := loadBenchmarkConfig(c.BenchmarkConfigPath)
+ if err != nil {
+ return err
+ }
+ suite.Benchmarks = benchmarks
+ return nil
+}
+
+func (s *Suite) revisionDir(revision Revision) string {
+ return path.Join(config.WorkingDir, s.revisionSHA(revision))
+}
+
+func (s *Suite) revisionSHA(revision Revision) string {
+ return s.Revisions[revision]
+}
+
+func (s *Suite) binDir(revision Revision) string {
+ return path.Join(s.revisionDir(revision), "bin")
+}
+
+func (s *Suite) artifactsDir(revision Revision) string {
+ return path.Join(s.revisionDir(revision), "artifacts")
+}
+
+func (s *Suite) artifactsURL(revision Revision) string {
+ return fmt.Sprintf("gs://%s/artifacts/%s/%s/",
+ Bucket, s.revisionSHA(revision), config.BuildID)
+}
+
+func (s *Suite) binURL(revision Revision, benchmark *Benchmark) string {
+ return fmt.Sprintf("gs://%s/builds/%s/bin/%s",
+ Bucket, s.revisionSHA(revision), benchmark.binaryName())
+}
+
+func makeRunCommand() *cobra.Command {
+ cmdFunc := func(cmd *cobra.Command, commandLine []string) error {
+ if err := config.loadSuite(); err != nil {
+ return err
+ }
+ return suite.Benchmarks.run()
+ }
+ cmd := &cobra.Command{
+ Use: "run",
+ Short: "run benchmarks and output artifacts",
+ Args: cobra.ExactArgs(0),
+ RunE: cmdFunc,
+ }
+ cmd.Flags().IntVar(&config.Group, "group", config.Group, "run only the benchmarks in the group (0 runs all groups)")
+ return cmd
+}
+
+func makeCompareCommand() *cobra.Command {
+ cmdFunc := func(cmd *cobra.Command, commandLine []string) error {
+ if err := config.loadSuite(); err != nil {
+ return err
+ }
+ results, err := suite.Benchmarks.compareBenchmarks()
+ if err != nil {
+ return err
+ }
+ if err = results.writeJSONSummary(config.SummaryPath); err != nil {
+ return err
+ }
+ return results.writeGitHubSummary(config.GitHubSummaryPath)
+ }
+ cmd := &cobra.Command{
+ Use: "compare",
+ Short: "compare artifacts for the given revisions and output an HTML report",
+ Args: cobra.ExactArgs(0),
+ RunE: cmdFunc,
+ }
+ cmd.Flags().StringVar(&config.SummaryPath, "summary", config.SummaryPath, "path to write comparison results to (JSON)")
+ return cmd
+}
+
+func makePostCommand() (*cobra.Command, error) {
+ repo := "cockroachdb/cockroach"
+ var prNumber int
+ cmdFunc := func(cmd *cobra.Command, commandLine []string) error {
+ summaryText, err := os.ReadFile(config.GitHubSummaryPath)
+ if err != nil {
+ return err
+ }
+ return post(string(summaryText), repo, prNumber)
+ }
+ cmd := &cobra.Command{
+ Use: "post",
+ Short: "post creates or updates a microbench-ci summary comment on a GitHub PR",
+ Args: cobra.ExactArgs(0),
+ RunE: cmdFunc,
+ }
+ cmd.Flags().StringVar(&repo, "repo", repo, "repository")
+ cmd.Flags().IntVar(&prNumber, "pr-number", 0, "PR number")
+ if err := cmd.MarkFlagRequired("pr-number"); err != nil {
+ return nil, err
+ }
+ return cmd, nil
+}
+
+func run() error {
+ cmd := &cobra.Command{
+ Use: "microbench-ci [command] (flags)",
+ Short: "microbench-ci is used by the Microbenchmark CI pipeline to run benchmarks.",
+ SilenceUsage: true,
+ SilenceErrors: true,
+ }
+
+ runCmd := makeRunCommand()
+ compareCmd := makeCompareCommand()
+ postCmd, err := makePostCommand()
+ if err != nil {
+ return err
+ }
+
+ for _, c := range []*cobra.Command{runCmd, compareCmd} {
+ c.Flags().StringVar(&config.WorkingDir, "working-dir", config.WorkingDir, "directory to store or load artifacts from")
+ c.Flags().StringVar(&config.BenchmarkConfigPath, "config", config.BenchmarkConfigPath, "suite configuration file")
+ c.Flags().StringVar(&config.Old, "old", "", "old commit")
+ c.Flags().StringVar(&config.New, "new", "", "new commit")
+ if err = c.MarkFlagRequired("old"); err != nil {
+ return err
+ }
+ if err = c.MarkFlagRequired("new"); err != nil {
+ return err
+ }
+ }
+
+ for _, c := range []*cobra.Command{postCmd, compareCmd} {
+ c.Flags().StringVar(&config.GitHubSummaryPath, "github-summary", config.GitHubSummaryPath, "path to write comparison results to (GitHub Markdown)")
+ }
+
+ compareCmd.Flags().StringVar(&config.BuildID, "build-id", config.BuildID, "GitHub build ID to identify this run")
+
+ cmd.AddCommand(runCmd)
+ cmd.AddCommand(compareCmd)
+ cmd.AddCommand(postCmd)
+
+ return cmd.Execute()
+}
+
+func main() {
+ config = defaultConfig()
+ if err := run(); err != nil {
+ log.Printf("ERROR: %+v", err)
+ os.Exit(1)
+ }
+}
diff --git a/pkg/cmd/microbench-ci/report.go b/pkg/cmd/microbench-ci/report.go
new file mode 100644
index 000000000000..e516fd1b769d
--- /dev/null
+++ b/pkg/cmd/microbench-ci/report.go
@@ -0,0 +1,204 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package main
+
+import (
+ _ "embed"
+ "encoding/json"
+ "fmt"
+ "math"
+ "os"
+ "sort"
+ "strings"
+ "text/template"
+
+ "golang.org/x/perf/benchmath"
+ "golang.org/x/perf/benchunit"
+)
+
+//go:embed template/github_summary.md
+var githubSummary string
+
+type SummaryData struct {
+ Metric string
+ OldCenter string
+ NewCenter string
+ Delta string
+ Note string
+ Threshold string
+ Status string
+}
+
+type GitHubData struct {
+ BenchmarkStatus string
+ DisplayName string
+ Labels string
+ Summaries []SummaryData
+ Benchdiff BenchdiffData
+}
+
+type BenchdiffData struct {
+ Package string
+ Run string
+ Dir map[Revision]string
+ BinURL map[Revision]string
+ BinDest map[Revision]string
+ TrimmedSHA map[Revision]string
+ Old Revision
+ New Revision
+}
+
+func (c *CompareResult) generateSummaryData(
+ statusTemplateFunc func(status Status) string,
+) []SummaryData {
+ summaryData := make([]SummaryData, 0, len(c.MetricMap))
+ for metricName, entry := range c.MetricMap {
+ benchmark := entry.BenchmarkEntries[c.EntryName]
+ cc := entry.ComputeComparison(c.EntryName, string(Old), string(New))
+ threshold := c.Benchmark.Thresholds[metricName] * 100.0
+ status := statusTemplateFunc(c.status(metricName))
+ oldSum := benchmark.Summaries[string(Old)]
+ newSum := benchmark.Summaries[string(New)]
+ summaryData = append(summaryData, SummaryData{
+ Metric: metricName,
+ OldCenter: fmt.Sprintf("%s ±%s", formatValue(oldSum.Center, metricName), oldSum.PctRangeString()),
+ NewCenter: fmt.Sprintf("%s ±%s", formatValue(newSum.Center, metricName), newSum.PctRangeString()),
+ Delta: cc.FormattedDelta,
+ Note: cc.Distribution.String(),
+ Threshold: fmt.Sprintf("%.1f%%", threshold),
+ Status: status,
+ })
+ }
+ sort.Slice(summaryData, func(i, j int) bool {
+ return summaryData[i].Metric > summaryData[j].Metric
+ })
+ return summaryData
+}
+
+func (c *CompareResult) benchdiffData() BenchdiffData {
+ d := BenchdiffData{
+ Package: c.Benchmark.Package,
+ Run: c.Benchmark.Name,
+ Dir: make(map[Revision]string),
+ BinURL: make(map[Revision]string),
+ BinDest: make(map[Revision]string),
+ TrimmedSHA: make(map[Revision]string),
+ Old: Old,
+ New: New,
+ }
+ for _, revision := range []Revision{Old, New} {
+ sha := suite.revisionSHA(revision)
+ shortSHA := sha[:int(math.Min(float64(len(sha)), 7))]
+ d.Dir[revision] = fmt.Sprintf("benchdiff/%s/bin/%s",
+ shortSHA, c.Benchmark.packageHash())
+ d.BinURL[revision] = suite.binURL(revision, c.Benchmark)
+ d.BinDest[revision] = fmt.Sprintf("%s/%s", d.Dir[revision],
+ "cockroachdb_cockroach_"+c.Benchmark.sanitizedPackageName())
+ d.TrimmedSHA[revision] = shortSHA
+ }
+
+ return d
+}
+
+// writeJSONSummary writes a JSON summary of the comparison results to the given
+// path.
+func (c CompareResults) writeJSONSummary(path string) error {
+ file, err := os.Create(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+ encoder := json.NewEncoder(file)
+
+ type Data struct {
+ Metric string
+ Summary benchmath.Summary
+ }
+ data := make(map[string][]Data)
+ for _, cr := range c {
+ for name, m := range cr.MetricMap {
+ for _, r := range []Revision{Old, New} {
+ data[string(r)] = append(data[string(r)], Data{
+ Metric: name,
+ Summary: *m.BenchmarkEntries[cr.EntryName].Summaries[string(r)],
+ })
+ }
+ }
+ }
+ return encoder.Encode(struct {
+ Summaries map[string][]Data
+ Revisions Revisions
+ }{
+ Summaries: data,
+ Revisions: suite.Revisions,
+ })
+}
+
+// writeGitHubSummary writes a markdown summary of the comparison results to the
+// given path.
+func (c CompareResults) writeGitHubSummary(path string) error {
+ file, err := os.Create(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ regressionDetected := false
+ summaries := make([]GitHubData, 0, len(c))
+ for _, cr := range c {
+ finalStatus := NoChange
+ data := GitHubData{
+ DisplayName: cr.Benchmark.DisplayName,
+ Labels: strings.Join(cr.Benchmark.Labels, ", "),
+ Benchdiff: cr.benchdiffData(),
+ }
+ summaryData := cr.generateSummaryData(func(status Status) string {
+ if status > finalStatus {
+ finalStatus = status
+ }
+ if status == Regression {
+ regressionDetected = true
+ }
+ return statusToDot(status)
+ })
+ data.BenchmarkStatus = statusToDot(finalStatus)
+ data.Summaries = summaryData
+ summaries = append(summaries, data)
+ }
+
+ tmpl, err := template.New("github").Parse(githubSummary)
+ if err != nil {
+ return err
+ }
+ description := "No regressions detected!"
+ if regressionDetected {
+ description = "A regression has been detected, please investigate further!"
+ }
+ return tmpl.Execute(file, struct {
+ GitHubSummaryData []GitHubData
+ Artifacts map[Revision]string
+ Description string
+ Commit string
+ }{
+ GitHubSummaryData: summaries,
+ Description: description,
+ Artifacts: map[Revision]string{
+ Old: suite.artifactsURL(Old),
+ New: suite.artifactsURL(New),
+ },
+ Commit: suite.revisionSHA(New),
+ })
+}
+
+func statusToDot(status Status) string {
+ return string([]rune("⚪🟢🟡🔴")[status])
+}
+
+// formatValue formats a value according to the unit of the metric.
+func formatValue(val float64, metric string) string {
+ cls := benchunit.ClassOf(metric)
+ return benchunit.Scale(val, cls)
+}
diff --git a/pkg/cmd/microbench-ci/run.go b/pkg/cmd/microbench-ci/run.go
new file mode 100644
index 000000000000..7d70a88a528e
--- /dev/null
+++ b/pkg/cmd/microbench-ci/run.go
@@ -0,0 +1,147 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package main
+
+import (
+ "log"
+ "os"
+ "os/exec"
+ "path"
+ "strings"
+
+ "github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench/parser"
+ "github.com/cockroachdb/errors"
+)
+
+// command returns the command to run the benchmark binary for a given revision
+// and iteration type.
+func (b *Benchmark) command(revision Revision, iterType BenchmarkIterationType) *exec.Cmd {
+ cmd := exec.Command(
+ path.Join(suite.binDir(revision), b.binaryName()),
+ b.args(suite.artifactsDir(revision), iterType)...,
+ )
+ cmd.Env = append(os.Environ(), "COCKROACH_RANDOM_SEED=1")
+ return cmd
+}
+
+// runIteration runs a single iteration of the benchmark for the given revision.
+func (b *Benchmark) runIteration(revision Revision) error {
+ cmd := b.command(revision, Measure)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return errors.Wrapf(err, "benchmark %q, command %q failed to run:\n%s",
+ b.DisplayName, cmd.String(), string(output))
+ }
+
+ results := parser.ExtractBenchmarkResults(string(output))
+ if results.Errors {
+ return errors.Newf("benchmark results contained errors:\n%s", string(output))
+ }
+ if results.Skipped {
+ return errors.Newf("benchmark invalid due to being skipped:\n%s", string(output))
+ }
+
+ var cleanedOutput strings.Builder
+ for _, result := range results.Results {
+ cleanedOutput.WriteString(strings.Join(result, " "))
+ cleanedOutput.WriteString("\n")
+ }
+ err = appendToFile(path.Join(suite.artifactsDir(revision), b.cleanLog()), cleanedOutput.String())
+ if err != nil {
+ return err
+ }
+ err = appendToFile(path.Join(suite.artifactsDir(revision), b.rawLog()), string(output))
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// collectProfiles collects the profiles of both revisions for the benchmark in
+// an interleaved manner.
+func (b *Benchmark) collectProfiles() error {
+ for _, profile := range []BenchmarkIterationType{ProfileCPU, ProfileMemory, ProfileMutex} {
+ for _, revision := range []Revision{New, Old} {
+ cmd := b.command(revision, profile)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return errors.Wrapf(err, "profile benchmark %q, command %q failed to run:\n%s",
+ b.DisplayName, cmd.String(), string(output))
+ }
+ }
+ }
+ return nil
+}
+
+// run runs the benchmark as configured in the Benchmark struct.
+func (b *Benchmark) run() error {
+ for _, revision := range []Revision{New, Old} {
+ err := os.MkdirAll(suite.artifactsDir(revision), 0755)
+ if err != nil {
+ return err
+ }
+ }
+
+ log.Printf("Running benchmark %q for %d iterations", b.Name, b.MeasureCount)
+ for i := 0; i < b.MeasureCount; i++ {
+ for _, revision := range []Revision{New, Old} {
+ log.Printf("%s binary iteration (%d out of %d)",
+ revision, i+1, b.MeasureCount,
+ )
+ err := b.runIteration(revision)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // Only collect profiles if there was a regression.
+ compareResult, err := b.compare()
+ if err != nil {
+ return err
+ }
+ if compareResult.regressed() {
+ // Mark the revision as failed.
+ for _, revision := range []Revision{New, Old} {
+ err = os.WriteFile(path.Join(suite.artifactsDir(revision), ".FAILED"), nil, 0644)
+ if err != nil {
+ return err
+ }
+ }
+ log.Printf("collecting profiles for each revision")
+ if err = b.collectProfiles(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func appendToFile(filename, data string) error {
+ f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ if _, err = f.WriteString(data); err != nil {
+ return err
+ }
+ return nil
+}
+
+// run runs all benchmarks in the Benchmarks suite for the active runner group.
+// The runner groups help split the benchmarks up between multiple VMs to reduce
+// overall CI runtime.
+func (b Benchmarks) run() error {
+ for _, benchmark := range b {
+ if config.Group != 0 && benchmark.RunnerGroup != config.Group {
+ continue
+ }
+ if err := benchmark.run(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/pkg/cmd/microbench-ci/template/github_summary.md b/pkg/cmd/microbench-ci/template/github_summary.md
new file mode 100644
index 000000000000..7b0cdeabd043
--- /dev/null
+++ b/pkg/cmd/microbench-ci/template/github_summary.md
@@ -0,0 +1,45 @@
+{{- range .GitHubSummaryData}}
+{{.BenchmarkStatus}} {{.DisplayName}} [{{.Labels}}]
+
+| Metric | Old Commit | New Commit | Delta | Note | Threshold |
+|-----------------------------|----------------|----------------|------------|--------------|----------------|
+{{- range .Summaries}}
+| {{.Status}} **{{.Metric}}** | {{.OldCenter}} | {{.NewCenter}} | {{.Delta}} | {{.Note}} | {{.Threshold}} |
+{{- end}}
+
+Reproduce
+
+**benchdiff binaries**:
+```shell
+{{- $benchdiff := .Benchdiff}}
+{{- range $rev, $dir := $benchdiff.Dir }}
+mkdir -p {{$dir}}
+gcloud storage cp {{index $benchdiff.BinURL $rev}} {{index $benchdiff.BinDest $rev}}
+chmod +x {{index $benchdiff.BinDest $rev}}
+{{- end}}
+```
+**benchdiff command**:
+```shell
+benchdiff --run=^{{$benchdiff.Run}}$ --old={{index $benchdiff.TrimmedSHA $benchdiff.Old}} --new={{index $benchdiff.TrimmedSHA $benchdiff.New}} ./{{$benchdiff.Package}}
+```
+
+Artifacts
+
+**download**:
+```shell
+{{- range $rev, $url := $.Artifacts }}
+mkdir -p {{$rev}}
+gcloud storage cp {{$url}}\* {{$rev}}/
+{{- end}}
+```
+
+