Update

.github/workflows/benchmark.yaml

@@ -41,6 +41,11 @@ jobs:
       run: |
         ./hack/e2e-test.sh e2e/kwokctl/benchmark
 
+    - name: Test Benchmark PGO
+      shell: bash
+      run: |
+        ./hack/e2e-test.sh e2e/kwokctl/benchmark-pgo
+
     - name: Test Benchmark Hack
       shell: bash
       run: |

hack/releases.sh

@@ -24,6 +24,7 @@ BINARY_PREFIX=""
 BINARY_NAME=""
 VERSION=""
 KUBE_VERSION=""
+PGO=""
 STAGING_PREFIX=""
 PRE_RELEASE=""
 DRY_RUN=false
@@ -45,6 +46,7 @@ function usage() {
   echo "  --binary-name <binary-name> is kwok binary name"
   echo "  --version <version> is version of binary"
   echo "  --kube-version <kube-version> is default version of Kubernetes"
+  echo "  --pgo <pgo-path is the pgo of the binary>"
   echo "  --staging-prefix <staging-prefix> is staging prefix for bucket"
   echo "  --push will push binary to bucket"
   echo "  --dry-run just show what would be done"
@@ -95,6 +97,10 @@ function args() {
       [[ "${arg#*=}" != "${arg}" ]] && KUBE_VERSION="${arg#*=}" || { KUBE_VERSION="${2}" && shift; } || :
       shift
       ;;
+    --pgo | --pgo=*)
+      [[ "${arg#*=}" != "${arg}" ]] && PGO="${arg#*=}" || { PGO="${2}" && shift; } || :
+      shift
+      ;;
     --staging-prefix | --staging-prefix=*)
       [[ "${arg#*=}" != "${arg}" ]] && STAGING_PREFIX="${arg#*=}" || { STAGING_PREFIX="${2}" && shift; } || :
       shift
@@ -174,6 +180,10 @@ function main() {
     extra_args+=("-ldflags" "'${LDFLAGS[*]}'")
   fi
 
+  if [[ "${PGO}" != "" ]]; then
+    extra_args+=("-pgo" "${PGO}")
+  fi
+
   for platform in "${PLATFORMS[@]}"; do
     os="${platform%%/*}"
     for binary in "${BINS[@]}"; do

pkg/apis/config/v1alpha1/kwok_configuration_types.go

@@ -128,7 +128,7 @@ type KwokConfigurationOptions struct {
 	EnableDebuggingHandlers *bool `json:"enableDebuggingHandlers,omitempty"`
 
 	// enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true.
-	// +default=false
+	// +default=true
 	EnableContentionProfiling *bool `json:"enableContentionProfiling,omitempty"`
 
 	// EnableProfiling enables /debug/pprof handler, if enableDebuggingHandlers is true.

pkg/kwok/server/profiling.go

@@ -31,6 +31,9 @@ func (s *Server) InstallProfilingHandler(enableProfilingLogHandler bool, enableC
 
 	// Setup pprof handlers.
 	s.restfulCont.Handle(pprofBasePath, http.HandlerFunc(pprof.Index))
+	s.restfulCont.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+	s.restfulCont.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+	s.restfulCont.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
 	if enableContentionProfiling {
 		runtime.SetBlockProfileRate(1)
 	}

test/e2e/benchmark.go

@@ -22,6 +22,8 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
+	"os"
 	"os/exec"
 	"strconv"
 	"testing"
@@ -31,13 +33,35 @@ import (
 	"sigs.k8s.io/e2e-framework/pkg/features"
 )
 
+func savePGO(output string, seconds int) error {
+	resp, err := http.Get("http://127.0.0.1:10247/debug/pprof/profile?seconds=" + strconv.Itoa(seconds))
+	if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	_, err = io.Copy(f, resp.Body)
+	return err
+}
+
 func waitResource(ctx context.Context, t *testing.T, kwokctlPath, name, resource, reason string, want, gap, tolerance int) error {
 	cmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "kubectl", "get", "--no-headers", "--watch", resource)
 
-	outpipe, err := cmd.StdoutPipe()
+	pr, pw, err := os.Pipe()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		_ = pw.Close()
+	}()
+
+	cmd.Stdout = pw
 
 	err = cmd.Start()
 	if err != nil {
@@ -48,7 +72,7 @@ func waitResource(ctx context.Context, t *testing.T, kwokctlPath, name, resource
 	prev := 0
 	got := 0
 	var latestTime time.Time
-	reader := bufio.NewReader(outpipe)
+	reader := bufio.NewReader(pr)
 	for {
 		line, _, err := reader.ReadLine()
 		if err != nil {
@@ -102,60 +126,88 @@ func waitResource(ctx context.Context, t *testing.T, kwokctlPath, name, resource
 	}
 }
 
-func scaleCreatePod(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error {
+func scaleCreatePod(ctx context.Context, t *testing.T, pgo bool, kwokctlPath string, name string, size int) error {
 	nodeName := "fake-node-000000"
 	scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "scale", "pod", "fake-pod", "--replicas", strconv.Itoa(size), "--param", fmt.Sprintf(".nodeName=%q", nodeName)) // #nosec G204
 	if err := scaleCmd.Start(); err != nil {
 		return fmt.Errorf("failed to start scale command: %w", err)
 	}
 
+	if pgo {
+		go func() {
+			err := savePGO("create-pod.pgo", 30)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}()
+	}
 	if err := waitResource(ctx, t, kwokctlPath, name, "Pod", "Running", size, 5, 10); err != nil {
 		return fmt.Errorf("failed to wait for resource: %w", err)
 	}
+
 	return nil
 }
 
-func scaleDeletePod(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error {
+func scaleDeletePod(ctx context.Context, t *testing.T, pgo bool, kwokctlPath string, name string, size int) error {
 	scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "scale", "pod", "fake-pod", "--replicas", strconv.Itoa(0)) // #nosec G204
 	if err := scaleCmd.Start(); err != nil {
 		return fmt.Errorf("failed to start scale command: %w", err)
 	}
 
+	if pgo {
+		go func() {
+			err := savePGO("delete-pod.pgo", 30)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}()
+	}
+
 	if err := waitResource(ctx, t, kwokctlPath, name, "Pod", "Terminating", size, 0, 0); err != nil {
 		return fmt.Errorf("failed to wait for resource: %w", err)
 	}
 	return nil
 }
 
-func scaleCreateNode(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error {
+func scaleCreateNode(ctx context.Context, t *testing.T, pgo bool, kwokctlPath string, name string, size int) error {
 	scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "scale", "node", "fake-node", "--replicas", strconv.Itoa(size)) // #nosec G204
 	if err := scaleCmd.Start(); err != nil {
 		return fmt.Errorf("failed to start scale command: %w", err)
 	}
 
+	if pgo {
+		go func() {
+			err := savePGO("create-node.pgo", 30)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}()
+	}
+
 	if err := waitResource(ctx, t, kwokctlPath, name, "Node", "Ready", size, 10, 100); err != nil {
 		return fmt.Errorf("failed to wait for resource: %w", err)
 	}
 	return nil
 }
 
-func CaseBenchmark(kwokctlPath, clusterName string) *features.FeatureBuilder {
+func CaseBenchmark(kwokctlPath, clusterName string, pgo bool) *features.FeatureBuilder {
 	return features.New("Benchmark").
 		Assess("Create nodes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
 			ctx0, cancel := context.WithTimeout(ctx, 240*time.Second)
 			defer cancel()
 
-			err := scaleCreateNode(ctx0, t, kwokctlPath, clusterName, 10000)
+			err := scaleCreateNode(ctx0, t, pgo, kwokctlPath, clusterName, 10000)
 			if err != nil {
 				t.Fatal(err)
 			}
+
 			return ctx
 		}).
 		Assess("Create pods", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
 			ctx0, cancel := context.WithTimeout(ctx, 480*time.Second)
 			defer cancel()
 
-			err := scaleCreatePod(ctx0, t, kwokctlPath, clusterName, 50000)
+			err := scaleCreatePod(ctx0, t, pgo, kwokctlPath, clusterName, 100000)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -165,7 +217,7 @@ func CaseBenchmark(kwokctlPath, clusterName string) *features.FeatureBuilder {
 			ctx0, cancel := context.WithTimeout(ctx, 480*time.Second)
 			defer cancel()
 
-			err := scaleDeletePod(ctx0, t, kwokctlPath, clusterName, 50000)
+			err := scaleDeletePod(ctx0, t, pgo, kwokctlPath, clusterName, 100000)
 			if err != nil {
 				t.Fatal(err)
 			}

test/e2e/benchmark_hack.go

@@ -173,7 +173,7 @@ func CaseBenchmarkWithHack(kwokctlPath, clusterName string) *features.FeatureBui
 			ctx0, cancel := context.WithTimeout(ctx, 240*time.Second)
 			defer cancel()
 
-			err := scaleCreateNodeWithHack(ctx0, t, kwokctlPath, clusterName, 10000)
+			err := scaleCreateNodeWithHack(ctx0, t, kwokctlPath, clusterName, 5000)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -183,7 +183,7 @@ func CaseBenchmarkWithHack(kwokctlPath, clusterName string) *features.FeatureBui
 			ctx0, cancel := context.WithTimeout(ctx, 480*time.Second)
 			defer cancel()
 
-			err := scaleCreatePodWithHack(ctx0, t, kwokctlPath, clusterName, 50000)
+			err := scaleCreatePodWithHack(ctx0, t, kwokctlPath, clusterName, 10000)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -193,7 +193,7 @@ func CaseBenchmarkWithHack(kwokctlPath, clusterName string) *features.FeatureBui
 			ctx0, cancel := context.WithTimeout(ctx, 480*time.Second)
 			defer cancel()
 
-			err := scaleDeletePodWithHack(ctx0, t, kwokctlPath, clusterName, 50000)
+			err := scaleDeletePodWithHack(ctx0, t, kwokctlPath, clusterName, 10000)
 			if err != nil {
 				t.Fatal(err)
 			}

test/e2e/helper/build.go

@@ -53,21 +53,26 @@ func BuildKwokImage(rootDir string, image string, builder string) env.Func {
 
 // BuildKwokBinary builds the kwok binary and returns a function that can be used
 func BuildKwokBinary(rootDir string) env.Func {
-	return buildBinary(rootDir, "kwok")
+	return buildBinary(rootDir, "kwok", "")
+}
+
+// BuildKwokBinaryWithPGO builds the kwok binary and returns a function that can be used
+func BuildKwokBinaryWithPGO(rootDir string, pgo string) env.Func {
+	return buildBinary(rootDir, "kwok", pgo)
 }
 
 // BuildKwokctlBinary builds the kwokctl binary and returns a function that can be used
 func BuildKwokctlBinary(rootDir string) env.Func {
-	return buildBinary(rootDir, "kwokctl")
+	return buildBinary(rootDir, "kwokctl", "")
 }
 
 // buildBinary builds the kwok binary and returns a function that can be used
-func buildBinary(rootDir string, binary string) env.Func {
+func buildBinary(rootDir string, binary string, pgo string) env.Func {
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 		ctx = exec.WithStdIO(ctx)
 		ctx = exec.WithDir(ctx, rootDir)
 
-		err := exec.Exec(ctx, "bash", "./hack/releases.sh", "--bin", binary, "--platform", runtime.GOOS+"/"+runtime.GOARCH)
+		err := exec.Exec(ctx, "bash", "./hack/releases.sh", "--bin", binary, "--pgo", pgo, "--platform", runtime.GOOS+"/"+runtime.GOARCH)
 		if err != nil {
 			return ctx, err
 		}

test/e2e/kwokctl/benchmark-pgo/kubectl_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package benchmark_test
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kwok/test/e2e"
+)
+
+func TestBenchmark(t *testing.T) {
+	f0 := e2e.CaseBenchmark(kwokctlPath, clusterName, false).
+		Feature()
+	testEnv.Test(t, f0)
+}

test/e2e/kwokctl/benchmark-pgo/main_test.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package benchmarking_test is a test benchmarking environment for kwok.
+package benchmark_test
+
+import (
+	"os"
+	"runtime"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/support/kwok"
+
+	"sigs.k8s.io/kwok/pkg/consts"
+	"sigs.k8s.io/kwok/pkg/utils/path"
+	"sigs.k8s.io/kwok/test/e2e/helper"
+)
+
+var (
+	runtimeEnv  = consts.RuntimeTypeBinary
+	pgoPath     = path.Join(rootDir, "test/e2e/kwokctl/benchmark/default.pgo")
+	testEnv     env.Environment
+	pwd         = os.Getenv("PWD")
+	rootDir     = path.Join(pwd, "../../../..")
+	logsDir     = path.Join(rootDir, "logs")
+	clusterName = envconf.RandomName("kwok-e2e-benchmark", 24)
+	kwokPath    = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwok"+helper.BinSuffix)
+	kwokctlPath = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwokctl"+helper.BinSuffix)
+	baseArgs    = []string{
+		"--kwok-controller-binary=" + kwokPath,
+		"--runtime=" + runtimeEnv,
+		"--wait=15m",
+		"--disable-kube-scheduler",
+		"--disable-qps-limits",
+	}
+)
+
+func init() {
+	_ = os.Setenv("KWOK_WORKDIR", path.Join(rootDir, "workdir"))
+}
+
+func TestMain(m *testing.M) {
+	testEnv = helper.Environment()
+
+	k := kwok.NewProvider().
+		WithName(clusterName).
+		WithPath(kwokctlPath)
+	testEnv.Setup(
+		helper.BuildKwokBinaryWithPGO(rootDir, pgoPath),
+		helper.BuildKwokctlBinary(rootDir),
+		helper.CreateCluster(k, baseArgs...),
+	)
+	testEnv.Finish(
+		helper.ExportLogs(k, logsDir),
+		helper.DestroyCluster(k),
+	)
+	os.Exit(testEnv.Run(m))
+}