diff --git a/.semaphore/push-images/mock-node.yml b/.semaphore/push-images/mock-node.yml new file mode 100644 index 00000000000..84938a748f5 --- /dev/null +++ b/.semaphore/push-images/mock-node.yml @@ -0,0 +1,47 @@ +version: v1.0 +name: Publish mock-node images +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 + +execution_time_limit: + minutes: 60 + +global_job_config: + env_vars: + - name: DEV_REGISTRIES + value: quay.io/calico docker.io/calico + secrets: + - name: docker + - name: quay-robot-calico+semaphoreci + prologue: + commands: + - checkout + # Semaphore is doing shallow clone on a commit without tags. + # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always) + - retry git fetch --unshallow + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - echo $QUAY_TOKEN | docker login --username "$QUAY_USER" --password-stdin quay.io + - export BRANCH_NAME=$SEMAPHORE_GIT_BRANCH + +blocks: + - name: Publish mock-node images + dependencies: [] + skip: + when: "branch !~ '.+'" + task: + jobs: + - name: Linux multi-arch + commands: + - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C test-tools/mocknode cd CONFIRM=true; fi + - name: Publish mock-node multi-arch manifests + dependencies: + - Publish mock-node images + skip: + when: "branch !~ '.+'" + task: + jobs: + - name: Linux multi-arch manifests + commands: + - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C test-tools/mocknode push-manifests-with-tag CONFIRM=true; fi diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 33f8eedd1d7..52739afcf48 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -62,6 +62,10 @@ promotions: pipeline_file: push-images/node.yml auto_promote: when: "branch =~ 'master|release-'" + - name: Push mock calico-node images + pipeline_file: push-images/mock-node.yml + auto_promote: + when: "branch =~ 'master|release-'" - name: Push cni-plugin images pipeline_file: push-images/cni-plugin.yml auto_promote: @@ -810,6 +814,19 @@ blocks: - mkdir logs - sudo journalctl > logs/journalctl.txt - artifact push job --expire-in 1d logs +- name: Mock node + run: + when: "true or change_in(['/*', '/test-tools/mocknode/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']${DEFAULT_BRANCH}})" + dependencies: + - Prerequisites + task: + prologue: + commands: + - cd test-tools/mocknode + jobs: + - name: Mock node + commands: + - ../../.semaphore/run-and-monitor make-ci.log make ci - name: release tooling run: when: "true or change_in(['/*', '/release/'], {exclude: ['/**/.gitignore', '/**/*.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e47c33c081b..08cc399ca9d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -62,6 +62,10 @@ promotions: pipeline_file: push-images/node.yml auto_promote: when: "branch =~ 'master|release-'" + - name: Push mock calico-node images + pipeline_file: push-images/mock-node.yml + auto_promote: + when: "branch =~ 'master|release-'" - name: Push cni-plugin images pipeline_file: push-images/cni-plugin.yml auto_promote: @@ -810,6 +814,19 @@ blocks: - mkdir logs - sudo journalctl > logs/journalctl.txt - artifact push job --expire-in 1d logs +- name: Mock node + run: + when: "false or change_in(['/*', '/test-tools/mocknode/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']${DEFAULT_BRANCH}})" + dependencies: + - Prerequisites + task: + prologue: + commands: + - cd test-tools/mocknode + jobs: + - name: Mock node + commands: + - ../../.semaphore/run-and-monitor make-ci.log make ci - name: release tooling run: when: "false or change_in(['/*', '/release/'], {exclude: ['/**/.gitignore', '/**/*.md', '/**/LICENSE']})" diff --git a/.semaphore/semaphore.yml.d/03-promotions.yml b/.semaphore/semaphore.yml.d/03-promotions.yml index dc902d1aeaa..e5f7bf3e709 100644 --- a/.semaphore/semaphore.yml.d/03-promotions.yml +++ b/.semaphore/semaphore.yml.d/03-promotions.yml @@ -31,6 +31,10 @@ promotions: pipeline_file: push-images/node.yml auto_promote: when: "branch =~ 'master|release-'" + - name: Push mock calico-node images + pipeline_file: push-images/mock-node.yml + auto_promote: + when: "branch =~ 'master|release-'" - name: Push cni-plugin images pipeline_file: push-images/cni-plugin.yml auto_promote: diff --git a/.semaphore/semaphore.yml.d/blocks/60-mock-node.yml b/.semaphore/semaphore.yml.d/blocks/60-mock-node.yml new file mode 100644 index 00000000000..7f15490694d --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/60-mock-node.yml @@ -0,0 +1,13 @@ +- name: Mock node + run: + when: "${FORCE_RUN} or change_in(['/*', '/test-tools/mocknode/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']${DEFAULT_BRANCH}})" + dependencies: + - Prerequisites + task: + prologue: + commands: + - cd test-tools/mocknode + jobs: + - name: Mock node + commands: + - ../../.semaphore/run-and-monitor make-ci.log make ci diff --git a/test-tools/mocknode/Dockerfile b/test-tools/mocknode/Dockerfile new file mode 100644 index 00000000000..6734c654544 --- /dev/null +++ b/test-tools/mocknode/Dockerfile @@ -0,0 +1,37 @@ +# Copyright (c) 2023 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM scratch AS source + +ARG TARGETARCH + +# And the mock-node itself... +COPY bin/test-tools/mocknode-${TARGETARCH} /usr/bin/mocknode + +FROM calico/base + +ARG GIT_VERSION=unknown + +# Required labels for certification +LABEL description="Calico mock node for scale testing" +LABEL maintainer="shaun@tigera.io" +LABEL name="Calico mock node" +LABEL release="1" +LABEL summary="Calico mock node for scale testing" +LABEL vendor="Tigera" +LABEL version=${GIT_VERSION} + +COPY --from=source / / + +ENTRYPOINT ["/usr/bin/mocknode"] diff --git a/test-tools/mocknode/Makefile b/test-tools/mocknode/Makefile new file mode 100644 index 00000000000..b6f59046d1c --- /dev/null +++ b/test-tools/mocknode/Makefile @@ -0,0 +1,95 @@ +include ../../metadata.mk + +PACKAGE_NAME?=github.com/projectcalico/calico/test-tools/mocknode + +IMAGE_NAME ?=mock-node +BUILD_IMAGES ?=$(IMAGE_NAME) + +############################################################################### +# Shortcut targets +############################################################################### +default: build image +test: ut ## Run the tests for the current platform/architecture + +############################################################################### +# Variables controlling the image +############################################################################### +CONTAINER_CREATED=.container.created-$(ARCH) +# Files that go into the image +BINARY=./bin/test-tools/mocknode-$(ARCH) +# Files to be built +SRC_FILES=$(shell find . -name '*.go' | grep -v vendor) \ + $(shell find ../../libcalico-go -name '*.go' | grep -v vendor)\ + $(shell find ../../typha -name '*.go' | grep -v vendor) + +############################################################################### +# Include ../../lib.Makefile +# Additions to EXTRA_DOCKER_ARGS need to happen before the include since +# that variable is evaluated when we declare DOCKER_RUN and siblings. +############################################################################### +include ../../lib.Makefile + +############################################################################### +## Clean enough that a new release build will be clean +############################################################################### +.PHONY: clean +clean: + find . -name '*.created' -exec rm -f {} + + find . -name '*.created-$(ARCH)' -exec rm -f {} + + find . -name '*.pyc' -exec rm -f {} + + rm -rf .go-pkg-cache bin + # Delete images that we built in this repo + -docker rmi $(IMAGE_NAME):latest-$(ARCH) + +############################################################################### +# Building the binary +############################################################################### + +LDFLAGS = -X main.VERSION=$(GIT_VERSION) + +.PHONY: build-all +## Build the binaries for all architectures and platforms +build-all: $(addprefix bin/mocknode-,$(VALIDARCHES)) + +.PHONY: build +## Build the binary for the current architecture and platform +build: $(BINARY) +bin/test-tools/mocknode-amd64: ARCH=amd64 +bin/test-tools/mocknode-%: $(SRC_FILES) + $(call build_binary, ./cmd/mocknode, $@) + +############################################################################### +# Building the image +############################################################################### +## Create the image for the current ARCH +image: $(IMAGE_NAME) + +## Create the images for all supported ARCHes +image-all: $(addprefix sub-image-,$(VALIDARCHES)) +sub-image-%: + $(MAKE) image ARCH=$* + +$(IMAGE_NAME): $(CONTAINER_CREATED) +$(CONTAINER_CREATED): Dockerfile $(BINARY) + $(DOCKER_BUILD) -t $(IMAGE_NAME):latest-$(ARCH) -f Dockerfile . + $(MAKE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest + touch $@ + +## Run the tests in a container. Useful for CI, Mac dev +ut: + mkdir -p report + $(DOCKER_RUN) $(CALICO_BUILD) /bin/bash -c "$(GIT_CONFIG_SSH) go test -v $(GOTEST_ARGS) ./..." + +fv: + echo "Currently has no FV tests." + +st: + echo "Currently has no STs." + +############################################################################### +# CI/CD +############################################################################### +.PHONY: ci +ci: clean image test static-checks +## Deploys images to registry +cd: image-all cd-common diff --git a/test-tools/mocknode/cmd/mocknode/main.go b/test-tools/mocknode/cmd/mocknode/main.go new file mode 100644 index 00000000000..dc9b28a5cee --- /dev/null +++ b/test-tools/mocknode/cmd/mocknode/main.go @@ -0,0 +1,239 @@ +// Copyright (c) 2025 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/libcalico-go/lib/backend/api" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" + "github.com/projectcalico/calico/libcalico-go/lib/logutils" + "github.com/projectcalico/calico/libcalico-go/lib/names" + "github.com/projectcalico/calico/typha/pkg/discovery" + "github.com/projectcalico/calico/typha/pkg/syncclient" + "github.com/projectcalico/calico/typha/pkg/syncproto" +) + +var counterLogErrors = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "mocknode_log_errors", + Help: "Number of errors encountered while logging.", +}) + +func init() { + prometheus.MustRegister( + counterLogErrors, + ) +} + +var VERSION string + +func newSyncerCallbacks(st syncproto.SyncerType) *syncerCallbacks { + return &syncerCallbacks{ + Type: st, + startTime: time.Now(), + logCtx: logrus.WithField("syncer", st), + cache: map[string]any{}, + } +} + +type syncerCallbacks struct { + Type syncproto.SyncerType + startTime time.Time + logCtx *logrus.Entry + + lock sync.Mutex + numUpdatesSeen int + cache map[string]any + status api.SyncStatus +} + +func (s *syncerCallbacks) OnStatusUpdated(status api.SyncStatus) { + s.lock.Lock() + defer s.lock.Unlock() + + s.logCtx.WithFields(logrus.Fields{ + "status": status, + "numKnownKVs": len(s.cache), + "timeSinceStart": time.Since(s.startTime), + }).Info("Status update from Typha") + s.status = status +} + +func (s *syncerCallbacks) OnUpdates(updates []api.Update) { + s.lock.Lock() + defer s.lock.Unlock() + + for _, u := range updates { + s.numUpdatesSeen++ + path, err := model.KeyToDefaultPath(u.Key) + if err != nil { + logrus.WithError(err).Panic("Failed to serialise key") + } + if u.KVPair.Value == nil { + delete(s.cache, path) + } else { + s.cache[path] = u.Value + } + } +} + +func (s *syncerCallbacks) LogStats() { + s.lock.Lock() + defer s.lock.Unlock() + + s.logCtx.WithFields(logrus.Fields{ + "status": s.status, + "numKnownKVs": len(s.cache), + "totalUpdates": s.numUpdatesSeen, + }).Info("Syncer stats") +} + +const ( + typhaNamespace = "calico-system" + typhaK8sServiceName = "calico-typha" + typhaCAFile = "/etc/pki/tls/certs/tigera-ca-bundle.crt" + typhaCertFile = "/node-certs/tls.crt" + typhaKeyFile = "/node-certs/tls.key" + typhaCN = "typha-server" + typhaURISAN = "" +) + +func main() { + defer func() { + logrus.WithField(logutils.FieldForceFlush, true).Warning("Exiting...") + }() + configureLogging() + logrus.WithFields(logrus.Fields{ + "version": VERSION, + }).Info("Mock Calico Node starting up") + + hostname, err := names.Hostname() + if err != nil { + logrus.WithError(err).Panic("Failed to get hostname") + } + + for _, st := range syncproto.AllSyncerTypes { + startTyphaClient(st, hostname) + } + logrus.Info("Started all clients.") + var cpuTimeUsed time.Duration + interval := 10 * time.Second + for { + time.Sleep(interval) + newTimeUsed := getMyCPUTime() + percent := float64(newTimeUsed-cpuTimeUsed) / float64(interval) + logrus.Infof("My CPU usage: %.2f%%", percent*100) + cpuTimeUsed = newTimeUsed + } +} + +func startTyphaClient(st syncproto.SyncerType, hostname string) { + logrus.Infof("Starting sycher of type: %v", st) + cbs := newSyncerCallbacks(st) + typhaDiscoverer := discovery.New( + discovery.WithInClusterKubeClient(), + discovery.WithKubeService(typhaNamespace, typhaK8sServiceName), + ) + _, err := typhaDiscoverer.LoadTyphaAddrs() + if err != nil { + logrus.WithError(err).Panic("Failed to discover Typha.") + } + client := syncclient.New(typhaDiscoverer, + VERSION, + hostname, + "", + cbs, + &syncclient.Options{ + KeyFile: typhaKeyFile, + CertFile: typhaCertFile, + CAFile: typhaCAFile, + ServerCN: typhaCN, + ServerURISAN: typhaURISAN, + SyncerType: st, + DebugDiscardKVUpdates: false, + }) + err = client.Start(context.Background()) + if err != nil { + logrus.WithError(err).Panic("Failed to start typha client.") + } + done := make(chan struct{}) + go func(st syncproto.SyncerType) { + defer close(done) + client.Finished.Wait() + logrus.WithField("syncer", st).Warning("Disconnected from Typha. (Will reconnect.)") + time.Sleep(2 * time.Second) + go startTyphaClient(st, hostname) + }(st) + go func() { + for { + select { + case <-time.After(10 * time.Second): + cbs.LogStats() + case <-done: + return + } + } + }() +} + +func getMyCPUTime() time.Duration { + rawStats, err := os.ReadFile("/proc/self/schedstat") + if err != nil { + logrus.WithError(err).Panic("Failed to read scheduler stats") + } + usedNanosStr := strings.SplitN(string(rawStats), " ", 2)[0] + usedNanosInt, err := strconv.ParseUint(usedNanosStr, 10, 64) + if err != nil { + logrus.WithError(err).Panic("Failed to read scheduler stats") + } + return time.Duration(usedNanosInt) +} + +func configureLogging() { + logLevel := logrus.InfoLevel + logrus.SetLevel(logLevel) + logutils.ConfigureFormatter("mocknode") + + // Disable logrus' default output, which only supports a single destination. We use the + // hook above to fan out logs to multiple destinations. + logrus.SetOutput(&logutils.NullWriter{}) + + // Since we push our logs onto a second thread via a channel, we can disable the + // Logger's built-in mutex completely. + logrus.StandardLogger().SetNoLock() + screenDest := logutils.NewStreamDestination( + logLevel, + os.Stdout, + make(chan logutils.QueuedLog, 1000), + false, + counterLogErrors, + ) + hook := logutils.NewBackgroundHook( + logutils.FilterLevels(logLevel), + logrus.PanicLevel, + []*logutils.Destination{screenDest}, + counterLogErrors, + ) + hook.Start() + logrus.AddHook(hook) +} diff --git a/test-tools/mocknode/hack/patch-cluster b/test-tools/mocknode/hack/patch-cluster new file mode 100755 index 00000000000..d1e93b30848 --- /dev/null +++ b/test-tools/mocknode/hack/patch-cluster @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +if [ -z "$REPO" ]; then + echo "Usage (after applying the manifest):" + echo " REPO= $0" + exit 1 +fi + +TAG=$(date -I)-$RANDOM +IMAGE=$REPO:$TAG + +echo "Building image..." +echo + +make image +echo +echo "Tagging image as $IMAGE" +echo +docker tag mock-node:latest-amd64 $IMAGE +docker push $IMAGE +kubectl set image -n calico-system deployment mock-calico-node mock-calico-node=$IMAGE +echo +echo "Set image $IMAGE" diff --git a/test-tools/mocknode/mock-node.yaml b/test-tools/mocknode/mock-node.yaml new file mode 100644 index 00000000000..56b63114358 --- /dev/null +++ b/test-tools/mocknode/mock-node.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mock-calico-node + namespace: calico-system +spec: + selector: + matchLabels: + k8s-app: mock-calico-node + template: + metadata: + labels: + k8s-app: mock-calico-node + spec: + containers: + - env: + - name: DATASTORE_TYPE + value: kubernetes + - name: NODENAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: FIPS_MODE_ENABLED + value: "false" + image: calico/mock-node:master + imagePullPolicy: IfNotPresent + name: mock-calico-node + volumeMounts: + - mountPath: /etc/pki/tls/certs + name: tigera-ca-bundle + readOnly: true + - mountPath: /etc/pki/tls/cert.pem + name: tigera-ca-bundle + readOnly: true + subPath: ca-bundle.crt + - mountPath: /node-certs + name: node-certs + readOnly: true + - mountPath: /calico-node-prometheus-server-tls + name: calico-node-prometheus-server-tls + readOnly: true + dnsPolicy: ClusterFirst + imagePullSecrets: + - name: tigera-pull-secret + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + serviceAccount: calico-node + serviceAccountName: calico-node + terminationGracePeriodSeconds: 5 + volumes: + - configMap: + defaultMode: 420 + name: tigera-ca-bundle + name: tigera-ca-bundle + - name: node-certs + secret: + defaultMode: 420 + secretName: node-certs + - name: calico-node-prometheus-server-tls + secret: + defaultMode: 420 + secretName: calico-node-prometheus-server-tls