Add initial flow aggregation code and API

.semaphore/push-images/goldmane.yml

@@ -0,0 +1,47 @@
+version: v1.0
+name: Publish goldmane images
+agent:
+  machine:
+    type: f1-standard-2
+    os_image: ubuntu2204
+
+execution_time_limit:
+  minutes: 60
+
+global_job_config:
+  env_vars:
+    - name: DEV_REGISTRIES
+      value: quay.io/calico docker.io/calico
+  secrets:
+    - name: docker
+    - name: quay-robot-calico+semaphoreci
+  prologue:
+    commands:
+      - checkout
+      # Semaphore is doing shallow clone on a commit without tags.
+      # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always)
+      - retry git fetch --unshallow
+      - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin
+      - echo $QUAY_TOKEN | docker login --username "$QUAY_USER" --password-stdin quay.io
+      - export BRANCH_NAME=$SEMAPHORE_GIT_BRANCH
+
+blocks:
+  - name: Publish goldmane images
+    dependencies: []
+    skip:
+      when: "branch !~ '.+'"
+    task:
+      jobs:
+        - name: Linux multi-arch
+          commands:
+            - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C goldmane cd CONFIRM=true; fi
+  - name: Publish goldmane multi-arch manifests
+    dependencies:
+      - Publish goldmane images
+    skip:
+      when: "branch !~ '.+'"
+    task:
+      jobs:
+        - name: Linux multi-arch manifests
+          commands:
+            - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C goldmane push-manifests-with-tag CONFIRM=true; fi

.semaphore/semaphore.yml.d/03-promotions.yml

@@ -51,6 +51,10 @@ promotions:
     pipeline_file: push-images/typha.yml
     auto_promote:
       when: "branch =~ 'master|release-'"
+  - name: Push Goldmane images
+    pipeline_file: push-images/goldmane.yml
+    auto_promote:
+      when: "branch =~ 'master|release-'"
   - name: Publish openstack packages
     pipeline_file: push-images/packaging.yaml
     auto_promote:

.semaphore/semaphore.yml.d/blocks/20-goldmane.yml

@@ -0,0 +1,24 @@
+- name: goldmane
+  run:
+    when: "${FORCE_RUN} or change_in(['/*', '/goldmane/'], {exclude: ['/**/.gitignore', '/**/README.md', '/**/LICENSE']})"
+  execution_time_limit:
+    minutes: 30
+  dependencies:
+    - Prerequisites
+  task:
+    prologue:
+      commands:
+        - cd goldmane
+    jobs:
+      - name: make ci
+        commands:
+          - ../.semaphore/run-and-monitor make-ci.log make ci
+      - name: Build binary
+        matrix:
+          - env_var: ARCH
+            values:
+              - arm64
+              - ppc64le
+              - s390x
+        commands:
+          - ../.semaphore/run-and-monitor image-$ARCH.log make build ARCH=$ARCH

goldmane/Makefile

@@ -0,0 +1,80 @@
+include ../metadata.mk
+
+PACKAGE_NAME = github.com/projectcalico/calico/goldmane
+IMAGE_BUILD_MARKER = goldmane_container-$(ARCH).created
+
+###############################################################################
+# include ../lib.Makefile
+#   Additions to EXTRA_DOCKER_ARGS need to happen before the include since
+#   that variable is evaluated when we declare DOCKER_RUN and siblings.
+###############################################################################
+include ../lib.Makefile
+
+# Configure variables used by ci/cd common targets from lib.Makefile.
+BUILD_IMAGES=goldmane
+
+.PHONY: image build
+image: $(IMAGE_BUILD_MARKER)
+build: bin/goldmane-$(ARCH)
+clean:
+	rm -rf bin
+	rm -f $(IMAGE_BUILD_MARKER)
+
+image-all: $(addprefix sub-image-,$(VALIDARCHES))
+sub-image-%:
+	$(MAKE) image ARCH=$*
+
+# Build goldmane image.
+calico/goldmane: $(IMAGE_BUILD_MARKER)
+$(IMAGE_BUILD_MARKER): bin/goldmane-$(ARCH)
+	$(DOCKER_BUILD) --build-arg TARGETARCH=$(ARCH) -t goldmane:latest-$(ARCH) -f docker/Dockerfile .
+	$(MAKE) retag-build-images-with-registries BUILD_IMAGES=$(BUILD_IMAGES) VALIDARCHES=$(ARCH) IMAGETAG=latest
+	touch $@
+
+bin/goldmane-$(ARCH): $(shell find . -name '*.go')
+	$(call build_binary, $(PACKAGE_NAME)/cmd/, $@)
+
+# Build the testserver image.
+.PHONY: testserver
+calico/flowgen: bin/flowgen
+	docker build -t calico/flowgen -f docker/flowgen/Dockerfile .
+
+bin/flowgen: $(shell find . -name '*.go')
+	$(call build_binary, $(PACKAGE_NAME)/cmd/flowgen, $@)
+
+# Update protobuf generation.
+protobuf proto/api.pb.go: proto/api.proto
+	$(DOCKER_RUN) -v $(CURDIR):/code \
+		$(CALICO_BUILD) sh -c 'protoc --proto_path=/code/proto --go_out=/code/proto --go-grpc_out=. --go_opt=paths=source_relative /code/proto/api.proto'
+	$(MAKE) fix-changed
+
+###############################################################################
+# UTs
+###############################################################################
+ci: static-checks ut
+ut:
+	$(DOCKER_GO_BUILD) go test ./... -cover -count 1
+
+###############################################################################
+# Release
+###############################################################################
+## Deploys images to registry
+cd: image-all cd-common
+
+release-build: .release-$(VERSION).created
+.release-$(VERSION).created:
+	$(MAKE) clean image-all RELEASE=true
+	$(MAKE) retag-build-images-with-registries RELEASE=true IMAGETAG=$(VERSION)
+	$(MAKE) retag-build-images-with-registries RELEASE=true IMAGETAG=latest
+
+release-verify: release-prereqs
+	@echo "Nothing to do"
+
+release-publish: release-prereqs release-verify .release-$(VERSION).published
+.release-$(VERSION).published:
+	$(MAKE) push-images-to-registries push-manifests IMAGETAG=$(VERSION) RELEASE=$(RELEASE) CONFIRM=$(CONFIRM)
+	$(MAKE) FIPS=true push-images-to-registries push-manifests IMAGETAG=$(VERSION)-fips RELEASE=$(RELEASE) CONFIRM=$(CONFIRM)
+
+	# Push Windows images.
+	$(MAKE) release-windows IMAGETAG=$(VERSION) CONFIRM=$(CONFIRM)
+	touch $@

goldmane/README.md

@@ -0,0 +1,11 @@
+## Goldmane
+
+Goldmane is a flow aggregation service. It provides a central, aggregated view of network flows in a Kubernetes cluster.
+
+Some key packages:
+
+- **proto/** defines the Flow structure and gRPC services provided by Goldmane.
+- **pkg/aggregator/** collects flow information from across the cluster and aggregates those flows across all nodes, building a cluster-wide view of network activity.
+- **pkg/collector/** provides a gRPC API that allows each Calico node instance to stream network flow information to a central location for aggregation and consumption.
+- **pkg/emitter/** periodically emits time-aggregated flow information to a configured endpoint.
+- **pkg/server/** allows for filtered querying of aggregated flow information.

goldmane/cmd/flowgen/main.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2025 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "github.com/projectcalico/calico/goldmane/pkg/flowgen"
+
+func main() {
+	flowgen.Start()
+}

goldmane/cmd/main.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2025 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "github.com/projectcalico/calico/goldmane/pkg/daemon"
+
+func main() {
+	daemon.Run()
+}

goldmane/config/testserver-daemonset.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: flow-log-generator
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      name: flow-log-generator
+  template:
+    metadata:
+      labels:
+        name: flow-log-generator
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      containers:
+      - name: generator
+        image: caseydavenport/flowgen
+        env:
+          - name: SERVER
+            value: "goldmane.calico-system:443"

goldmane/docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM scratch
+
+ARG TARGETARCH
+COPY ./bin/goldmane-${TARGETARCH} /goldmane
+
+CMD ["/goldmane"]

goldmane/docker/flowgen/Dockerfile

@@ -0,0 +1,5 @@
+FROM scratch
+
+COPY ./bin/flowgen /flowgen
+
+CMD ["/flowgen"]