diff --git a/Earthfile b/Earthfile index 55a1cad..7ce13b3 100644 --- a/Earthfile +++ b/Earthfile @@ -50,11 +50,12 @@ build-binary: build-docker: FROM alpine:${ALPINE_VERSION} COPY +build-binary/prom-aggregation-gateway . + ENV GIN_MODE=release ENTRYPOINT ["/prom-aggregation-gateway"] SAVE IMAGE --push ${image_name}:${version} continuous-deploy: - BUILD +release-helm + BUILD +build-helm build-binaries: FROM golang:${GOLANG_VERSION} @@ -162,21 +163,13 @@ test-helm: build-helm: FROM quay.io/helmpack/chart-releaser:v${CHART_RELEASER_VERSION} - WORKDIR /src - COPY . /src - - RUN cr --config .github/cr.yaml package charts/* - SAVE ARTIFACT .cr-release-packages/ AS LOCAL ./dist - -release-helm: - FROM quay.io/helmpack/chart-releaser:v${CHART_RELEASER_VERSION} - ARG token WORKDIR /src COPY . /src RUN cr --config .github/cr.yaml package charts/* + SAVE ARTIFACT .cr-release-packages/ AS LOCAL ./dist RUN mkdir -p .cr-index RUN git config --global user.email "opensource@zapier.com" diff --git a/Earthfile.md b/Earthfile.md index 49b3f36..902cb53 100644 --- a/Earthfile.md +++ b/Earthfile.md @@ -1,26 +1,17 @@ ```mermaid graph TD - lint-golang --> test - test-golang --> test - test-helm --> test - - build-binary --> build - build-docker --> build - build-helm --> build - - build-binary --> build-docker - release-binary --> release - release-binary -.create release.-> github - release-docker --> release - release-docker -.push package.-> github - - build-binary --> release-binary - build-docker --> release-docker - - release-helm --> continuous-deploy - release-helm -.push to gh-pages.-> github - - go-deps --> build-binary - go-deps --> lint-golang - go-deps --> test-golang + build --> build-docker + build --> build-helm + build-docker --> build-binary + continuous-deploy --> build-helm + lint-golang --> go-deps + test-golang --> go-deps + test --> ci-golang + release --> build-docker + release --> release-binaries + build-binary --> go-deps + ci-golang --> lint-golang + ci-golang --> test-golang + ci-helm --> test-helm + release-binaries --> build-binaries ``` diff --git a/aggregate.go b/aggregate.go index 0023ada..aed9449 100644 --- a/aggregate.go +++ b/aggregate.go @@ -78,6 +78,13 @@ func (ao *aggregateOptions) formatIgnoredLabels() { sort.Strings(ao.ignoredLabels) } +func (a *aggregate) Len() int { + a.familiesLock.RLock() + count := len(a.families) + a.familiesLock.RUnlock() + return count +} + // setFamilyOrGetExistingFamily either sets a new family or returns an existing family func (a *aggregate) setFamilyOrGetExistingFamily(familyName string, family *dto.MetricFamily) *metricFamily { a.familiesLock.Lock() @@ -126,8 +133,12 @@ func (a *aggregate) parseAndMerge(r io.Reader, job string) error { return err } + MetricCountByFamily.WithLabelValues(name).Set(float64(len(family.Metric))) + } + TotalFamiliesGauge.Set(float64(a.Len())) + return nil } @@ -140,9 +151,18 @@ func (a *aggregate) handleRender(c *gin.Context) { defer a.familiesLock.RUnlock() metricNames := []string{} - for name := range a.families { + metricTypeCounts := make(map[string]int) + for name, family := range a.families { metricNames = append(metricNames, name) + var typeName string + if family.Type == nil { + typeName = "unknown" + } else { + typeName = dto.MetricType_name[int32(*family.Type)] + } + metricTypeCounts[typeName]++ } + sort.Strings(metricNames) for _, name := range metricNames { @@ -151,6 +171,11 @@ func (a *aggregate) handleRender(c *gin.Context) { } } + MetricCountByType.Reset() + for typeName, count := range metricTypeCounts { + MetricCountByType.WithLabelValues(typeName).Set(float64(count)) + } + // TODO reset gauges } @@ -180,4 +205,6 @@ func (a *aggregate) handleInsert(c *gin.Context) { http.Error(c.Writer, err.Error(), http.StatusBadRequest) return } + + MetricPushes.WithLabelValues(job).Inc() } diff --git a/aggregate_test.go b/aggregate_test.go index 90f95ea..c1ded72 100644 --- a/aggregate_test.go +++ b/aggregate_test.go @@ -8,6 +8,8 @@ import ( "testing" "github.com/pmezard/go-difflib/difflib" + "github.com/prometheus/client_golang/prometheus" + metrics "github.com/slok/go-http-metrics/metrics/prometheus" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -172,7 +174,7 @@ func TestAggregate(t *testing.T) { } { t.Run(c.testName, func(t *testing.T) { agg := newAggregate(AddIgnoredLabels(c.ignoredLabels...)) - router := setupAPIRouter("*", agg) + router := setupAPIRouter("*", agg, metrics.Config{ Registry: prometheus.NewRegistry()}) err := agg.parseAndMerge(strings.NewReader(c.a), "test") require.NoError(t, err) diff --git a/main.go b/main.go index 8b15406..30a1bcf 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "syscall" "github.com/gin-gonic/gin" + metrics "github.com/slok/go-http-metrics/metrics/prometheus" ) var ( @@ -39,10 +40,14 @@ func runServers(corsDomain string, apiListen string, lifecycleListen string) { agg := newAggregate() - apiRouter := setupAPIRouter(corsDomain, agg) + promMetricsConfig := metrics.Config{ + Registry: promRegistry, + } + + apiRouter := setupAPIRouter(corsDomain, agg, promMetricsConfig) go runServer("api", apiRouter, apiListen) - lifecycleRouter := setupLifecycleRouter() + lifecycleRouter := setupLifecycleRouter(promRegistry) go runServer("lifecycle", lifecycleRouter, lifecycleListen) // Block until an interrupt or term signal is sent diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..fbf9ae5 --- /dev/null +++ b/metrics.go @@ -0,0 +1,56 @@ +package main + +import "github.com/prometheus/client_golang/prometheus" + +const MetricsNamespace = "prom_agg_gateway" + +var promRegistry = prometheus.NewRegistry() + +func init() { + promRegistry.MustRegister( + TotalFamiliesGauge, + MetricCountByFamily, + MetricPushes, + ) +} + +var TotalFamiliesGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: MetricsNamespace, + Name: "total_families", + Help: "Total number of metric families", + }, +) + +var MetricCountByFamily = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: MetricsNamespace, + Name: "metrics_by_family", + Help: "Metric count by family", + }, + []string{ + "family", + }, +) + +var MetricCountByType = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: MetricsNamespace, + Name: "metrics_by_type", + Help: "Metric count by type", + }, + []string{ + "metric_type", + }, +); + +var MetricPushes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: "metric_pushes", + Help: "Total number of metric push requests, per job", + }, + []string{ + "push_job", + }, +) diff --git a/privateRouter.go b/privateRouter.go index 5397c48..090be93 100644 --- a/privateRouter.go +++ b/privateRouter.go @@ -4,15 +4,21 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) -func setupLifecycleRouter() *gin.Engine { +func setupLifecycleRouter(promRegistry *prometheus.Registry) *gin.Engine { r := gin.New() + metricsHandler := promhttp.InstrumentMetricHandler( + promRegistry, + promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{}), + ) + r.GET("/healthy", handleHealthCheck) r.GET("/ready", handleHealthCheck) - r.GET("/metrics", convertHandler(promhttp.Handler())) + r.GET("/metrics", convertHandler(metricsHandler)) return r } diff --git a/publicRouter.go b/publicRouter.go index 1a30b02..d4e6358 100644 --- a/publicRouter.go +++ b/publicRouter.go @@ -3,13 +3,12 @@ package main import ( "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" - "github.com/prometheus/client_golang/prometheus" metrics "github.com/slok/go-http-metrics/metrics/prometheus" "github.com/slok/go-http-metrics/middleware" mGin "github.com/slok/go-http-metrics/middleware/gin" ) -func setupAPIRouter(corsDomain string, agg *aggregate) *gin.Engine { +func setupAPIRouter(corsDomain string, agg *aggregate, promConfig metrics.Config) *gin.Engine { corsConfig := cors.Config{} if corsDomain != "*" { corsConfig.AllowOrigins = []string{corsDomain} @@ -17,10 +16,8 @@ func setupAPIRouter(corsDomain string, agg *aggregate) *gin.Engine { corsConfig.AllowAllOrigins = true } - cfg := new(metrics.Config) - cfg.Registry = prometheus.NewRegistry() metricsMiddleware := middleware.New(middleware.Config{ - Recorder: metrics.NewRecorder(*cfg), + Recorder: metrics.NewRecorder(promConfig), }) r := gin.New() diff --git a/skaffold.yaml b/skaffold.yaml index 7d8b144..33f1db5 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -5,9 +5,9 @@ metadata: name: prom-aggregation-gateway build: artifacts: - - image: ghcr.io/zapier/prom-aggregation-gateway + - image: prom-aggregation-gateway custom: - buildCommand: earthly +build-docker --version=dev --image_name=ghcr.io/zapier/prom-aggregation-gateway + buildCommand: earthly +build-docker --version=$IMAGE_TAG --image_name=$IMAGE_REPO tagPolicy: customTemplate: template: dev @@ -18,6 +18,7 @@ manifests: chartPath: ./charts/prom-aggregation-gateway createNamespace: true setValues: - image.tag: dev + controller.image.repository: prom-aggregation-gateway + controller.image.tag: dev podMonitor.create: false serviceMonitor.create: false