From 7f97eeef45bbe0ad9540343baee0fe14e744f499 Mon Sep 17 00:00:00 2001 From: Adam Hamsik Date: Mon, 13 May 2024 14:11:47 +0200 Subject: [PATCH] Go 1.22 update (#111) * Bump GO to 1.22 * Extending firewall metrics with rule names * Adding account specific rule to firewall metrics * Add Cloudflare Logpush metrics for account and zone * Selector is included in labels Bump chart version * Add Account Label to Metrics This commit adds the Account label to the metrics label list. The Account label provides additional context to the metrics, allowing for better categorization and analysis of data. This change enhances the accuracy and granularity of the metrics collected. - Added Account label to metrics - Improved data categorization and analysis Signed-off-by: Denys Lemeshko Fix missing label * Add account to labels on logpush events, bump golangci fix golang-ci lint action some golangci-lint fixes * Use cobra and viper as maintained alternatives to namsral/flag library. Add basic tests just make sure you provide .env file with right variables. --------- Co-authored-by: Oleksii Serhiienko Co-authored-by: Bhushan Thakur Co-authored-by: Mark Pierce Co-authored-by: Denys Lemeshko --- .editorconfig | 19 ++ .github/dependabot.yml | 5 + .github/workflows/docker-master.yaml | 2 +- .github/workflows/docker-release.yaml | 1 - .github/workflows/go-build.yml | 2 +- .github/workflows/golangci-lint.yml | 7 +- .github/workflows/lint-test.yaml | 2 +- .github/workflows/release.yaml | 15 +- .gitignore | 2 + .golangci.yaml | 22 +- .pre-commit-config.yaml | 13 +- Dockerfile | 2 +- Makefile | 10 +- README.md | 4 + chart_schema.yaml | 37 +++ charts/cloudflare-exporter/Chart.yaml | 2 +- .../templates/_helpers.tpl | 5 + .../templates/deployment.yaml | 2 +- cloudflare.go | 239 ++++++++++++++-- ct.yaml | 11 + go.mod | 55 ++-- go.sum | 88 ++++++ lintconf.yaml | 42 +++ main.go | 158 ++++++++--- prometheus.go | 267 ++++++++++++------ run_e2e.sh | 22 ++ tests/basic_tests.yml | 25 ++ 27 files changed, 860 insertions(+), 199 deletions(-) create mode 100644 .editorconfig create mode 100644 chart_schema.yaml create mode 100644 ct.yaml create mode 100644 lintconf.yaml create mode 100755 run_e2e.sh create mode 100644 tests/basic_tests.yml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..a785002 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +# EditorConfig is awesome: http://EditorConfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 + +[*.{yml, yaml}] +indent_size = 2 + +[*.go] +indent_style = tab + +[Makefile] +indent_style = tab diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8c39cbc..68a5549 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,8 @@ updates: interval: "weekly" reviewers: - "Shopify/infrasec" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index c8b10f8..210adf5 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -53,4 +53,4 @@ jobs: push: true - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml index 96f09d8..811cde3 100644 --- a/.github/workflows/docker-release.yaml +++ b/.github/workflows/docker-release.yaml @@ -55,4 +55,3 @@ jobs: - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} - diff --git a/.github/workflows/go-build.yml b/.github/workflows/go-build.yml index 68415c0..ee4399f 100644 --- a/.github/workflows/go-build.yml +++ b/.github/workflows/go-build.yml @@ -23,4 +23,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - args: "./dist/*" \ No newline at end of file + args: "./dist/*" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 3409fa7..83cf9a6 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -11,7 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-go@v5 + with: + go-version: "1.22" - name: golangci-lint - uses: golangci/golangci-lint-action@v2 # https://github.com/marketplace/actions/run-golangci-lint + uses: golangci/golangci-lint-action@v6 # https://github.com/marketplace/actions/run-golangci-lint with: - version: v1.43 + version: latest diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index f0f7da3..cc4a032 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -21,7 +21,7 @@ jobs: python-version: 3.7 - name: Set up chart-testing - uses: helm/chart-testing-action@v2.0.1 + uses: helm/chart-testing-action@v2.6.1 - name: Run chart-testing (list-changed) id: list-changed diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5d8e328..79a1c15 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,11 +20,16 @@ jobs: git config user.email "$GITHUB_ACTOR@users.noreply.github.com" # See https://github.com/helm/chart-releaser-action/issues/6 + # - name: Install Helm + # run: | + # curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + # chmod 700 get_helm.sh + # ./get_helm.sh + - name: Install Helm - run: | - curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 get_helm.sh - ./get_helm.sh + uses: azure/setup-helm@v4.2.0 + with: + version: 3.14.0 - name: Add dependency chart repos run: | @@ -32,6 +37,6 @@ jobs: helm repo add incubator https://charts.helm.sh/incubator - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.0.0 + uses: helm/chart-releaser-action@v1.6.0 env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index 58088a4..b53ed42 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ vendor/ *.swp .idea/ +.env +venom* diff --git a/.golangci.yaml b/.golangci.yaml index 2e14660..da397a9 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,9 +1,25 @@ linters: + disable: + - errcheck enable: - - megacheck - - gofmt + #- gosimple + - staticcheck - govet - - revive # replacement for golint + + - asciicheck + - errorlint + - gofmt + - goimports + - gosec + # - gocritic + - importas + # - prealloc + - revive + - misspell + - stylecheck + - unconvert + - unused + - whitespace linters-settings: gofmt: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d5d2df..34c4fd2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,19 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.6.0 hooks: - - id: trailing-whitespace - id: check-merge-conflict + - id: trailing-whitespace - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: destroyed-symlinks + - id: detect-private-key + - id: check-ast + - id: check-case-conflict + - id: debug-statements - repo: https://github.com/golangci/golangci-lint - rev: v1.43.0 + rev: v1.58.0 hooks: - id: golangci-lint diff --git a/Dockerfile b/Dockerfile index 177734f..479f7ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ COPY go.mod go.mod COPY go.sum go.sum RUN go get -d -v -RUN CGO_ENABLED=0 GOOS=linux go build -o cloudflare_exporter . +RUN CGO_ENABLED=0 GOOS=linux go build --ldflags '-w -s -extldflags "-static"' -o cloudflare_exporter . FROM alpine:3.18 diff --git a/Makefile b/Makefile index 6a48adf..f4333c3 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,9 @@ .PHONY: build -build: - CGO_ENABLED=0 go build -o cloudflare_exporter . +build: lint + CGO_ENABLED=0 go build --ldflags '-w -s -extldflags "-static"' -o cloudflare_exporter . +lint: + golangci-lint run +clean: + rm cloudflare_exporter venom*.log basic_tests.* pprof_cpu* +test: + ./run_e2e.sh diff --git a/README.md b/README.md index dd7d326..8a48a41 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ Required authentication scopes: - `Account.Account Analytics:Read` is required for Worker metrics - `Account Settings:Read` is required for Worker metrics (for listing accessible accounts, scraping all available Workers included in authentication scope) +- `Firewall Services:Read` is required to fetch zone rule name for `cloudflare_zone_firewall_events_count` metric +- `Account. Account Rulesets:Read` is required to fetch account rule name for `cloudflare_zone_firewall_events_count` metric To authenticate this way, only set `CF_API_TOKEN` (omit `CF_API_EMAIL` and `CF_API_KEY`) @@ -100,6 +102,8 @@ Note: `ZONE_` configuration is not supported as flag. # HELP cloudflare_zone_uniques_total Uniques per zone # HELP cloudflare_zone_pool_health_status Reports the health of a pool, 1 for healthy, 0 for unhealthy # HELP cloudflare_zone_pool_requests_total Requests per pool +# HELP cloudflare_logpush_failed_jobs_account_count Number of failed logpush jobs on the account level +# HELP cloudflare_logpush_failed_jobs_zone_count Number of failed logpush jobs on the zone level ``` ## Helm chart repository diff --git a/chart_schema.yaml b/chart_schema.yaml new file mode 100644 index 0000000..2a26d9b --- /dev/null +++ b/chart_schema.yaml @@ -0,0 +1,37 @@ +name: str() +home: str(required=False) +version: str() +apiVersion: str() +appVersion: any(str(), num(), required=False) +description: str(required=False) +keywords: list(str(), required=False) +sources: list(str(), required=False) +maintainers: list(include('maintainer'), required=False) +dependencies: list(include('dependency'), required=False) +icon: str(required=False) +engine: str(required=False) +condition: str(required=False) +tags: str(required=False) +deprecated: bool(required=False) +kubeVersion: str(required=False) +annotations: map(str(), str(), required=False) +type: str(required=False) +--- +maintainer: + name: str() + email: str(required=False) + url: str(required=False) +--- +dependency: + name: str() + version: str() + repository: str(required=False) + condition: str(required=False) + tags: list(str(), required=False) + enabled: bool(required=False) + import-values: any(list(str()), list(include('import-value')), required=False) + alias: str(required=False) +--- +import-value: + child: str() + parent: str() diff --git a/charts/cloudflare-exporter/Chart.yaml b/charts/cloudflare-exporter/Chart.yaml index 9bbbebe..df55fcb 100644 --- a/charts/cloudflare-exporter/Chart.yaml +++ b/charts/cloudflare-exporter/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: cloudflare-exporter -version: 0.1.9 +version: 0.2.0 appVersion: 0.0.9 home: https://github.com/lablabs/cloudflare-exporter description: A Helm chart for cloudflare exporter diff --git a/charts/cloudflare-exporter/templates/_helpers.tpl b/charts/cloudflare-exporter/templates/_helpers.tpl index 7b49003..95b6bb2 100644 --- a/charts/cloudflare-exporter/templates/_helpers.tpl +++ b/charts/cloudflare-exporter/templates/_helpers.tpl @@ -36,6 +36,11 @@ Common labels {{- define "cloudflare-exporter.labels" -}} helm.sh/chart: {{ include "cloudflare-exporter.chart" . }} {{ include "cloudflare-exporter.selectorLabels" . }} +{{- if .Values.labels }} +{{- range $key, $value := .Values.labels }} +{{ $key }}: "{{ $value }}" +{{- end }} +{{- end }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} diff --git a/charts/cloudflare-exporter/templates/deployment.yaml b/charts/cloudflare-exporter/templates/deployment.yaml index 331400c..7a743b1 100644 --- a/charts/cloudflare-exporter/templates/deployment.yaml +++ b/charts/cloudflare-exporter/templates/deployment.yaml @@ -19,7 +19,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "cloudflare-exporter.selectorLabels" . | nindent 8 }} + {{- include "cloudflare-exporter.labels" . | nindent 8 }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: diff --git a/cloudflare.go b/cloudflare.go index f1a3037..4b96d94 100644 --- a/cloudflare.go +++ b/cloudflare.go @@ -2,11 +2,13 @@ package main import ( "context" + "strings" "time" - "github.com/cloudflare/cloudflare-go" + cloudflare "github.com/cloudflare/cloudflare-go" "github.com/machinebox/graphql" log "github.com/sirupsen/logrus" + "github.com/spf13/viper" ) var ( @@ -37,6 +39,32 @@ type cloudflareResponseLb struct { } `json:"viewer"` } +type cloudflareResponseLogpushAccount struct { + Viewer struct { + Accounts []logpushResponse `json:"accounts"` + } `json:"viewer"` +} + +type cloudflareResponseLogpushZone struct { + Viewer struct { + Zones []logpushResponse `json:"zones"` + } `json:"viewer"` +} + +type logpushResponse struct { + LogpushHealthAdaptiveGroups []struct { + Count uint64 `json:"count"` + + Dimensions struct { + Datetime string `json:"datetime"` + DestinationType string `json:"destinationType"` + JobID int `json:"jobId"` + Status int `json:"status"` + Final int `json:"final"` + } + } `json:"logpushHealthAdaptiveGroups"` +} + type accountResp struct { WorkersInvocationsAdaptive []struct { Dimensions struct { @@ -142,6 +170,7 @@ type zoneResp struct { Dimensions struct { Action string `json:"action"` Source string `json:"source"` + RuleID string `json:"ruleId"` ClientCountryName string `json:"clientCountryName"` ClientRequestHTTPHost string `json:"clientRequestHTTPHost"` } `json:"dimensions"` @@ -224,10 +253,10 @@ type lbResp struct { func fetchZones() []cloudflare.Zone { var api *cloudflare.API var err error - if len(cfgCfAPIToken) > 0 { - api, err = cloudflare.NewWithAPIToken(cfgCfAPIToken) + if len(viper.GetString("cf_api_token")) > 0 { + api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token")) } else { - api, err = cloudflare.New(cfgCfAPIKey, cfgCfAPIEmail) + api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email")) } if err != nil { log.Fatal(err) @@ -242,13 +271,57 @@ func fetchZones() []cloudflare.Zone { return z } +func fetchFirewallRules(zoneID string) map[string]string { + var api *cloudflare.API + var err error + if len(viper.GetString("cf_api_token")) > 0 { + api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token")) + } else { + api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email")) + } + if err != nil { + log.Fatal(err) + } + + ctx := context.Background() + listOfRules, _, err := api.FirewallRules(ctx, + cloudflare.ZoneIdentifier(zoneID), + cloudflare.FirewallRuleListParams{}) + if err != nil { + log.Fatal(err) + } + firewallRulesMap := make(map[string]string) + + for _, rule := range listOfRules { + firewallRulesMap[rule.ID] = rule.Description + } + + listOfRulesets, err := api.ListRulesets(ctx, cloudflare.ZoneIdentifier(zoneID), cloudflare.ListRulesetsParams{}) + if err != nil { + log.Fatal(err) + } + for _, rulesetDesc := range listOfRulesets { + if rulesetDesc.Phase == "http_request_firewall_managed" { + ruleset, err := api.GetRuleset(ctx, cloudflare.ZoneIdentifier(zoneID), rulesetDesc.ID) + if err != nil { + log.Fatal(err) + } + for _, rule := range ruleset.Rules { + firewallRulesMap[rule.ID] = rule.Description + } + } + } + + return firewallRulesMap +} + func fetchAccounts() []cloudflare.Account { var api *cloudflare.API var err error - if len(cfgCfAPIToken) > 0 { - api, err = cloudflare.NewWithAPIToken(cfgCfAPIToken) + if len(viper.GetString("cf_api_token")) > 0 { + api, err = cloudflare.NewWithAPIToken(viper.GetString("cf_api_token")) } else { - api, err = cloudflare.New(cfgCfAPIKey, cfgCfAPIEmail) + api, err = cloudflare.New(viper.GetString("cf_api_key"), viper.GetString("cf_api_email")) } if err != nil { log.Fatal(err) @@ -264,7 +337,7 @@ func fetchAccounts() []cloudflare.Account { } func fetchZoneTotals(zoneIDs []string) (*cloudflareResponse, error) { - now := time.Now().Add(-time.Duration(cfgScrapeDelay) * time.Second).UTC() + now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC() s := 60 * time.Second now = now.Truncate(s) now1mAgo := now.Add(-60 * time.Second) @@ -332,6 +405,7 @@ query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) { dimensions { action source + ruleId clientRequestHTTPHost clientCountryName } @@ -365,11 +439,11 @@ query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) { } } `) - if len(cfgCfAPIToken) > 0 { - request.Header.Set("Authorization", "Bearer "+cfgCfAPIToken) + if len(viper.GetString("cf_api_token")) > 0 { + request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token")) } else { - request.Header.Set("X-AUTH-EMAIL", cfgCfAPIEmail) - request.Header.Set("X-AUTH-KEY", cfgCfAPIKey) + request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email")) + request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key")) } request.Var("limit", 9999) request.Var("maxtime", now) @@ -389,7 +463,7 @@ query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) { } func fetchColoTotals(zoneIDs []string) (*cloudflareResponseColo, error) { - now := time.Now().Add(-time.Duration(cfgScrapeDelay) * time.Second).UTC() + now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC() s := 60 * time.Second now = now.Truncate(s) now1mAgo := now.Add(-60 * time.Second) @@ -421,11 +495,11 @@ func fetchColoTotals(zoneIDs []string) (*cloudflareResponseColo, error) { } } `) - if len(cfgCfAPIToken) > 0 { - request.Header.Set("Authorization", "Bearer "+cfgCfAPIToken) + if len(viper.GetString("cf_api_token")) > 0 { + request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token")) } else { - request.Header.Set("X-AUTH-EMAIL", cfgCfAPIEmail) - request.Header.Set("X-AUTH-KEY", cfgCfAPIKey) + request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email")) + request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key")) } request.Var("limit", 9999) request.Var("maxtime", now) @@ -444,7 +518,7 @@ func fetchColoTotals(zoneIDs []string) (*cloudflareResponseColo, error) { } func fetchWorkerTotals(accountID string) (*cloudflareResponseAccts, error) { - now := time.Now().Add(-time.Duration(cfgScrapeDelay) * time.Second).UTC() + now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC() s := 60 * time.Second now = now.Truncate(s) now1mAgo := now.Add(-60 * time.Second) @@ -481,11 +555,11 @@ func fetchWorkerTotals(accountID string) (*cloudflareResponseAccts, error) { } } `) - if len(cfgCfAPIToken) > 0 { - request.Header.Set("Authorization", "Bearer "+cfgCfAPIToken) + if len(viper.GetString("cf_api_token")) > 0 { + request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token")) } else { - request.Header.Set("X-AUTH-EMAIL", cfgCfAPIEmail) - request.Header.Set("X-AUTH-KEY", cfgCfAPIKey) + request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email")) + request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key")) } request.Var("limit", 9999) request.Var("maxtime", now) @@ -504,7 +578,7 @@ func fetchWorkerTotals(accountID string) (*cloudflareResponseAccts, error) { } func fetchLoadBalancerTotals(zoneIDs []string) (*cloudflareResponseLb, error) { - now := time.Now().Add(-time.Duration(cfgScrapeDelay) * time.Second).UTC() + now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC() s := 60 * time.Second now = now.Truncate(s) now1mAgo := now.Add(-60 * time.Second) @@ -558,11 +632,11 @@ func fetchLoadBalancerTotals(zoneIDs []string) (*cloudflareResponseLb, error) { } } `) - if len(cfgCfAPIToken) > 0 { - request.Header.Set("Authorization", "Bearer "+cfgCfAPIToken) + if len(viper.GetString("cf_api_token")) > 0 { + request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token")) } else { - request.Header.Set("X-AUTH-EMAIL", cfgCfAPIEmail) - request.Header.Set("X-AUTH-KEY", cfgCfAPIKey) + request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email")) + request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key")) } request.Var("limit", 9999) request.Var("maxtime", now) @@ -579,14 +653,119 @@ func fetchLoadBalancerTotals(zoneIDs []string) (*cloudflareResponseLb, error) { return &resp, nil } -func findZoneName(zones []cloudflare.Zone, ID string) string { +func fetchLogpushAccount(accountID string) (*cloudflareResponseLogpushAccount, error) { + now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC() + s := 60 * time.Second + now = now.Truncate(s) + now1mAgo := now.Add(-60 * time.Second) + + request := graphql.NewRequest(`query($accountID: String!, $limit: Int!, $mintime: Time!, $maxtime: Time!) { + viewer { + accounts(filter: {accountTag : $accountID }) { + logpushHealthAdaptiveGroups( + filter: { + datetime_geq: $mintime + datetime_lt: $maxtime + status_neq: 200 + } + limit: $limit + ) { + count + dimensions { + jobId + status + destinationType + datetime + final + } + } + } + } + }`) + + if len(viper.GetString("cf_api_token")) > 0 { + request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token")) + } else { + request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email")) + request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key")) + } + + request.Var("accountID", accountID) + request.Var("limit", 9999) + request.Var("maxtime", now) + request.Var("mintime", now1mAgo) + + ctx := context.Background() + graphqlClient := graphql.NewClient(cfGraphQLEndpoint) + var resp cloudflareResponseLogpushAccount + if err := graphqlClient.Run(ctx, request, &resp); err != nil { + log.Error(err) + return nil, err + } + return &resp, nil +} + +func fetchLogpushZone(zoneIDs []string) (*cloudflareResponseLogpushZone, error) { + now := time.Now().Add(-time.Duration(viper.GetInt("scrape_delay")) * time.Second).UTC() + s := 60 * time.Second + now = now.Truncate(s) + now1mAgo := now.Add(-60 * time.Second) + + request := graphql.NewRequest(`query($zoneIDs: String!, $limit: Int!, $mintime: Time!, $maxtime: Time!) { + viewer { + zones(filter: {zoneTag_in : $zoneIDs }) { + logpushHealthAdaptiveGroups( + filter: { + datetime_geq: $mintime + datetime_lt: $maxtime + status_neq: 200 + } + limit: $limit + ) { + count + dimensions { + jobId + status + destinationType + datetime + final + } + } + } + } + }`) + + if len(viper.GetString("cf_api_token")) > 0 { + request.Header.Set("Authorization", "Bearer "+viper.GetString("cf_api_token")) + } else { + request.Header.Set("X-AUTH-EMAIL", viper.GetString("cf_api_email")) + request.Header.Set("X-AUTH-KEY", viper.GetString("cf_api_key")) + } + + request.Var("zoneIDs", zoneIDs) + request.Var("limit", 9999) + request.Var("maxtime", now) + request.Var("mintime", now1mAgo) + + ctx := context.Background() + graphqlClient := graphql.NewClient(cfGraphQLEndpoint) + var resp cloudflareResponseLogpushZone + if err := graphqlClient.Run(ctx, request, &resp); err != nil { + log.Error(err) + return nil, err + } + + return &resp, nil +} + +func findZoneAccountName(zones []cloudflare.Zone, ID string) (string, string) { for _, z := range zones { if z.ID == ID { - return z.Name + return z.Name, strings.ToLower(strings.ReplaceAll(z.Account.Name, " ", "-")) } } - return "" + return "", "" } func extractZoneIDs(zones []cloudflare.Zone) []string { diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 0000000..7713e97 --- /dev/null +++ b/ct.yaml @@ -0,0 +1,11 @@ +remote: origin +target-branch: master +chart-dirs: + - charts +chart-repos: + - bitnami=https://charts.bitnami.com/bitnami +helm-extra-args: --timeout 600s +#debug: true +#ValidateChartSchema: false +#LintConf: charts/cloudflare-exporter/lintconf.yaml +#ChartYamlSchema: charts/cloudflare-exporter/chart_schema.yaml diff --git a/go.mod b/go.mod index dcd4078..855ca7a 100644 --- a/go.mod +++ b/go.mod @@ -1,34 +1,55 @@ module github.com/lablabs/cloudflare-exporter -go 1.19 +go 1.22 require ( - github.com/biter777/countries v1.5.6 - github.com/cloudflare/cloudflare-go v0.49.0 + github.com/biter777/countries v1.7.4 + github.com/cloudflare/cloudflare-go v0.94.0 github.com/machinebox/graphql v0.2.2 github.com/namsral/flag v1.7.4-pre github.com/nelkinda/health-go v0.0.1 - github.com/prometheus/client_golang v1.13.0 - github.com/sirupsen/logrus v1.9.0 + github.com/prometheus/client_golang v1.19.0 + github.com/sirupsen/logrus v1.9.3 ) require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.5 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/matryer/is v1.4.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/nelkinda/http-go v0.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect - golang.org/x/sys v0.0.0-20220913175220-63ea55921009 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect - google.golang.org/protobuf v1.28.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect + github.com/prometheus/procfs v0.14.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.18.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4153f72..076b56f 100644 --- a/go.sum +++ b/go.sum @@ -47,10 +47,14 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/biter777/countries v1.5.6 h1:YdvI0OYZR4gmI8BO+LrAuKmoZgiv4RrMdGBj6iORfn8= github.com/biter777/countries v1.5.6/go.mod h1:1HSpZ526mYqKJcpT5Ti1kcGQ0L0SrXWIaptUWjFfv2E= +github.com/biter777/countries v1.7.4 h1:590JZkxrv+/JBTAw2GHULx9l7vUZxz2HWMZ9HkruiOc= +github.com/biter777/countries v1.7.4/go.mod h1:1HSpZ526mYqKJcpT5Ti1kcGQ0L0SrXWIaptUWjFfv2E= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/christianhujer/assert v0.0.2 h1:j+nZAzx9h4su7L8hw0NGdd93J1BtjwnTyp8jd4wiRXs= github.com/christianhujer/assert v0.0.2/go.mod h1:yszWvVhUvkosrPxaPy9FqnC6XH16zFoFs8+hPXKs4ZQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -59,7 +63,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.49.0 h1:KqJYk/YQ5ZhmyYz1oa4kGDskfF1gVuZfqesaJ/XDLto= github.com/cloudflare/cloudflare-go v0.49.0/go.mod h1:h0QgcIZ3qEXwFiwfBO8sQxjVdYsLX+PfD7NFEnANaKg= +github.com/cloudflare/cloudflare-go v0.94.0 h1:WADmVhCdnn1A9sm5NU08by49Vbh4Lj/JBgTWTr7q7Qc= +github.com/cloudflare/cloudflare-go v0.94.0/go.mod h1:N1u1cLZ4lG6NeezGOWi7P6aq1DK2iVYg9ze7GZbUmZE= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w= github.com/cucumber/godog v0.9.0/go.mod h1:roWCHkpeK6UTOyIRRl7IR+fgfBeZ4vZR7OSq2J/NbM4= @@ -68,11 +75,15 @@ github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rX github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -109,6 +120,8 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -140,6 +153,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -176,12 +191,18 @@ github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrj github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -208,15 +229,22 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo= github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -232,13 +260,17 @@ github.com/nelkinda/health-go v0.0.1/go.mod h1:oNvFVrveHIH/xPW5DqjFfdtlyhLXHFmNz github.com/nelkinda/http-go v0.0.1 h1:RL3RttZzzs/kzQaVPmtv5dxMaWValqCqGNjCXyjPI1k= github.com/nelkinda/http-go v0.0.1/go.mod h1:DxPiZGVufTVSeO63nmVR5QO01TmSC0HHtEIZTHL5QEk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -246,17 +278,23 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -264,9 +302,16 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= +github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -274,16 +319,40 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= @@ -296,6 +365,10 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -315,6 +388,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -369,6 +444,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -430,6 +507,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220913175220-63ea55921009 h1:PuvuRMeLWqsf/ZdT1UUZz0syhioyv1mzuFZsXs4fvhw= golang.org/x/sys v0.0.0-20220913175220-63ea55921009/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -440,11 +519,15 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -571,12 +654,16 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -586,6 +673,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/lintconf.yaml b/lintconf.yaml new file mode 100644 index 0000000..90f48c8 --- /dev/null +++ b/lintconf.yaml @@ -0,0 +1,42 @@ +--- +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + colons: + max-spaces-before: 0 + max-spaces-after: 1 + commas: + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + require-starting-space: true + min-spaces-from-content: 2 + document-end: disable + document-start: disable # No --- to start a file + empty-lines: + max: 2 + max-start: 0 + max-end: 0 + hyphens: + max-spaces-after: 1 + indentation: + spaces: consistent + indent-sequences: whatever # - list indentation will handle both indentation and without + check-multi-line-strings: false + key-duplicates: enable + line-length: disable # Lines can be any length + new-line-at-end-of-file: enable + new-lines: + type: unix + trailing-spaces: enable + truthy: + level: warning diff --git a/main.go b/main.go index d07c180..625cfec 100644 --- a/main.go +++ b/main.go @@ -7,32 +7,34 @@ import ( "sync" "time" - "github.com/cloudflare/cloudflare-go" - "github.com/namsral/flag" "github.com/nelkinda/health-go" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + cloudflare "github.com/cloudflare/cloudflare-go" log "github.com/sirupsen/logrus" ) -var ( - cfgListen = ":8080" - cfgCfAPIKey = "" - cfgCfAPIEmail = "" - cfgCfAPIToken = "" - cfgMetricsPath = "/metrics" - cfgZones = "" - cfgExcludeZones = "" - cfgScrapeDelay = 300 - cfgFreeTier = false - cfgBatchSize = 10 - cfgMetricsDenylist = "" -) +// var ( +// cfgListen = ":8080" +// cfgCfAPIKey = "" +// cfgCfAPIEmail = "" +// cfgCfAPIToken = "" +// cfgMetricsPath = "/metrics" +// cfgZones = "" +// cfgExcludeZones = "" +// cfgScrapeDelay = 300 +// cfgFreeTier = false +// cfgBatchSize = 10 +// cfgMetricsDenylist = "" +// ) func getTargetZones() []string { var zoneIDs []string - if len(cfgZones) > 0 { - zoneIDs = strings.Split(cfgZones, ",") + if len(viper.GetString("cfg_zones")) > 0 { + zoneIDs = strings.Split(viper.GetString("cfg_zones"), ",") } else { // deprecated for _, e := range os.Environ() { @@ -48,8 +50,8 @@ func getTargetZones() []string { func getExcludedZones() []string { var zoneIDs []string - if len(cfgExcludeZones) > 0 { - zoneIDs = strings.Split(cfgExcludeZones, ",") + if len(viper.GetString("cf_exclude_zones")) > 0 { + zoneIDs = strings.Split(viper.GetString("cf_exclude_zones"), ",") } return zoneIDs } @@ -108,13 +110,14 @@ func fetchMetrics() { for _, a := range accounts { go fetchWorkerAnalytics(a, &wg) + go fetchLogpushAnalyticsForAccount(a, &wg) } // Make requests in groups of cfgBatchSize to avoid rate limit // 10 is the maximum amount of zones you can request at once for len(filteredZones) > 0 { - sliceLength := cfgBatchSize - if len(filteredZones) < cfgBatchSize { + sliceLength := viper.GetInt("cf_batch_size") + if len(filteredZones) < viper.GetInt("cf_batch_size") { sliceLength = len(filteredZones) } @@ -124,28 +127,28 @@ func fetchMetrics() { go fetchZoneAnalytics(targetZones, &wg) go fetchZoneColocationAnalytics(targetZones, &wg) go fetchLoadBalancerAnalytics(targetZones, &wg) + go fetchLogpushAnalyticsForZone(targetZones, &wg) } wg.Wait() } -func main() { - flag.StringVar(&cfgListen, "listen", cfgListen, "listen on addr:port ( default :8080), omit addr to listen on all interfaces") - flag.StringVar(&cfgMetricsPath, "metrics_path", cfgMetricsPath, "path for metrics, default /metrics") - flag.StringVar(&cfgCfAPIKey, "cf_api_key", cfgCfAPIKey, "cloudflare api key, works with api_email flag") - flag.StringVar(&cfgCfAPIEmail, "cf_api_email", cfgCfAPIEmail, "cloudflare api email, works with api_key flag") - flag.StringVar(&cfgCfAPIToken, "cf_api_token", cfgCfAPIToken, "cloudflare api token (preferred)") - flag.StringVar(&cfgZones, "cf_zones", cfgZones, "cloudflare zones to export, comma delimited list") - flag.StringVar(&cfgExcludeZones, "cf_exclude_zones", cfgExcludeZones, "cloudflare zones to exclude, comma delimited list") - flag.IntVar(&cfgScrapeDelay, "scrape_delay", cfgScrapeDelay, "scrape delay in seconds, defaults to 300") - flag.IntVar(&cfgBatchSize, "cf_batch_size", cfgBatchSize, "cloudflare zones batch size (1-10), defaults to 10") - flag.BoolVar(&cfgFreeTier, "free_tier", cfgFreeTier, "scrape only metrics included in free plan") - flag.StringVar(&cfgMetricsDenylist, "metrics_denylist", cfgMetricsDenylist, "metrics to not expose, comma delimited list") - flag.Parse() - if !(len(cfgCfAPIToken) > 0 || (len(cfgCfAPIEmail) > 0 && len(cfgCfAPIKey) > 0)) { +func runExpoter() { + // fmt.Println(" :", viper.GetString("cf_api_email")) + // fmt.Println(" :", viper.GetString("cf_api_key")) + + // fmt.Println(" :", viper.GetString("metrics_path")) + + // fmt.Println(":ASD :", viper.GetString("listen")) + + // fmt.Println(" :", cfgListen) + + cfgMetricsPath := viper.GetString("metrics_path") + + if !(len(viper.GetString("cf_api_token")) > 0 || (len(viper.GetString("cf_api_email")) > 0 && len(viper.GetString("cf_api_key")) > 0)) { log.Fatal("Please provide CF_API_KEY+CF_API_EMAIL or CF_API_TOKEN") } - if cfgBatchSize < 1 || cfgBatchSize > 10 { + if viper.GetInt("cf_batch_size") < 1 || viper.GetInt("cf_batch_size") > 10 { log.Fatal("CF_BATCH_SIZE must be between 1 and 10") } customFormatter := new(log.TextFormatter) @@ -154,8 +157,8 @@ func main() { customFormatter.FullTimestamp = true metricsDenylist := []string{} - if len(cfgMetricsDenylist) > 0 { - metricsDenylist = strings.Split(cfgMetricsDenylist, ",") + if len(viper.GetString("metrics_denylist")) > 0 { + metricsDenylist = strings.Split(viper.GetString("metrics_denylist"), ",") } deniedMetricsSet, err := buildDeniedMetricsSet(metricsDenylist) if err != nil { @@ -169,14 +172,81 @@ func main() { } }() - //This section will start the HTTP server and expose - //any metrics on the /metrics endpoint. - if !strings.HasPrefix(cfgMetricsPath, "/") { - cfgMetricsPath = "/" + cfgMetricsPath + // This section will start the HTTP server and expose + // any metrics on the /metrics endpoint. + if !strings.HasPrefix(viper.GetString("metrics_path"), "/") { + cfgMetricsPath = "/" + viper.GetString("metrics_path") } + http.Handle(cfgMetricsPath, promhttp.Handler()) h := health.New(health.Health{}) http.HandleFunc("/health", h.Handler) - log.Info("Beginning to serve on port", cfgListen, ", metrics path ", cfgMetricsPath) - log.Fatal(http.ListenAndServe(cfgListen, nil)) + + log.Info("Beginning to serve metrics on ", viper.GetString("listen"), cfgMetricsPath) + + server := &http.Server{ + Addr: viper.GetString("listen"), + ReadHeaderTimeout: 3 * time.Second, + } + + log.Fatal(server.ListenAndServe()) +} + +func main() { + var cmd = &cobra.Command{ + Use: "viper-test", + Short: "testing viper", + Run: func(_ *cobra.Command, _ []string) { + runExpoter() + }, + } + + //vip := viper.New() + viper.AutomaticEnv() + + flags := cmd.Flags() + + flags.String("listen", ":8080", "listen on addr:port ( default :8080), omit addr to listen on all interfaces") + viper.BindEnv("listen") + viper.SetDefault("listen", ":8080") + + flags.String("metrics_path", "/metrics", "path for metrics, default /metrics") + viper.BindEnv("metrics_path") + viper.SetDefault("metrics_path", "/metrics") + + flags.String("cf_api_key", "", "cloudflare api key, works with api_email flag") + viper.BindEnv("cf_api_key") + + flags.String("cf_api_email", "", "cloudflare api email, works with api_key flag") + viper.BindEnv("cf_api_email") + + flags.String("cf_api_token", "", "cloudflare api token (preferred)") + viper.BindEnv("cf_api_token") + + flags.String("cf_zones", "", "cloudflare zones to export, comma delimited list") + viper.BindEnv("cf_zones") + viper.SetDefault("cf_zones", "") + + flags.String("cf_exclude_zones", "", "cloudflare zones to exclude, comma delimited list") + viper.BindEnv("cf_exclude_zones") + viper.SetDefault("cf_exclude_zones", "") + + flags.Int("scrape_delay", 300, "scrape delay in seconds, defaults to 300") + viper.BindEnv("scrape_delay") + viper.SetDefault("scrape_delay", 300) + + flags.Int("cf_batch_size", 10, "cloudflare zones batch size (1-10), defaults to 10") + viper.BindEnv("cf_batch_size") + viper.SetDefault("cf_batch_size", 10) + + flags.Bool("free_tier", false, "scrape only metrics included in free plan") + viper.BindEnv("free_tier") + viper.SetDefault("free_tier", false) + + flags.String("metrics_denylist", "", "metrics to not expose, comma delimited list") + viper.BindEnv("metrics_denylist") + viper.SetDefault("metrics_denylist", "") + + viper.BindPFlags(flags) + cmd.Execute() } diff --git a/prometheus.go b/prometheus.go index d10ef20..77b05af 100644 --- a/prometheus.go +++ b/prometheus.go @@ -3,11 +3,13 @@ package main import ( "fmt" "strconv" + "strings" "sync" "github.com/biter777/countries" cloudflare "github.com/cloudflare/cloudflare-go" "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/viper" ) type MetricName string @@ -47,6 +49,8 @@ const ( workerDurationMetricName MetricName = "cloudflare_worker_duration" poolHealthStatusMetricName MetricName = "cloudflare_zone_pool_health_status" poolRequestsTotalMetricName MetricName = "cloudflare_zone_pool_requests_total" + logpushFailedJobsAccountMetricName MetricName = "cloudflare_logpush_failed_jobs_account_count" + logpushFailedJobsZoneMetricName MetricName = "cloudflare_logpush_failed_jobs_zone_count" ) type MetricsSet map[MetricName]struct{} @@ -65,183 +69,198 @@ var ( zoneRequestTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestTotalMetricName.String(), Help: "Number of requests for zone", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneRequestCached = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestCachedMetricName.String(), Help: "Number of cached requests for zone", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneRequestSSLEncrypted = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestSSLEncryptedMetricName.String(), Help: "Number of encrypted requests for zone", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneRequestContentType = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestContentTypeMetricName.String(), Help: "Number of request for zone per content type", - }, []string{"zone", "content_type"}, + }, []string{"zone", "account", "content_type"}, ) zoneRequestCountry = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestCountryMetricName.String(), Help: "Number of request for zone per country", - }, []string{"zone", "country", "region"}, + }, []string{"zone", "account", "country", "region"}, ) zoneRequestHTTPStatus = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestHTTPStatusMetricName.String(), Help: "Number of request for zone per HTTP status", - }, []string{"zone", "status"}, + }, []string{"zone", "account", "status"}, ) zoneRequestBrowserMap = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestBrowserMapMetricName.String(), Help: "Number of successful requests for HTML pages per zone", - }, []string{"zone", "family"}, + }, []string{"zone", "account", "family"}, ) zoneRequestOriginStatusCountryHost = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestOriginStatusCountryHostMetricName.String(), Help: "Count of not cached requests for zone per origin HTTP status per country per host", - }, []string{"zone", "status", "country", "host"}, + }, []string{"zone", "account", "status", "country", "host"}, ) zoneRequestStatusCountryHost = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneRequestStatusCountryHostMetricName.String(), Help: "Count of requests for zone per edge HTTP status per country per host", - }, []string{"zone", "status", "country", "host"}, + }, []string{"zone", "account", "status", "country", "host"}, ) zoneBandwidthTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneBandwidthTotalMetricName.String(), Help: "Total bandwidth per zone in bytes", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneBandwidthCached = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneBandwidthCachedMetricName.String(), Help: "Cached bandwidth per zone in bytes", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneBandwidthSSLEncrypted = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneBandwidthSSLEncryptedMetricName.String(), Help: "Encrypted bandwidth per zone in bytes", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneBandwidthContentType = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneBandwidthContentTypeMetricName.String(), Help: "Bandwidth per zone per content type", - }, []string{"zone", "content_type"}, + }, []string{"zone", "account", "content_type"}, ) zoneBandwidthCountry = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneBandwidthCountryMetricName.String(), Help: "Bandwidth per country per zone", - }, []string{"zone", "country", "region"}, + }, []string{"zone", "account", "country", "region"}, ) zoneThreatsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneThreatsTotalMetricName.String(), Help: "Threats per zone", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneThreatsCountry = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneThreatsCountryMetricName.String(), Help: "Threats per zone per country", - }, []string{"zone", "country", "region"}, + }, []string{"zone", "account", "country", "region"}, ) zoneThreatsType = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneThreatsTypeMetricName.String(), Help: "Threats per zone per type", - }, []string{"zone", "type"}, + }, []string{"zone", "account", "type"}, ) zonePageviewsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zonePageviewsTotalMetricName.String(), Help: "Pageviews per zone", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneUniquesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneUniquesTotalMetricName.String(), Help: "Uniques per zone", - }, []string{"zone"}, + }, []string{"zone", "account"}, ) zoneColocationVisits = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneColocationVisitsMetricName.String(), Help: "Total visits per colocation", - }, []string{"zone", "colocation", "host"}, + }, []string{"zone", "account", "colocation", "host"}, ) zoneColocationEdgeResponseBytes = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneColocationEdgeResponseBytesMetricName.String(), Help: "Edge response bytes per colocation", - }, []string{"zone", "colocation", "host"}, + }, []string{"zone", "account", "colocation", "host"}, ) zoneColocationRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneColocationRequestsTotalMetricName.String(), Help: "Total requests per colocation", - }, []string{"zone", "colocation", "host"}, + }, []string{"zone", "account", "colocation", "host"}, ) zoneFirewallEventsCount = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneFirewallEventsCountMetricName.String(), Help: "Count of Firewall events", - }, []string{"zone", "action", "source", "host", "country"}, + }, []string{"zone", "account", "action", "source", "rule", "host", "country"}, ) zoneHealthCheckEventsOriginCount = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: zoneHealthCheckEventsOriginCountMetricName.String(), Help: "Number of Heath check events per region per origin", - }, []string{"zone", "health_status", "origin_ip", "region", "fqdn"}, + }, []string{"zone", "account", "health_status", "origin_ip", "region", "fqdn"}, ) workerRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: workerRequestsMetricName.String(), Help: "Number of requests sent to worker by script name", - }, []string{"script_name"}, + }, []string{"script_name", "account"}, ) workerErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: workerErrorsMetricName.String(), Help: "Number of errors by script name", - }, []string{"script_name"}, + }, []string{"script_name", "account"}, ) workerCPUTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: workerCPUTimeMetricName.String(), Help: "CPU time quantiles by script name", - }, []string{"script_name", "quantile"}, + }, []string{"script_name", "account", "quantile"}, ) workerDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: workerDurationMetricName.String(), Help: "Duration quantiles by script name (GB*s)", - }, []string{"script_name", "quantile"}, + }, []string{"script_name", "account", "quantile"}, ) poolHealthStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: poolHealthStatusMetricName.String(), Help: "Reports the health of a pool, 1 for healthy, 0 for unhealthy.", }, - []string{"zone", "load_balancer_name", "pool_name"}, + []string{"zone", "account", "load_balancer_name", "pool_name"}, ) poolRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: poolRequestsTotalMetricName.String(), Help: "Requests per pool", }, - []string{"zone", "load_balancer_name", "pool_name", "origin_name"}, + []string{"zone", "account", "load_balancer_name", "pool_name", "origin_name"}, + ) + + // TODO: Update this to counter vec and use counts from the query to add + logpushFailedJobsAccount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: logpushFailedJobsAccountMetricName.String(), + Help: "Number of failed logpush jobs on the account level", + }, + []string{"account", "destination", "job_id", "final"}, + ) + + logpushFailedJobsZone = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: logpushFailedJobsZoneMetricName.String(), + Help: "Number of failed logpush jobs on the zone level", + }, + []string{"destination", "job_id", "final"}, ) ) @@ -277,6 +296,8 @@ func buildAllMetricsSet() MetricsSet { allMetricsSet.Add(workerDurationMetricName) allMetricsSet.Add(poolHealthStatusMetricName) allMetricsSet.Add(poolRequestsTotalMetricName) + allMetricsSet.Add(logpushFailedJobsAccountMetricName) + allMetricsSet.Add(logpushFailedJobsZoneMetricName) return allMetricsSet } @@ -383,6 +404,12 @@ func mustRegisterMetrics(deniedMetrics MetricsSet) { if !deniedMetrics.Has(poolRequestsTotalMetricName) { prometheus.MustRegister(poolRequestsTotal) } + if !deniedMetrics.Has(logpushFailedJobsAccountMetricName) { + prometheus.MustRegister(logpushFailedJobsAccount) + } + if !deniedMetrics.Has(logpushFailedJobsZoneMetricName) { + prometheus.MustRegister(logpushFailedJobsZone) + } } func fetchWorkerAnalytics(account cloudflare.Account, wg *sync.WaitGroup) { @@ -394,18 +421,73 @@ func fetchWorkerAnalytics(account cloudflare.Account, wg *sync.WaitGroup) { return } + // Replace spaces with hyphens and convert to lowercase + accountName := strings.ToLower(strings.ReplaceAll(account.Name, " ", "-")) + for _, a := range r.Viewer.Accounts { for _, w := range a.WorkersInvocationsAdaptive { - workerRequests.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName}).Add(float64(w.Sum.Requests)) - workerErrors.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName}).Add(float64(w.Sum.Errors)) - workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P50"}).Set(float64(w.Quantiles.CPUTimeP50)) - workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P75"}).Set(float64(w.Quantiles.CPUTimeP75)) - workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P99"}).Set(float64(w.Quantiles.CPUTimeP99)) - workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P999"}).Set(float64(w.Quantiles.CPUTimeP999)) - workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P50"}).Set(float64(w.Quantiles.DurationP50)) - workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P75"}).Set(float64(w.Quantiles.DurationP75)) - workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P99"}).Set(float64(w.Quantiles.DurationP99)) - workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "quantile": "P999"}).Set(float64(w.Quantiles.DurationP999)) + workerRequests.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName}).Add(float64(w.Sum.Requests)) + workerErrors.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName}).Add(float64(w.Sum.Errors)) + workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P50"}).Set(float64(w.Quantiles.CPUTimeP50)) + workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P75"}).Set(float64(w.Quantiles.CPUTimeP75)) + workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P99"}).Set(float64(w.Quantiles.CPUTimeP99)) + workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P999"}).Set(float64(w.Quantiles.CPUTimeP999)) + workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P50"}).Set(float64(w.Quantiles.DurationP50)) + workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P75"}).Set(float64(w.Quantiles.DurationP75)) + workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P99"}).Set(float64(w.Quantiles.DurationP99)) + workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "account": accountName, "quantile": "P999"}).Set(float64(w.Quantiles.DurationP999)) + } + } +} + +func fetchLogpushAnalyticsForAccount(account cloudflare.Account, wg *sync.WaitGroup) { + wg.Add(1) + defer wg.Done() + + if viper.GetBool("free_tier") { + return + } + + r, err := fetchLogpushAccount(account.ID) + + if err != nil { + return + } + + for _, acc := range r.Viewer.Accounts { + for _, LogpushHealthAdaptiveGroup := range acc.LogpushHealthAdaptiveGroups { + logpushFailedJobsAccount.With(prometheus.Labels{"account": account.ID, + "destination": LogpushHealthAdaptiveGroup.Dimensions.DestinationType, + "job_id": strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.JobID), + "final": strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.Final)}).Add(float64(LogpushHealthAdaptiveGroup.Count)) + } + } +} + +func fetchLogpushAnalyticsForZone(zones []cloudflare.Zone, wg *sync.WaitGroup) { + wg.Add(1) + defer wg.Done() + + if viper.GetBool("free_tier") { + return + } + + zoneIDs := extractZoneIDs(filterNonFreePlanZones(zones)) + if len(zoneIDs) == 0 { + return + } + + r, err := fetchLogpushZone(zoneIDs) + + if err != nil { + return + } + + for _, zone := range r.Viewer.Zones { + for _, LogpushHealthAdaptiveGroup := range zone.LogpushHealthAdaptiveGroups { + logpushFailedJobsZone.With(prometheus.Labels{"destination": LogpushHealthAdaptiveGroup.Dimensions.DestinationType, + "job_id": strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.JobID), + "final": strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.Final)}).Add(float64(LogpushHealthAdaptiveGroup.Count)) } } } @@ -415,7 +497,7 @@ func fetchZoneColocationAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) { defer wg.Done() // Colocation metrics are not available in non-enterprise zones - if cfgFreeTier { + if viper.GetBool("free_tier") { return } @@ -428,15 +510,13 @@ func fetchZoneColocationAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) { if err != nil { return } - for _, z := range r.Viewer.Zones { - cg := z.ColoGroups - name := findZoneName(zones, z.ZoneTag) + name, account := findZoneAccountName(zones, z.ZoneTag) for _, c := range cg { - zoneColocationVisits.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.Visits)) - zoneColocationEdgeResponseBytes.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.EdgeResponseBytes)) - zoneColocationRequestsTotal.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Count)) + zoneColocationVisits.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.Visits)) + zoneColocationEdgeResponseBytes.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.EdgeResponseBytes)) + zoneColocationRequestsTotal.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Count)) } } } @@ -446,7 +526,7 @@ func fetchZoneAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) { defer wg.Done() // None of the below referenced metrics are available in the free tier - if cfgFreeTier { + if viper.GetBool("free_tier") { return } @@ -461,82 +541,96 @@ func fetchZoneAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) { } for _, z := range r.Viewer.Zones { - name := findZoneName(zones, z.ZoneTag) - addHTTPGroups(&z, name) - addFirewallGroups(&z, name) - addHealthCheckGroups(&z, name) - addHTTPAdaptiveGroups(&z, name) + name, account := findZoneAccountName(zones, z.ZoneTag) + z := z + + addHTTPGroups(&z, name, account) + addFirewallGroups(&z, name, account) + addHealthCheckGroups(&z, name, account) + addHTTPAdaptiveGroups(&z, name, account) } } -func addHTTPGroups(z *zoneResp, name string) { +func addHTTPGroups(z *zoneResp, name string, account string) { // Nothing to do. if len(z.HTTP1mGroups) == 0 { return } + zt := z.HTTP1mGroups[0] - zoneRequestTotal.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.Requests)) - zoneRequestCached.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.CachedRequests)) - zoneRequestSSLEncrypted.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.EncryptedRequests)) + zoneRequestTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.Requests)) + zoneRequestCached.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.CachedRequests)) + zoneRequestSSLEncrypted.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.EncryptedRequests)) for _, ct := range zt.Sum.ContentType { - zoneRequestContentType.With(prometheus.Labels{"zone": name, "content_type": ct.EdgeResponseContentType}).Add(float64(ct.Requests)) - zoneBandwidthContentType.With(prometheus.Labels{"zone": name, "content_type": ct.EdgeResponseContentType}).Add(float64(ct.Bytes)) + zoneRequestContentType.With(prometheus.Labels{"zone": name, "account": account, "content_type": ct.EdgeResponseContentType}).Add(float64(ct.Requests)) + zoneBandwidthContentType.With(prometheus.Labels{"zone": name, "account": account, "content_type": ct.EdgeResponseContentType}).Add(float64(ct.Bytes)) } for _, country := range zt.Sum.Country { c := countries.ByName(country.ClientCountryName) region := c.Info().Region.Info().Name - zoneRequestCountry.With(prometheus.Labels{"zone": name, "country": country.ClientCountryName, "region": region}).Add(float64(country.Requests)) - zoneBandwidthCountry.With(prometheus.Labels{"zone": name, "country": country.ClientCountryName, "region": region}).Add(float64(country.Bytes)) - zoneThreatsCountry.With(prometheus.Labels{"zone": name, "country": country.ClientCountryName, "region": region}).Add(float64(country.Threats)) + zoneRequestCountry.With(prometheus.Labels{"zone": name, "account": account, "country": country.ClientCountryName, "region": region}).Add(float64(country.Requests)) + zoneBandwidthCountry.With(prometheus.Labels{"zone": name, "account": account, "country": country.ClientCountryName, "region": region}).Add(float64(country.Bytes)) + zoneThreatsCountry.With(prometheus.Labels{"zone": name, "account": account, "country": country.ClientCountryName, "region": region}).Add(float64(country.Threats)) } for _, status := range zt.Sum.ResponseStatus { - zoneRequestHTTPStatus.With(prometheus.Labels{"zone": name, "status": strconv.Itoa(status.EdgeResponseStatus)}).Add(float64(status.Requests)) + zoneRequestHTTPStatus.With(prometheus.Labels{"zone": name, "account": account, "status": strconv.Itoa(status.EdgeResponseStatus)}).Add(float64(status.Requests)) } for _, browser := range zt.Sum.BrowserMap { - zoneRequestBrowserMap.With(prometheus.Labels{"zone": name, "family": browser.UaBrowserFamily}).Add(float64(browser.PageViews)) + zoneRequestBrowserMap.With(prometheus.Labels{"zone": name, "account": account, "family": browser.UaBrowserFamily}).Add(float64(browser.PageViews)) } - zoneBandwidthTotal.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.Bytes)) - zoneBandwidthCached.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.CachedBytes)) - zoneBandwidthSSLEncrypted.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.EncryptedBytes)) + zoneBandwidthTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.Bytes)) + zoneBandwidthCached.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.CachedBytes)) + zoneBandwidthSSLEncrypted.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.EncryptedBytes)) - zoneThreatsTotal.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.Threats)) + zoneThreatsTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.Threats)) for _, t := range zt.Sum.ThreatPathing { - zoneThreatsType.With(prometheus.Labels{"zone": name, "type": t.Name}).Add(float64(t.Requests)) + zoneThreatsType.With(prometheus.Labels{"zone": name, "account": account, "type": t.Name}).Add(float64(t.Requests)) } - zonePageviewsTotal.With(prometheus.Labels{"zone": name}).Add(float64(zt.Sum.PageViews)) + zonePageviewsTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.PageViews)) // Uniques - zoneUniquesTotal.With(prometheus.Labels{"zone": name}).Add(float64(zt.Unique.Uniques)) + zoneUniquesTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Unique.Uniques)) } -func addFirewallGroups(z *zoneResp, name string) { +func addFirewallGroups(z *zoneResp, name string, account string) { // Nothing to do. if len(z.FirewallEventsAdaptiveGroups) == 0 { return } - + rulesMap := fetchFirewallRules(z.ZoneTag) for _, g := range z.FirewallEventsAdaptiveGroups { zoneFirewallEventsCount.With( prometheus.Labels{ "zone": name, + "account": account, "action": g.Dimensions.Action, "source": g.Dimensions.Source, + "rule": normalizeRuleName(rulesMap[g.Dimensions.RuleID]), "host": g.Dimensions.ClientRequestHTTPHost, "country": g.Dimensions.ClientCountryName, }).Add(float64(g.Count)) } } -func addHealthCheckGroups(z *zoneResp, name string) { +func normalizeRuleName(initialText string) string { + maxLength := 200 + nonSpaceName := strings.ReplaceAll(strings.ToLower(initialText), " ", "_") + if len(nonSpaceName) > maxLength { + return nonSpaceName[:maxLength] + } + return nonSpaceName +} + +func addHealthCheckGroups(z *zoneResp, name string, account string) { if len(z.HealthCheckEventsAdaptiveGroups) == 0 { return } @@ -545,6 +639,7 @@ func addHealthCheckGroups(z *zoneResp, name string) { zoneHealthCheckEventsOriginCount.With( prometheus.Labels{ "zone": name, + "account": account, "health_status": g.Dimensions.HealthStatus, "origin_ip": g.Dimensions.OriginIP, "region": g.Dimensions.Region, @@ -553,12 +648,12 @@ func addHealthCheckGroups(z *zoneResp, name string) { } } -func addHTTPAdaptiveGroups(z *zoneResp, name string) { - +func addHTTPAdaptiveGroups(z *zoneResp, name string, account string) { for _, g := range z.HTTPRequestsAdaptiveGroups { zoneRequestOriginStatusCountryHost.With( prometheus.Labels{ "zone": name, + "account": account, "status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)), "country": g.Dimensions.ClientCountryName, "host": g.Dimensions.ClientRequestHTTPHost, @@ -569,12 +664,12 @@ func addHTTPAdaptiveGroups(z *zoneResp, name string) { zoneRequestStatusCountryHost.With( prometheus.Labels{ "zone": name, + "account": account, "status": strconv.Itoa(int(g.Dimensions.EdgeResponseStatus)), "country": g.Dimensions.ClientCountryName, "host": g.Dimensions.ClientRequestHTTPHost, }).Add(float64(g.Count)) } - } func fetchLoadBalancerAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) { @@ -582,7 +677,7 @@ func fetchLoadBalancerAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) { defer wg.Done() // None of the below referenced metrics are available in the free tier - if cfgFreeTier { + if viper.GetBool("free_tier") { return } @@ -596,18 +691,19 @@ func fetchLoadBalancerAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) { return } for _, lb := range l.Viewer.Zones { - name := findZoneName(zones, lb.ZoneTag) - addLoadBalancingRequestsAdaptive(&lb, name) - addLoadBalancingRequestsAdaptiveGroups(&lb, name) + name, account := findZoneAccountName(zones, lb.ZoneTag) + lb := lb + addLoadBalancingRequestsAdaptive(&lb, name, account) + addLoadBalancingRequestsAdaptiveGroups(&lb, name, account) } } -func addLoadBalancingRequestsAdaptiveGroups(z *lbResp, name string) { - +func addLoadBalancingRequestsAdaptiveGroups(z *lbResp, name string, account string) { for _, g := range z.LoadBalancingRequestsAdaptiveGroups { poolRequestsTotal.With( prometheus.Labels{ "zone": name, + "account": account, "load_balancer_name": g.Dimensions.LbName, "pool_name": g.Dimensions.SelectedPoolName, "origin_name": g.Dimensions.SelectedOriginName, @@ -615,17 +711,16 @@ func addLoadBalancingRequestsAdaptiveGroups(z *lbResp, name string) { } } -func addLoadBalancingRequestsAdaptive(z *lbResp, name string) { - +func addLoadBalancingRequestsAdaptive(z *lbResp, name string, account string) { for _, g := range z.LoadBalancingRequestsAdaptive { for _, p := range g.Pools { poolHealthStatus.With( prometheus.Labels{ "zone": name, + "account": account, "load_balancer_name": g.LbName, "pool_name": p.PoolName, }).Set(float64(p.Healthy)) } } - } diff --git a/run_e2e.sh b/run_e2e.sh new file mode 100755 index 0000000..937e625 --- /dev/null +++ b/run_e2e.sh @@ -0,0 +1,22 @@ +#!/bin/bash +export basePort="8081" +export metricsPath='/metrics' +export baseUrl="localhost:${basePort}" + +source .env +# Test if we have cloudflare api/key variables configured. + +# Run cloudflare-exporter +nohup ./cloudflare_exporter --listen="${baseUrl}" >/tmp/cloudflare-expoter-test.out 2>&1 & +export pid=$! +sleep 5 + +# Get metrics +curl -s -o /tmp/cloudflare_exporter_test_output http://${baseUrl}${metricsPath} + +# Run Tests +venom run tests/basic_tests.yml + +# Cleanup +rm venom*.log +kill ${pid} diff --git a/tests/basic_tests.yml b/tests/basic_tests.yml new file mode 100644 index 0000000..49e67bb --- /dev/null +++ b/tests/basic_tests.yml @@ -0,0 +1,25 @@ +name: Cloudflare exporter testsuite +vars: + basePort: "8081" + baseUrl: "http://localhost" +testcases: + - name: Check Health endpoint + steps: + - type: http + method: GET + url: "{{.baseUrl}}:{{.basePort}}/health" + assertions: + - result.statuscode ShouldEqual 200 + - result.body ShouldNotBeEmpty + + - name: Get Cloudflare metrics test case + steps: + - type: http + method: GET + url: "{{.baseUrl}}:{{.basePort}}/metrics" + assertions: + - result.body ShouldContainSubstring cloudflare_zone_threats_total + - result.body ShouldContainSubstring promhttp_metric_handler_requests_in_flight + - result.body ShouldContainSubstring go_gc_duration_seconds_count + - result.statuscode ShouldEqual 200 + - result.body ShouldNotBeEmpty