From a4f3b93f0311e0fb2835c97166f638d1e5bc1691 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Fri, 8 Mar 2024 10:32:10 +0100 Subject: [PATCH 01/32] golangcli lint --- .github/workflows/golangci-lint.yml | 17 ++++ .gitignore | 5 + .tool-versions | 3 + README.md | 5 +- cmd/exporter/main.go | 23 +++++ config/example.yaml | 14 +++ go.mod | 43 ++++++++ go.sum | 88 ++++++++++++++++ internal/app/application.go | 92 +++++++++++++++++ internal/app/config.go | 12 +++ internal/exporter/exporter.go | 78 +++++++++++++++ internal/exporter/exporter_test.go | 79 +++++++++++++++ internal/exporter/options.go | 28 ++++++ internal/http/http.go | 68 +++++++++++++ internal/http/http_test.go | 40 ++++++++ internal/scrape/config.go | 51 ++++++++++ internal/scrape/quotas/collector.go | 69 +++++++++++++ internal/scrape/quotas/config.go | 23 +++++ internal/scrape/script/collector.go | 62 ++++++++++++ internal/scrape/script/config.go | 60 +++++++++++ internal/scrape/script/script.go | 138 ++++++++++++++++++++++++++ internal/scrape/script/script_test.go | 106 ++++++++++++++++++++ pkg/config/yaml.go | 24 +++++ pkg/flags/parse.go | 21 ++++ pkg/log/log.go | 39 ++++++++ pkg/quota/client.go | 63 ++++++++++++ pkg/quota/options.go | 28 ++++++ pkg/service/ctx.go | 13 +++ pkg/service/mng.go | 42 ++++++++ test/config.go | 19 ++++ test/configs/eip.yaml | 14 +++ test/log.go | 16 +++ 32 files changed, 1382 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 .tool-versions create mode 100644 cmd/exporter/main.go create mode 100644 config/example.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/app/application.go create mode 100644 internal/app/config.go create mode 100644 internal/exporter/exporter.go create mode 100644 internal/exporter/exporter_test.go create mode 100644 internal/exporter/options.go create mode 100644 internal/http/http.go create mode 100644 internal/http/http_test.go create mode 100644 internal/scrape/config.go create mode 100644 internal/scrape/quotas/collector.go create mode 100644 internal/scrape/quotas/config.go create mode 100644 internal/scrape/script/collector.go create mode 100644 internal/scrape/script/config.go create mode 100644 internal/scrape/script/script.go create mode 100644 internal/scrape/script/script_test.go create mode 100644 pkg/config/yaml.go create mode 100644 pkg/flags/parse.go create mode 100644 pkg/log/log.go create mode 100644 pkg/quota/client.go create mode 100644 pkg/quota/options.go create mode 100644 pkg/service/ctx.go create mode 100644 pkg/service/mng.go create mode 100644 test/config.go create mode 100644 test/configs/eip.yaml create mode 100644 test/log.go diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..a1f9078 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,17 @@ +name: golangci-lint +on: + push: + pull_request: +permissions: + contents: read + +jobs: + golangci: + name: GO lang CI linter + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: v1.56.2 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3b735ec..ec29974 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,8 @@ # Go workspace file go.work + +.env +.envrc + +.idea \ No newline at end of file diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..db143de --- /dev/null +++ b/.tool-versions @@ -0,0 +1,3 @@ +helm 3.14.2 +awscli 2.7.14 +golang 1.22.0 diff --git a/README.md b/README.md index 002342f..bb6a45e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,5 @@ # aws-service-quotas-exporter -AWS Quotas utilisation prometheus exporter + +AWS Quotas utilisation prometheus exporter is Prometheus exporter allows you to easily export real usage of AWS resources +and monitor/alert usage of over exited of those resources with respect to applied Quotas. + diff --git a/cmd/exporter/main.go b/cmd/exporter/main.go new file mode 100644 index 0000000..fadce8d --- /dev/null +++ b/cmd/exporter/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "github.com/lablabs/aws-service-quotas-exporter/internal/app" + "github.com/lablabs/aws-service-quotas-exporter/pkg/flags" + log "github.com/lablabs/aws-service-quotas-exporter/pkg/log" + "github.com/lablabs/aws-service-quotas-exporter/pkg/service" + "os" +) + +func main() { + cfg := app.Config{} + flags.ParseOrFail(&cfg, os.Args) + logger := log.NewLoggerOrFail(cfg.Log.Format, cfg.Log.Level) + app, err := app.NewApplication(logger, cfg) + if err != nil { + logger.Fatal(err) + } + ctx := service.SignContext() + if err := app.Run(ctx); err != nil { + logger.Fatal(err) + } +} diff --git a/config/example.yaml b/config/example.yaml new file mode 100644 index 0000000..42287a0 --- /dev/null +++ b/config/example.yaml @@ -0,0 +1,14 @@ +#quotas: +# - serviceCode: "ec2" +# quotaCode: "L-0263D0A3" +metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + command: "aws route53 list-hosted-zones" + list: ".HostedZones" + value: ".ResourceRecordSetCount" + labels: + - name: "id" + jqValue: ".Id" + - name: "name" + jqValue: ".Name" \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..116ea0b --- /dev/null +++ b/go.mod @@ -0,0 +1,43 @@ +module github.com/lablabs/aws-service-quotas-exporter + +go 1.22.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.25.2 + github.com/aws/aws-sdk-go-v2/config v1.27.4 + github.com/aws/aws-sdk-go-v2/service/cloudcontrol v1.18.1 + github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.1 + github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.4 + golang.org/x/sync v0.6.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.17.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 // indirect + github.com/aws/smithy-go v1.20.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-test/deep v1.1.0 // indirect + github.com/jessevdk/go-flags v1.5.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/ohler55/ojg v1.21.3 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + golang.org/x/sys v0.15.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..6b6385f --- /dev/null +++ b/go.sum @@ -0,0 +1,88 @@ +github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w= +github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo= +github.com/aws/aws-sdk-go-v2/config v1.27.4 h1:AhfWb5ZwimdsYTgP7Od8E9L1u4sKmDW2ZVeLcf2O42M= +github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.4 h1:h5Vztbd8qLppiPwX+y0Q6WiwMZgpd9keKe2EAENgAuI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 h1:bNo4LagzUKbjdxE0tIcR9pMzLR2U/Tgie1Hq1HQ3iH8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 h1:EtOU5jsPdIQNP+6Q2C5e3d65NKT1PeCiQk+9OdzO12Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/cloudcontrol v1.18.1 h1:v7LcvMEl5uRm3SOYY+JtXbs8F16S8kfJZdONU+KADoE= +github.com/aws/aws-sdk-go-v2/service/cloudcontrol v1.18.1/go.mod h1:fBKrOkINcm/C9bXnnO5NwwIT7tGIaUb/LRXHNpwXpUM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 h1:5ffmXjPtwRExp1zc7gENLgCPyHFbhEPwVTkTiH9niSk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.1 h1:CCd+AWX79LVGzTXkgW9fBaPRFiC+J67zGuMcsER8Q1g= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.1/go.mod h1:+M0h5pY1hwymLXfxTAiAB0D87KxkvGrqmDz0gQbrm4A= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 h1:utEGkfdQ4L6YW/ietH7111ZYglLJvS+sLriHJ1NBJEQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 h1:9/GylMS45hGGFCcMrUZDVayQE1jYSIN6da9jo7RAYIw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 h1:3I2cBEYgKhrWlwyZgfpSO2BpaMY1LHPqXYk/QGlu2ew= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA= +github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= +github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/ohler55/ojg v1.21.3 h1:0smW0EKpyPBBIpTKhM+UbCDeQFbR0oEUxym+rFv2Y/8= +github.com/ohler55/ojg v1.21.3/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/application.go b/internal/app/application.go new file mode 100644 index 0000000..980a919 --- /dev/null +++ b/internal/app/application.go @@ -0,0 +1,92 @@ +package app + +import ( + "context" + "fmt" + "github.com/lablabs/aws-service-quotas-exporter/internal/exporter" + "github.com/lablabs/aws-service-quotas-exporter/internal/http" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/quotas" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/lablabs/aws-service-quotas-exporter/pkg/quota" + "github.com/lablabs/aws-service-quotas-exporter/pkg/service" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" +) + +const ( + PrometheusNamespace = "quota_exporter" +) + +func NewApplication(log *logrus.Logger, cfg Config) (*Application, error) { + mng, err := service.NewManager() + if err != nil { + return nil, err + } + scCfg, err := scrape.LoadAndValidateConfig(cfg.Config) + if err != nil { + return nil, fmt.Errorf("unable to configure application: %w", err) + } + + registry := prometheus.NewRegistry() + cls := make([]exporter.Collector, 0) + client, err := quota.NewClient(log) + if err != nil { + return nil, err + } + qcl, err := quotas.NewCollector(log, scCfg.Quotas, PrometheusNamespace, client) + if err != nil { + return nil, err + } + qcl.Register(registry) + cls = append(cls, qcl) + + scl, err := script.NewCollector(log, scCfg.Metrics, PrometheusNamespace) + if err != nil { + return nil, err + } + scl.Register(registry) + cls = append(cls, scl) + + exp, err := exporter.NewExporter(log, cls, exporterOptions(scCfg)...) + if err != nil { + return nil, err + } + mng.Add(exp) + + http, err := http.NewHttp(log, cfg.Address, registry) + if err != nil { + return nil, err + } + mng.Add(http) + a := Application{ + log: log, + cfg: cfg, + mng: mng, + } + return &a, nil +} + +type Application struct { + log *logrus.Logger + cfg Config + mng *service.Manager +} + +func (a *Application) Run(ctx context.Context) error { + a.log.Infof("exporter is starting") + err := a.mng.StartAndWait(ctx) + if err != nil { + return err + } + <-ctx.Done() + a.log.Infof("exporter exit OK") + return nil +} + +func exporterOptions(cfg *scrape.Config) []exporter.Option { + return []exporter.Option{ + exporter.WithInterval(cfg.Global.Interval), + exporter.WithTimeout(cfg.Global.Timeout), + } +} diff --git a/internal/app/config.go b/internal/app/config.go new file mode 100644 index 0000000..8d30e96 --- /dev/null +++ b/internal/app/config.go @@ -0,0 +1,12 @@ +package app + +type Log struct { + Level string `long:"level" description:"Log level" choice:"DEBUG" choice:"INFO" default:"DEBUG"` + Format string `long:"format" description:"Format of message logs" choice:"json" default:"json"` +} + +type Config struct { + Address string `long:"address" description:"Http address" default:"0.0.0.0:8080"` + Log Log `group:"log" namespace:"log"` + Config string `long:"config" required:"true" description:"Path to scraper scrape file"` +} diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go new file mode 100644 index 0000000..a7ef518 --- /dev/null +++ b/internal/exporter/exporter.go @@ -0,0 +1,78 @@ +package exporter + +import ( + "context" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "time" +) + +type Collector interface { + Register(r *prometheus.Registry) error + Collect(g *errgroup.Group, ctx context.Context) +} + +const ( + defaultScrapeInterval = time.Second * 60 + defaultCollectorTimeout = time.Second * 5 +) + +func NewExporter(log *logrus.Logger, cls []Collector, options ...Option) (*Exporter, error) { + cfg := config{ + interval: defaultScrapeInterval, + timeout: defaultCollectorTimeout, + } + for _, o := range options { + if err := o(&cfg); err != nil { + return nil, fmt.Errorf("unable to configure exporter: %w", err) + } + } + e := Exporter{ + log: log, + cfg: &cfg, + cls: cls, + } + return &e, nil +} + +type Exporter struct { + log *logrus.Logger + cfg *config + cls []Collector +} + +func (e *Exporter) Run(ctx context.Context) error { + err := e.scrape(ctx) + if err != nil { + return err + } + ticker := time.NewTicker(e.cfg.interval) + e.log.Debugf("scrape metrics every: %v", e.cfg.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + err := e.scrape(ctx) + if err != nil { + e.log.Errorf("unable to scrape metric: %v", err) + } + } + } + return nil +} + +func (e *Exporter) scrape(ctx context.Context) error { + e.log.Debugf("start scraping metrics %v with timeout: %v", time.Now().Format(time.RFC3339), e.cfg.timeout) + ctx, cancel := context.WithTimeout(ctx, e.cfg.timeout) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + for _, c := range e.cls { + c.Collect(g, ctx) + } + err := g.Wait() + return err +} diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go new file mode 100644 index 0000000..0fb9796 --- /dev/null +++ b/internal/exporter/exporter_test.go @@ -0,0 +1,79 @@ +package exporter_test + +import ( + "context" + "errors" + "github.com/lablabs/aws-service-quotas-exporter/internal/exporter" + "github.com/lablabs/aws-service-quotas-exporter/test" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "golang.org/x/sync/errgroup" + "testing" + "time" +) + +func TestExporter_Run(t *testing.T) { + type fields struct { + log *logrus.Logger + cls []exporter.Collector + ops []exporter.Option + ctx func() (context.Context, func()) + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "Exporter OK", + fields: fields{ + log: test.DefaultLogger(), + cls: []exporter.Collector{&testCollector{}}, + ops: []exporter.Option{}, + ctx: func() (context.Context, func()) { + return context.WithTimeout(context.Background(), time.Second*1) + }, + }, + wantErr: false, + }, + { + name: "Exporter timeout", + fields: fields{ + log: test.DefaultLogger(), + cls: []exporter.Collector{&testCollector{err: errors.New("timeout")}}, + ops: []exporter.Option{}, + ctx: func() (context.Context, func()) { + return context.WithTimeout(context.Background(), time.Second*1) + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e, err := exporter.NewExporter(tt.fields.log, tt.fields.cls) + assert.NoError(t, err) + ctx, cancel := tt.fields.ctx() + defer cancel() + if err := e.Run(ctx); (err != nil) != tt.wantErr { + t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + } + + }) + } +} + +type testCollector struct { + err error +} + +func (t *testCollector) Register(r *prometheus.Registry) error { + return nil +} + +func (t *testCollector) Collect(g *errgroup.Group, ctx context.Context) { + g.Go(func() error { + return t.err + }) +} diff --git a/internal/exporter/options.go b/internal/exporter/options.go new file mode 100644 index 0000000..8705cfe --- /dev/null +++ b/internal/exporter/options.go @@ -0,0 +1,28 @@ +package exporter + +import "time" + +type config struct { + interval time.Duration + timeout time.Duration +} + +type Option func(c *config) error + +func WithInterval(i time.Duration) Option { + return func(c *config) error { + if i.Nanoseconds() != 0 { + c.interval = i + } + return nil + } +} + +func WithTimeout(t time.Duration) Option { + return func(c *config) error { + if t.Nanoseconds() != 0 { + c.timeout = t + } + return nil + } +} diff --git a/internal/http/http.go b/internal/http/http.go new file mode 100644 index 0000000..3128c07 --- /dev/null +++ b/internal/http/http.go @@ -0,0 +1,68 @@ +package http + +import ( + "context" + "errors" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "net" + "net/http" +) + +func NewHttp(log *logrus.Logger, address string, registry *prometheus.Registry) (*Http, error) { + ln, err := net.Listen("tcp", address) + if err != nil { + return nil, err + } + handler := http.NewServeMux() + s := http.Server{ + Handler: handler, + } + h := Http{ + log: log, + ln: ln, + s: s, + } + RegisterMetricEndpoint(handler, registry) + return &h, nil +} + +type Http struct { + log *logrus.Logger + ln net.Listener + s http.Server +} + +func (h *Http) Run(ctx context.Context) error { + h.log.Infof("start http endpoint: %s", h.ln.Addr()) + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + err := h.s.Serve(h.ln) + if !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("unable to serve http enpoint: %w", err) + } + return nil + }) + <-ctx.Done() + h.log.Debugf("http endpoint exiting...") + err := h.s.Shutdown(ctx) + if err != nil { + return err + } + err = g.Wait() + if err != nil { + return err + } + h.log.Infof("http endpoint exit OK") + return nil +} + +func RegisterMetricEndpoint(mux *http.ServeMux, registry *prometheus.Registry) { + registry.MustRegister(collectors.NewGoCollector()) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{Registry: registry})) +} diff --git a/internal/http/http_test.go b/internal/http/http_test.go new file mode 100644 index 0000000..b822396 --- /dev/null +++ b/internal/http/http_test.go @@ -0,0 +1,40 @@ +package http_test + +import ( + "context" + httpApi "github.com/lablabs/aws-service-quotas-exporter/internal/http" + "github.com/lablabs/aws-service-quotas-exporter/test" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "net/http" + "sync" + "testing" +) + +func TestNewHttp(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + address, close := StartHttp(t, ctx) + defer func() { + cancel() + close() + }() + resp, err := http.Get("http://" + address + "/metrics") + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +func StartHttp(t *testing.T, ctx context.Context) (string, func()) { + address := "0.0.0.0:8080" + http, err := httpApi.NewHttp(test.DefaultLogger(), address, prometheus.NewRegistry()) + assert.NoError(t, err) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := http.Run(ctx) + assert.NoError(t, err) + }() + return address, func() { + wg.Wait() + } +} diff --git a/internal/scrape/config.go b/internal/scrape/config.go new file mode 100644 index 0000000..655296d --- /dev/null +++ b/internal/scrape/config.go @@ -0,0 +1,51 @@ +package scrape + +import ( + "fmt" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/quotas" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/lablabs/aws-service-quotas-exporter/pkg/config" + "time" +) + +type Global struct { + Interval time.Duration `json:"interval,omitempty" yaml:"interval,omitempty"` + Timeout time.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` +} + +type Config struct { + Global Global `json:"global,omitempty" yaml:"global,omitempty"` + Quotas []quotas.Config `json:"quotas,omitempty" yaml:"quotas,omitempty"` + Metrics []script.Config `json:"metrics,omitempty" yaml:"metrics,omitempty"` +} + +func (c *Config) Validate() error { + for _, q := range c.Quotas { + if err := q.Validate(); err != nil { + return err + } + } + for _, m := range c.Metrics { + if err := m.Validate(); err != nil { + return err + } + } + return nil +} + +func LoadAndValidateConfig(path string) (*Config, error) { + cfg := Config{} + err := config.ParseYamlFromFile(path, &cfg) + if err != nil { + return nil, fmt.Errorf("unable to parse metric scrape from file: %w", err) + } + err = cfg.Validate() + if err != nil { + return nil, err + } + return &cfg, nil +} + +type Validator interface { + Validate() error +} diff --git a/internal/scrape/quotas/collector.go b/internal/scrape/quotas/collector.go new file mode 100644 index 0000000..9897e51 --- /dev/null +++ b/internal/scrape/quotas/collector.go @@ -0,0 +1,69 @@ +package quotas + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/servicequotas/types" + "github.com/lablabs/aws-service-quotas-exporter/pkg/quota" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +const ( + name = "name" + code = "code" + serviceCode = "service_code" +) + +type Quota interface { + GetQuota(ctx context.Context, serviceCode string, quotaCode string, options ...quota.Option) (*types.ServiceQuota, error) +} + +func NewCollector(log *logrus.Logger, cfg []Config, ns string, qcl Quota) (*Collector, error) { + gvq := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ns, + Name: "quota", + Help: "AWS service quota", + }, []string{name, code, serviceCode}) + cl := Collector{ + log: log, + qcl: qcl, + gvq: gvq, + cfg: cfg, + } + return &cl, nil +} + +type Collector struct { + log *logrus.Logger + qcl Quota + gvq *prometheus.GaugeVec + cfg []Config +} + +func (c *Collector) Register(r *prometheus.Registry) error { + r.MustRegister(c.gvq) + return nil +} + +func (c *Collector) Collect(g *errgroup.Group, ctx context.Context) { + for _, q := range c.cfg { + g.Go(func() error { + res, err := c.qcl.GetQuota(ctx, q.ServiceCode, q.QuotaCode, quota.WithRegion(q.Region)) + if err != nil { + return err + } + setMetric(c.gvq, res) + return nil + }) + } +} + +func setMetric(gc *prometheus.GaugeVec, q *types.ServiceQuota) { + gc.With(prometheus.Labels{ + name: aws.ToString(q.QuotaName), + code: aws.ToString(q.QuotaCode), + serviceCode: aws.ToString(q.ServiceCode), + }).Set(aws.ToFloat64(q.Value)) +} diff --git a/internal/scrape/quotas/config.go b/internal/scrape/quotas/config.go new file mode 100644 index 0000000..5f853ae --- /dev/null +++ b/internal/scrape/quotas/config.go @@ -0,0 +1,23 @@ +package quotas + +import ( + "fmt" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" +) + +type Config struct { + ServiceCode string `json:"serviceCode,omitempty" yaml:"serviceCode,omitempty"` + QuotaCode string `json:"quotaCode,omitempty" yaml:"quotaCode,omitempty"` + Region string `json:"region,omitempty" yaml:"region,omitempty"` + Usage script.Config `json:"usage,omitempty" yaml:"usage,omitempty"` +} + +func (c Config) Validate() error { + if c.ServiceCode == "" { + return fmt.Errorf("serviceCode must not be empty") + } + if c.QuotaCode == "" { + return fmt.Errorf("quotaCode must not be empty") + } + return nil +} diff --git a/internal/scrape/script/collector.go b/internal/scrape/script/collector.go new file mode 100644 index 0000000..5c630f5 --- /dev/null +++ b/internal/scrape/script/collector.go @@ -0,0 +1,62 @@ +package script + +import ( + "context" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +func NewCollector(log *logrus.Logger, cfg []Config, ns string) (*Collector, error) { + + tks := make([]task, 0) + for _, c := range cfg { + m := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ns, + Name: c.Name, + Help: c.Help, + }, c.LabelNames()) + tks = append(tks, task{ + m: m, + cfg: c, + }) + } + + cl := Collector{ + log: log, + tasks: tks, + } + return &cl, nil +} + +type Collector struct { + log *logrus.Logger + tasks []task +} + +func (c *Collector) Register(r *prometheus.Registry) error { + for _, t := range c.tasks { + r.MustRegister(t.m) + } + return nil +} + +func (c *Collector) Collect(g *errgroup.Group, ctx context.Context) { + for _, t := range c.tasks { + g.Go(func() error { + data, err := Run(ctx, t.cfg) + if err != nil { + return err + } + for _, d := range data { + t.m.With(d.Labels).Set(d.Value) + } + return nil + }) + } +} + +type task struct { + m *prometheus.GaugeVec + cfg Config +} diff --git a/internal/scrape/script/config.go b/internal/scrape/script/config.go new file mode 100644 index 0000000..ef53813 --- /dev/null +++ b/internal/scrape/script/config.go @@ -0,0 +1,60 @@ +package script + +import "fmt" + +type Config struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Help string `json:"help,omitempty" yaml:"help,omitempty"` + Command string `json:"command,omitempty" yaml:"command,omitempty"` + Envs []Env `json:"env,omitempty" yaml:"envs,omitempty"` + List string `json:"list,omitempty" yaml:"list,omitempty"` + Value string `json:"value,omitempty" yaml:"value,omitempty"` + Labels []Label `json:"labels,omitempty" yaml:"labels,omitempty"` +} + +type Env struct { + Name string `json:"name,omitempty" yaml:"name"` + Value string `json:"value,omitempty" yaml:"value"` +} + +func (c *Config) LabelNames() []string { + l := make([]string, 0) + for _, c := range c.Labels { + l = append(l, c.Name) + } + return l +} + +func (c *Config) Validate() error { + if c.Name == "" { + return fmt.Errorf("name attribute is required") + } + if c.Command == "" { + return fmt.Errorf("command attribute is required") + } + if c.Value == "" { + return fmt.Errorf("value attribute is required") + } + for _, l := range c.Labels { + if l.Name == "" { + return fmt.Errorf("attribute name for label is required") + } + if l.JqValue == "" { + return fmt.Errorf("jqValue for label is required") + } + } + return nil +} + +func (c *Config) FormatEnvs() []string { + evs := make([]string, 0) + for _, e := range c.Envs { + evs = append(evs, fmt.Sprintf("%s=%s", e.Name, e.Value)) + } + return evs +} + +type Label struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + JqValue string `json:"jqValue,omitempty" yaml:"jqValue,omitempty"` +} diff --git a/internal/scrape/script/script.go b/internal/scrape/script/script.go new file mode 100644 index 0000000..6613cce --- /dev/null +++ b/internal/scrape/script/script.go @@ -0,0 +1,138 @@ +package script + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/ohler55/ojg/jp" + "os" + "os/exec" + "strings" +) + +type Data struct { + Value float64 + Labels map[string]string +} + +func Run(ctx context.Context, cfg Config) ([]Data, error) { + + cs := strings.Split(cfg.Command, " ") + prg := cs[0] + args := cs[1:] + cmd := exec.CommandContext(ctx, prg, args...) + var stdout, stderr bytes.Buffer + envs := make([]string, 0) + envs = append(envs, os.Environ()...) + envs = append(envs, cfg.FormatEnvs()...) + cmd.Env = envs + cmd.Stderr = &stderr + cmd.Stdout = &stdout + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("script error: %w, std err: %s", err, stderr.String()) + } + data, err := ParseJson(stdout.Bytes()) + if err != nil { + return nil, fmt.Errorf("unable to parse response from command: %v", cfg.Command) + } + + result := make([]Data, 0) + if cfg.List != "" { + items, err := GetArray(data, cfg.List) + if err != nil { + return nil, fmt.Errorf("unable to parse list items jq: %w", err) + } + for _, it := range items { + r, err := ParseRecord(it, cfg) + if err != nil { + return nil, err + } + result = append(result, r) + } + return result, nil + } + r, err := ParseRecord(data, cfg) + if err != nil { + return nil, err + } + result = append(result, r) + return result, nil +} + +func ParseRecord(r any, c Config) (Data, error) { + v, err := GetFloat64(r, c.Value) + if err != nil { + return Data{}, err + } + labels := make(map[string]string) + for _, l := range c.Labels { + lv, err := GetString(r, l.JqValue) + if err != nil { + return Data{}, err + } + labels[l.Name] = lv + } + return Data{ + Value: v, + Labels: labels, + }, nil +} + +func ParseJson(data []byte) (map[string]interface{}, error) { + var out map[string]interface{} + err := json.Unmarshal(data, &out) + if err != nil { + return nil, fmt.Errorf("unable parse to json: %w", err) + } + return out, nil +} + +func GetFloat64(data any, query string) (float64, error) { + res, err := ParseJq(data, query) + if err != nil { + return 0, err + } + v, ok := res[0].(float64) + if !ok { + return 0, fmt.Errorf("invalid data. jq expression must return valid float64") + } + return v, nil +} + +func GetString(data any, query string) (string, error) { + res, err := ParseJq(data, query) + if err != nil { + return "", err + } + v, ok := res[0].(string) + if !ok { + return "", fmt.Errorf("invalid data. jq expression must return valid string") + } + return v, nil +} + +func GetArray(data any, query string) ([]any, error) { + res, err := ParseJq(data, query) + if err != nil { + return nil, err + } + v, ok := res[0].([]interface{}) + if !ok { + return v, fmt.Errorf("invalid data. jq expression must return valid string") + } + return v, nil +} + +func ParseJq(data any, qs string) ([]any, error) { + q, err := jp.ParseString(qs) + if err != nil { + return nil, fmt.Errorf("unable to parse jq selector: %w", err) + } + result := q.Get(data) + if len(result) != 1 { + return nil, fmt.Errorf("empty data returned from jq expression: %s", qs) + } + return result, nil +} diff --git a/internal/scrape/script/script_test.go b/internal/scrape/script/script_test.go new file mode 100644 index 0000000..907f629 --- /dev/null +++ b/internal/scrape/script/script_test.go @@ -0,0 +1,106 @@ +package script + +import ( + "context" + "reflect" + "testing" + "time" +) + +func TestScrapper_Run(t *testing.T) { + type fields struct { + cfg Config + } + tests := []struct { + name string + fields fields + want []Data + wantErr bool + }{ + { + name: "Scrape items OK", + fields: fields{ + cfg: Config{ + Command: "echo {\"items\":[{\"v\": 1,\"name\":\"a\"},{\"v\":2,\"name\":\"b\"}]}", + List: ".items", + Value: ".v", + Labels: []Label{ + { + Name: "name", + JqValue: ".name", + }, + }, + }, + }, + want: []Data{ + { + Value: 1, + Labels: map[string]string{ + "name": "a", + }, + }, + { + Value: 2, + Labels: map[string]string{ + "name": "b", + }, + }, + }, + wantErr: false, + }, + { + name: "Scrape one value OK", + fields: fields{ + cfg: Config{ + Command: "echo {\"v\": 1,\"name\":\"a\"}", + Value: ".v", + Labels: []Label{ + { + Name: "name", + JqValue: ".name", + }, + }, + }, + }, + want: []Data{ + { + Value: 1, + Labels: map[string]string{ + "name": "a", + }, + }, + }, + wantErr: false, + }, + { + name: "Scrape cmd Json Error", + fields: fields{ + cfg: Config{ + Command: "echo not json", + Value: ".v", + Labels: []Label{ + { + Name: "name", + JqValue: ".name", + }, + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + got, err := Run(ctx, tt.fields.cfg) + cancel() + if (err != nil) != tt.wantErr { + t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Run() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/config/yaml.go b/pkg/config/yaml.go new file mode 100644 index 0000000..e50586b --- /dev/null +++ b/pkg/config/yaml.go @@ -0,0 +1,24 @@ +package config + +import ( + "fmt" + "gopkg.in/yaml.v3" + "os" +) + +// ParseYaml bytes to scrape struct +func ParseYaml(data []byte, c interface{}) error { + err := yaml.Unmarshal(data, c) + if err != nil { + return fmt.Errorf("invalid yaml config: %w", err) + } + return nil +} + +func ParseYamlFromFile(path string, c interface{}) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("unable to read file: %w", err) + } + return ParseYaml(data, c) +} diff --git a/pkg/flags/parse.go b/pkg/flags/parse.go new file mode 100644 index 0000000..8619bbc --- /dev/null +++ b/pkg/flags/parse.go @@ -0,0 +1,21 @@ +package flags + +import ( + "github.com/jessevdk/go-flags" + "os" +) + +func ParseFlags(config interface{}, args []string) error { + _, err := flags.NewParser(config, flags.Default).ParseArgs(args) + if err != nil { + return err + } + return nil +} + +func ParseOrFail(config interface{}, args []string) { + err := ParseFlags(config, args) + if err != nil { + os.Exit(1) + } +} diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..a72c0b4 --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,39 @@ +package log + +import ( + log "github.com/sirupsen/logrus" + "os" +) + +func DefaultLoggerOrFail() *log.Logger { + logger := NewLoggerOrFail("json", "DEBUG") + return logger +} + +func NewLoggerOrFail(format string, level string) *log.Logger { + log, err := NewLogger(format, level) + if err != nil { + panic(err) + } + return log +} + +func NewLogger(format string, logLevel string) (*log.Logger, error) { + logger := log.New() + level, err := log.ParseLevel(logLevel) + if err != nil { + return nil, err + } + switch format { + case "json": + logger.SetFormatter(&log.JSONFormatter{}) + case "text": + logger.SetFormatter(&log.TextFormatter{ + DisableColors: true, + FullTimestamp: true, + }) + } + logger.SetOutput(os.Stdout) + logger.SetLevel(level) + return logger, nil +} diff --git a/pkg/quota/client.go b/pkg/quota/client.go new file mode 100644 index 0000000..58aa1ea --- /dev/null +++ b/pkg/quota/client.go @@ -0,0 +1,63 @@ +package quota + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/servicequotas" + "github.com/aws/aws-sdk-go-v2/service/servicequotas/types" + "github.com/sirupsen/logrus" +) + +func NewClient(log *logrus.Logger) (*Client, error) { + + cfg, err := awsConfig.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, err + } + squ := servicequotas.NewFromConfig(cfg) + c := Client{ + log: log, + squ: squ, + } + return &c, nil +} + +type Client struct { + log *logrus.Logger + squ *servicequotas.Client +} + +func (c *Client) GetQuota(ctx context.Context, serviceCode string, quotaCode string, options ...Option) (*types.ServiceQuota, error) { + res, err := c.squ.GetServiceQuota(ctx, &servicequotas.GetServiceQuotaInput{ + QuotaCode: aws.String(quotaCode), + ServiceCode: aws.String(serviceCode), + }, buildOptions(options...)) + if err != nil { + return nil, fmt.Errorf("unable to get quota with service: %s, code: %s, %w", serviceCode, quotaCode, err) + } + return res.Quota, nil +} + +func (c *Client) GetQuotas(ctx context.Context, serviceCode string, options ...Option) ([]types.ServiceQuota, error) { + qs := make([]types.ServiceQuota, 0) + var token *string = nil + for { + res, err := c.squ.ListServiceQuotas(ctx, &servicequotas.ListServiceQuotasInput{ + ServiceCode: aws.String(serviceCode), + NextToken: token, + }, buildOptions(options...)) + if err != nil { + return nil, err + } + for _, q := range res.Quotas { + qs = append(qs, q) + } + if res.NextToken == nil { + return qs, nil + } + token = res.NextToken + } + return qs, nil +} diff --git a/pkg/quota/options.go b/pkg/quota/options.go new file mode 100644 index 0000000..34844c9 --- /dev/null +++ b/pkg/quota/options.go @@ -0,0 +1,28 @@ +package quota + +import "github.com/aws/aws-sdk-go-v2/service/servicequotas" + +type Options struct { + *servicequotas.Options +} + +type Option func(c *Options) + +func WithRegion(region string) Option { + return func(c *Options) { + if region != "" { + c.Region = region + } + } +} + +func buildOptions(option ...Option) func(*servicequotas.Options) { + return func(awsSq *servicequotas.Options) { + op := Options{ + awsSq, + } + for _, o := range option { + o(&op) + } + } +} diff --git a/pkg/service/ctx.go b/pkg/service/ctx.go new file mode 100644 index 0000000..0532216 --- /dev/null +++ b/pkg/service/ctx.go @@ -0,0 +1,13 @@ +package service + +import ( + "context" + "os" + "os/signal" + "syscall" +) + +func SignContext() context.Context { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) + return ctx +} diff --git a/pkg/service/mng.go b/pkg/service/mng.go new file mode 100644 index 0000000..d73abad --- /dev/null +++ b/pkg/service/mng.go @@ -0,0 +1,42 @@ +package service + +import ( + "context" + "golang.org/x/sync/errgroup" +) + +func NewManager() (*Manager, error) { + m := Manager{ + services: make([]Starter, 0), + } + return &m, nil +} + +type Starter interface { + // Run controller and block until finish + Run(ctx context.Context) error +} + +type Manager struct { + services []Starter +} + +func (m *Manager) Add(s Starter) { + m.services = append(m.services, s) +} + +func (m *Manager) StartAndWait(ctx context.Context) error { + group, ctx := errgroup.WithContext(ctx) + for _, s := range m.services { + service := s + group.Go(func() error { + err := service.Run(ctx) + if err != nil { + return err + } + <-ctx.Done() + return err + }) + } + return group.Wait() +} diff --git a/test/config.go b/test/config.go new file mode 100644 index 0000000..40c119a --- /dev/null +++ b/test/config.go @@ -0,0 +1,19 @@ +package test + +import ( + _ "embed" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +//go:embed configs/eip.yaml +var configEip string + +func TmpConfigMetricFile(t *testing.T) string { + f, err := os.CreateTemp(os.TempDir(), "exporter") + assert.NoError(t, err) + _, err = f.WriteString(configEip) + assert.NoError(t, err) + return f.Name() +} diff --git a/test/configs/eip.yaml b/test/configs/eip.yaml new file mode 100644 index 0000000..a3a98ac --- /dev/null +++ b/test/configs/eip.yaml @@ -0,0 +1,14 @@ +quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" +metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + command: "aws route53 list-hosted-zones" + list: ".HostedZones" + value: ".ResourceRecordSetCount" + labels: + - name: "id" + jqValue: ".Id" + - name: "name" + jqValue: ".Name" \ No newline at end of file diff --git a/test/log.go b/test/log.go new file mode 100644 index 0000000..361c9e6 --- /dev/null +++ b/test/log.go @@ -0,0 +1,16 @@ +package test + +import ( + logUtil "github.com/lablabs/aws-service-quotas-exporter/pkg/log" + log "github.com/sirupsen/logrus" +) + +const ( + debugLevel = "DEBUG" + format = "json " +) + +func DefaultLogger() *log.Logger { + logger := logUtil.NewLoggerOrFail(format, debugLevel) + return logger +} From 7a689ef0dbbc794352def49092470b0ca3b20432 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Fri, 8 Mar 2024 10:55:18 +0100 Subject: [PATCH 02/32] lint fix --- .golangci.yaml | 14 ++++++++++++++ internal/app/application.go | 10 ++++++++-- internal/exporter/exporter.go | 2 +- internal/http/http.go | 4 ++-- internal/scrape/quotas/collector.go | 20 ++++++++++++-------- internal/scrape/script/collector.go | 24 ++++++++++++++---------- pkg/quota/client.go | 6 ++---- 7 files changed, 53 insertions(+), 27 deletions(-) create mode 100644 .golangci.yaml diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..a0b1a1e --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,14 @@ +linters: + enable: + - megacheck + - gofmt + - govet + - revive # replacement for golint + +linters-settings: + staticcheck: + checks: + - '-SA5008' + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true \ No newline at end of file diff --git a/internal/app/application.go b/internal/app/application.go index 980a919..f3b365d 100644 --- a/internal/app/application.go +++ b/internal/app/application.go @@ -38,14 +38,20 @@ func NewApplication(log *logrus.Logger, cfg Config) (*Application, error) { if err != nil { return nil, err } - qcl.Register(registry) + err = qcl.Register(registry) + if err != nil { + return nil, err + } cls = append(cls, qcl) scl, err := script.NewCollector(log, scCfg.Metrics, PrometheusNamespace) if err != nil { return nil, err } - scl.Register(registry) + err = scl.Register(registry) + if err != nil { + return nil, err + } cls = append(cls, scl) exp, err := exporter.NewExporter(log, cls, exporterOptions(scCfg)...) diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index a7ef518..5df582b 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -54,7 +54,7 @@ func (e *Exporter) Run(ctx context.Context) error { for { select { case <-ctx.Done(): - return nil + break case <-ticker.C: err := e.scrape(ctx) if err != nil { diff --git a/internal/http/http.go b/internal/http/http.go index 3128c07..ff1a42e 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -25,7 +25,7 @@ func NewHttp(log *logrus.Logger, address string, registry *prometheus.Registry) h := Http{ log: log, ln: ln, - s: s, + s: &s, } RegisterMetricEndpoint(handler, registry) return &h, nil @@ -34,7 +34,7 @@ func NewHttp(log *logrus.Logger, address string, registry *prometheus.Registry) type Http struct { log *logrus.Logger ln net.Listener - s http.Server + s *http.Server } func (h *Http) Run(ctx context.Context) error { diff --git a/internal/scrape/quotas/collector.go b/internal/scrape/quotas/collector.go index 9897e51..1633c38 100644 --- a/internal/scrape/quotas/collector.go +++ b/internal/scrape/quotas/collector.go @@ -49,14 +49,18 @@ func (c *Collector) Register(r *prometheus.Registry) error { func (c *Collector) Collect(g *errgroup.Group, ctx context.Context) { for _, q := range c.cfg { - g.Go(func() error { - res, err := c.qcl.GetQuota(ctx, q.ServiceCode, q.QuotaCode, quota.WithRegion(q.Region)) - if err != nil { - return err - } - setMetric(c.gvq, res) - return nil - }) + g.Go(c.run(ctx, q)) + } +} + +func (c *Collector) run(ctx context.Context, q Config) func() error { + return func() error { + res, err := c.qcl.GetQuota(ctx, q.ServiceCode, q.QuotaCode, quota.WithRegion(q.Region)) + if err != nil { + return err + } + setMetric(c.gvq, res) + return nil } } diff --git a/internal/scrape/script/collector.go b/internal/scrape/script/collector.go index 5c630f5..103d582 100644 --- a/internal/scrape/script/collector.go +++ b/internal/scrape/script/collector.go @@ -43,16 +43,7 @@ func (c *Collector) Register(r *prometheus.Registry) error { func (c *Collector) Collect(g *errgroup.Group, ctx context.Context) { for _, t := range c.tasks { - g.Go(func() error { - data, err := Run(ctx, t.cfg) - if err != nil { - return err - } - for _, d := range data { - t.m.With(d.Labels).Set(d.Value) - } - return nil - }) + g.Go(t.run(ctx)) } } @@ -60,3 +51,16 @@ type task struct { m *prometheus.GaugeVec cfg Config } + +func (t task) run(ctx context.Context) func() error { + return func() error { + data, err := Run(ctx, t.cfg) + if err != nil { + return err + } + for _, d := range data { + t.m.With(d.Labels).Set(d.Value) + } + return nil + } +} diff --git a/pkg/quota/client.go b/pkg/quota/client.go index 58aa1ea..0e5b66a 100644 --- a/pkg/quota/client.go +++ b/pkg/quota/client.go @@ -51,11 +51,9 @@ func (c *Client) GetQuotas(ctx context.Context, serviceCode string, options ...O if err != nil { return nil, err } - for _, q := range res.Quotas { - qs = append(qs, q) - } + qs = append(qs, res.Quotas...) if res.NextToken == nil { - return qs, nil + break } token = res.NextToken } From e706b4c9fd057b37b12651e9bbdc5e2c95148f05 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Fri, 8 Mar 2024 13:43:05 +0100 Subject: [PATCH 03/32] go lint --- .golangci.yaml | 2 +- internal/app/application.go | 2 +- internal/exporter/exporter.go | 7 ++++--- internal/exporter/exporter_test.go | 4 ++-- internal/http/http.go | 8 ++++---- internal/http/http_test.go | 15 +++++++++------ internal/scrape/quotas/collector.go | 2 +- internal/scrape/script/collector.go | 2 +- internal/scrape/script/script.go | 4 ++-- pkg/quota/client.go | 2 +- 10 files changed, 26 insertions(+), 22 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index a0b1a1e..8032a42 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -3,7 +3,7 @@ linters: - megacheck - gofmt - govet - - revive # replacement for golint + - revive linters-settings: staticcheck: diff --git a/internal/app/application.go b/internal/app/application.go index f3b365d..74a8e28 100644 --- a/internal/app/application.go +++ b/internal/app/application.go @@ -60,7 +60,7 @@ func NewApplication(log *logrus.Logger, cfg Config) (*Application, error) { } mng.Add(exp) - http, err := http.NewHttp(log, cfg.Address, registry) + http, err := http.NewHTTP(log, cfg.Address, registry) if err != nil { return nil, err } diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index 5df582b..43c239f 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -11,7 +11,7 @@ import ( type Collector interface { Register(r *prometheus.Registry) error - Collect(g *errgroup.Group, ctx context.Context) + Collect(ctx context.Context, g *errgroup.Group) } const ( @@ -51,10 +51,11 @@ func (e *Exporter) Run(ctx context.Context) error { ticker := time.NewTicker(e.cfg.interval) e.log.Debugf("scrape metrics every: %v", e.cfg.interval) defer ticker.Stop() +end: for { select { case <-ctx.Done(): - break + break end case <-ticker.C: err := e.scrape(ctx) if err != nil { @@ -71,7 +72,7 @@ func (e *Exporter) scrape(ctx context.Context) error { defer cancel() g, ctx := errgroup.WithContext(ctx) for _, c := range e.cls { - c.Collect(g, ctx) + c.Collect(ctx, g) } err := g.Wait() return err diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go index 0fb9796..22d455d 100644 --- a/internal/exporter/exporter_test.go +++ b/internal/exporter/exporter_test.go @@ -68,11 +68,11 @@ type testCollector struct { err error } -func (t *testCollector) Register(r *prometheus.Registry) error { +func (t *testCollector) Register(_ *prometheus.Registry) error { return nil } -func (t *testCollector) Collect(g *errgroup.Group, ctx context.Context) { +func (t *testCollector) Collect(_ context.Context, g *errgroup.Group) { g.Go(func() error { return t.err }) diff --git a/internal/http/http.go b/internal/http/http.go index ff1a42e..9688a6a 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -13,7 +13,7 @@ import ( "net/http" ) -func NewHttp(log *logrus.Logger, address string, registry *prometheus.Registry) (*Http, error) { +func NewHTTP(log *logrus.Logger, address string, registry *prometheus.Registry) (*HTTP, error) { ln, err := net.Listen("tcp", address) if err != nil { return nil, err @@ -22,7 +22,7 @@ func NewHttp(log *logrus.Logger, address string, registry *prometheus.Registry) s := http.Server{ Handler: handler, } - h := Http{ + h := HTTP{ log: log, ln: ln, s: &s, @@ -31,13 +31,13 @@ func NewHttp(log *logrus.Logger, address string, registry *prometheus.Registry) return &h, nil } -type Http struct { +type HTTP struct { log *logrus.Logger ln net.Listener s *http.Server } -func (h *Http) Run(ctx context.Context) error { +func (h *HTTP) Run(ctx context.Context) error { h.log.Infof("start http endpoint: %s", h.ln.Addr()) g, ctx := errgroup.WithContext(ctx) g.Go(func() error { diff --git a/internal/http/http_test.go b/internal/http/http_test.go index b822396..367c283 100644 --- a/internal/http/http_test.go +++ b/internal/http/http_test.go @@ -11,21 +11,24 @@ import ( "testing" ) +const ( + address = "0.0.0.0:8080" +) + func TestNewHttp(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - address, close := StartHttp(t, ctx) + closeHTTP := StartHTTP(ctx, t) defer func() { cancel() - close() + closeHTTP() }() resp, err := http.Get("http://" + address + "/metrics") assert.NoError(t, err) assert.Equal(t, http.StatusOK, resp.StatusCode) } -func StartHttp(t *testing.T, ctx context.Context) (string, func()) { - address := "0.0.0.0:8080" - http, err := httpApi.NewHttp(test.DefaultLogger(), address, prometheus.NewRegistry()) +func StartHTTP(ctx context.Context, t *testing.T) func() { + http, err := httpApi.NewHTTP(test.DefaultLogger(), address, prometheus.NewRegistry()) assert.NoError(t, err) var wg sync.WaitGroup wg.Add(1) @@ -34,7 +37,7 @@ func StartHttp(t *testing.T, ctx context.Context) (string, func()) { err := http.Run(ctx) assert.NoError(t, err) }() - return address, func() { + return func() { wg.Wait() } } diff --git a/internal/scrape/quotas/collector.go b/internal/scrape/quotas/collector.go index 1633c38..29ce414 100644 --- a/internal/scrape/quotas/collector.go +++ b/internal/scrape/quotas/collector.go @@ -47,7 +47,7 @@ func (c *Collector) Register(r *prometheus.Registry) error { return nil } -func (c *Collector) Collect(g *errgroup.Group, ctx context.Context) { +func (c *Collector) Collect(ctx context.Context, g *errgroup.Group) { for _, q := range c.cfg { g.Go(c.run(ctx, q)) } diff --git a/internal/scrape/script/collector.go b/internal/scrape/script/collector.go index 103d582..b58c595 100644 --- a/internal/scrape/script/collector.go +++ b/internal/scrape/script/collector.go @@ -41,7 +41,7 @@ func (c *Collector) Register(r *prometheus.Registry) error { return nil } -func (c *Collector) Collect(g *errgroup.Group, ctx context.Context) { +func (c *Collector) Collect(ctx context.Context, g *errgroup.Group) { for _, t := range c.tasks { g.Go(t.run(ctx)) } diff --git a/internal/scrape/script/script.go b/internal/scrape/script/script.go index 6613cce..23c1251 100644 --- a/internal/scrape/script/script.go +++ b/internal/scrape/script/script.go @@ -33,7 +33,7 @@ func Run(ctx context.Context, cfg Config) ([]Data, error) { if err != nil { return nil, fmt.Errorf("script error: %w, std err: %s", err, stderr.String()) } - data, err := ParseJson(stdout.Bytes()) + data, err := ParseJSON(stdout.Bytes()) if err != nil { return nil, fmt.Errorf("unable to parse response from command: %v", cfg.Command) } @@ -80,7 +80,7 @@ func ParseRecord(r any, c Config) (Data, error) { }, nil } -func ParseJson(data []byte) (map[string]interface{}, error) { +func ParseJSON(data []byte) (map[string]interface{}, error) { var out map[string]interface{} err := json.Unmarshal(data, &out) if err != nil { diff --git a/pkg/quota/client.go b/pkg/quota/client.go index 0e5b66a..65e9287 100644 --- a/pkg/quota/client.go +++ b/pkg/quota/client.go @@ -42,7 +42,7 @@ func (c *Client) GetQuota(ctx context.Context, serviceCode string, quotaCode str func (c *Client) GetQuotas(ctx context.Context, serviceCode string, options ...Option) ([]types.ServiceQuota, error) { qs := make([]types.ServiceQuota, 0) - var token *string = nil + var token *string for { res, err := c.squ.ListServiceQuotas(ctx, &servicequotas.ListServiceQuotasInput{ ServiceCode: aws.String(serviceCode), From 24de7026a4f337bca468fcdf06d30bfc6ebe4c06 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 07:37:20 +0100 Subject: [PATCH 04/32] image build and Dockerfile --- .github/workflows/docker-master.yaml | 58 ++++++++++++++++++++++++++++ .github/workflows/go-build.yml | 26 +++++++++++++ Dockerfile | 39 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 .github/workflows/docker-master.yaml create mode 100644 .github/workflows/go-build.yml create mode 100644 Dockerfile diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml new file mode 100644 index 0000000..d1ee4de --- /dev/null +++ b/.github/workflows/docker-master.yaml @@ -0,0 +1,58 @@ +name: Build and Publish Docker Images after push to master branch +#on: +# push: +# branches: [master] +on: + push: + pull_request: + +jobs: + build-and-push-docker-image: + name: Build Docker image and push to repositories with tag latest + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + lablabs/aws-service-quotas-exporter + ghcr.io/lablabs/aws-service-quotas-exporter + # generate Docker tags based on the following events/attributes + tags: type=raw,value=latest + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Github Packages + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build image and push to Docker Hub and GitHub Container Registry + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.github/workflows/go-build.yml b/.github/workflows/go-build.yml new file mode 100644 index 0000000..ff063bd --- /dev/null +++ b/.github/workflows/go-build.yml @@ -0,0 +1,26 @@ +name: Generate release-artifacts +on: + release: + types: + - created +jobs: + generate: + name: Generate cross-platform builds + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + - name: Generate build files + uses: thatisuday/go-cross-build@v1 + with: + platforms: 'linux/amd64, darwin/amd64, linux/arm64' + package: '' + name: 'aws-service-quotas-exporter' + compress: 'false' + dest: 'dist' + - name: Copy build-artifacts + uses: skx/github-action-publish-binaries@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + args: "./dist/*" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3b65c27 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM golang:1.22-alpine3.19 as builder + +ARG GOOS=linux +ARG GOARCH=amd64 + +RUN apk --update add ca-certificates + +WORKDIR $GOPATH/src/github.com/lablabs/aws-service-quotas-exporter +COPY go.mod go.sum ./ +COPY . . +RUN go mod download +RUN go mod vendor +RUN go mod verify + +RUN cd cmd/exporter && \ + GOOS=$GOOS GOARCH=$GOARCH \ + CGO_ENABLED=0 \ + go build -o /aws-service-quotas-exporter . + +FROM alpine:3.19.1 as security + +RUN apk add -U --no-cache \ + tzdata \ + ca-certificates + +RUN addgroup -S nonroot \ + && adduser -S nonroot -G nonroot + +FROM scratch + +COPY --from=security /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=security /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=security /etc/passwd /etc/passwd +COPY --from=security /etc/group /etc/group + +COPY --from=builder /aws-service-quotas-exporter . + +ENTRYPOINT ["/aws-service-quotas-exporter"] + From c8b6203c6d069303f153345028766a132a0ad98f Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 07:49:35 +0100 Subject: [PATCH 05/32] test Dockerfile --- .github/workflows/docker-master.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index d1ee4de..77cb690 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -26,16 +26,16 @@ jobs: uses: docker/metadata-action@v5 with: images: | - lablabs/aws-service-quotas-exporter +# lablabs/aws-service-quotas-exporter ghcr.io/lablabs/aws-service-quotas-exporter # generate Docker tags based on the following events/attributes tags: type=raw,value=latest - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} +# - name: Login to DockerHub +# uses: docker/login-action@v3 +# with: +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Github Packages uses: docker/login-action@v3 From 5c99e468e21b9f1f64ff5d664926aa2be17edc36 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 07:51:01 +0100 Subject: [PATCH 06/32] fix typo --- .github/workflows/docker-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index 77cb690..bd61e1b 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -26,8 +26,8 @@ jobs: uses: docker/metadata-action@v5 with: images: | -# lablabs/aws-service-quotas-exporter ghcr.io/lablabs/aws-service-quotas-exporter +# lablabs/aws-service-quotas-exporter # generate Docker tags based on the following events/attributes tags: type=raw,value=latest From 7845bc300abf8d3e7e0a02aae2774563c92d877a Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 09:00:28 +0100 Subject: [PATCH 07/32] test build --- .github/workflows/docker-master.yaml | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index bd61e1b..542b290 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -11,24 +11,23 @@ jobs: name: Build Docker image and push to repositories with tag latest runs-on: ubuntu-latest steps: - - name: Checkout code + - + name: Checkout code uses: actions/checkout@v4 - - - name: Set up QEMU + - + name: Set up QEMU uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx + - + name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - - - name: Docker meta + - + name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ghcr.io/lablabs/aws-service-quotas-exporter -# lablabs/aws-service-quotas-exporter - # generate Docker tags based on the following events/attributes tags: type=raw,value=latest # - name: Login to DockerHub @@ -37,22 +36,22 @@ jobs: # username: ${{ secrets.DOCKERHUB_USERNAME }} # password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to Github Packages + - + name: Login to Github Packages uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build image and push to Docker Hub and GitHub Container Registry - uses: docker/build-push-action@v2 + - + name: Build and push + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - push: true - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file +# +# - name: Image digest +# run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file From 15843586938f4434faa22e00296f72f5e078aba9 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 09:13:59 +0100 Subject: [PATCH 08/32] push true --- .github/workflows/docker-master.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index 542b290..4378296 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -48,8 +48,10 @@ jobs: uses: docker/build-push-action@v5 with: context: . + provenance: false file: ./Dockerfile platforms: linux/amd64,linux/arm64 + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} # From 5900fbb4f2f210b7b173052c4eca4758d1bca465 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 11:47:25 +0100 Subject: [PATCH 09/32] prepare helm chart --- .github/workflows/docker-master.yaml | 6 +- .github/workflows/docker-release.yaml | 58 +++++++++ .github/workflows/helm-release.yml | 36 ++++++ .github/workflows/helm-test.yml | 45 +++++++ .tool-versions | 1 - .../aws-service-quotas-exporter/.helmignore | 23 ++++ charts/aws-service-quotas-exporter/Chart.yaml | 9 ++ .../aws-service-quotas-exporter-values.yaml | 18 +++ .../templates/_helpers.tpl | 62 +++++++++ .../templates/deployment.yaml | 82 ++++++++++++ .../templates/scrape-config.yaml | 10 ++ .../templates/service.yaml | 15 +++ .../templates/serviceaccount.yaml | 13 ++ .../templates/servicemonitor.yaml | 47 +++++++ .../aws-service-quotas-exporter/values.yaml | 122 ++++++++++++++++++ config/example.yaml | 24 ++-- internal/app/config.go | 2 +- internal/scrape/script/config.go | 15 ++- internal/scrape/script/script.go | 14 +- 19 files changed, 580 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/docker-release.yaml create mode 100644 .github/workflows/helm-release.yml create mode 100644 .github/workflows/helm-test.yml create mode 100644 charts/aws-service-quotas-exporter/.helmignore create mode 100644 charts/aws-service-quotas-exporter/Chart.yaml create mode 100644 charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml create mode 100644 charts/aws-service-quotas-exporter/templates/_helpers.tpl create mode 100644 charts/aws-service-quotas-exporter/templates/deployment.yaml create mode 100644 charts/aws-service-quotas-exporter/templates/scrape-config.yaml create mode 100644 charts/aws-service-quotas-exporter/templates/service.yaml create mode 100644 charts/aws-service-quotas-exporter/templates/serviceaccount.yaml create mode 100644 charts/aws-service-quotas-exporter/templates/servicemonitor.yaml create mode 100644 charts/aws-service-quotas-exporter/values.yaml diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index 4378296..a0693c3 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -54,6 +54,6 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} -# -# - name: Image digest -# run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml new file mode 100644 index 0000000..6ed4021 --- /dev/null +++ b/.github/workflows/docker-release.yaml @@ -0,0 +1,58 @@ +name: Build and Publish Docker Images after create release tag + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + build-and-push-docker-image: + name: Build Docker image and push to repositories with tag + runs-on: ubuntu-latest + steps: + - + name: Checkout code + uses: actions/checkout@v4 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + tags: type=ref,event=tag + + # - name: Login to DockerHub + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + - + name: Login to Github Packages + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + provenance: false + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml new file mode 100644 index 0000000..9d13671 --- /dev/null +++ b/.github/workflows/helm-release.yml @@ -0,0 +1,36 @@ +name: Release Charts + +#on: +# push: +# branches: +# - main + +on: + push: + pull_request: + +jobs: + release: + # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions + # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v3 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.6.0 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml new file mode 100644 index 0000000..7ef3cf1 --- /dev/null +++ b/.github/workflows/helm-test.yml @@ -0,0 +1,45 @@ +name: Lint and Test Charts + +on: pull_request + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.12.1 + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.6.0 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Run chart-testing (lint) + if: steps.list-changed.outputs.changed == 'true' + run: ct lint --target-branch ${{ github.event.repository.default_branch }} + + - name: Create kind cluster + if: steps.list-changed.outputs.changed == 'true' + uses: helm/kind-action@v1.8.0 + + - name: Run chart-testing (install) + if: steps.list-changed.outputs.changed == 'true' + run: ct install --target-branch ${{ github.event.repository.default_branch }} \ No newline at end of file diff --git a/.tool-versions b/.tool-versions index db143de..e99a6ab 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,2 @@ helm 3.14.2 awscli 2.7.14 -golang 1.22.0 diff --git a/charts/aws-service-quotas-exporter/.helmignore b/charts/aws-service-quotas-exporter/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/aws-service-quotas-exporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/aws-service-quotas-exporter/Chart.yaml b/charts/aws-service-quotas-exporter/Chart.yaml new file mode 100644 index 0000000..0a3b98b --- /dev/null +++ b/charts/aws-service-quotas-exporter/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: aws-service-quotas-exporter +description: A Helm chart for Kubernetes + +type: application + +version: 0.0.1 + +appVersion: "v0.0.1" diff --git a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml new file mode 100644 index 0000000..4b6c8bf --- /dev/null +++ b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml @@ -0,0 +1,18 @@ +exporter: + config: + quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" + metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + command: "aws route53 list-hosted-zones" + list: ".HostedZones" + value: ".ResourceRecordSetCount" + labels: + - name: "id" + jqValue: ".Id" + - name: "name" + jqValue: ".Name" + - name: "quota" + value: "L-0263D0A3" \ No newline at end of file diff --git a/charts/aws-service-quotas-exporter/templates/_helpers.tpl b/charts/aws-service-quotas-exporter/templates/_helpers.tpl new file mode 100644 index 0000000..f2a103d --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-service-quotas-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-service-quotas-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-service-quotas-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "aws-service-quotas-exporter.labels" -}} +helm.sh/chart: {{ include "aws-service-quotas-exporter.chart" . }} +{{ include "aws-service-quotas-exporter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "aws-service-quotas-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aws-service-quotas-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "aws-service-quotas-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "aws-service-quotas-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/deployment.yaml b/charts/aws-service-quotas-exporter/templates/deployment.yaml new file mode 100644 index 0000000..a1c28d1 --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "aws-service-quotas-exporter.fullname" . }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "aws-service-quotas-exporter.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "--log.level={{ .Values.exporter.log.level }}" + - "--log.format={{ .Values.exporter.log.format }}" + - "--config=/etc/exporter/scrape.yaml" + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: scrape-cfg + mountPath: /etc/exporter + readOnly: true + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: scrape-cfg + configMap: + name: {{ printf "%s-cfg" (include "aws-service-quotas-exporter.fullname" .) }} + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/scrape-config.yaml b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml new file mode 100644 index 0000000..de70ab0 --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-cfg" (include "aws-service-quotas-exporter.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} +data: + scrape.yaml: + {{- .Values.exporter.config | toYaml | nindent 4 }} diff --git a/charts/aws-service-quotas-exporter/templates/service.yaml b/charts/aws-service-quotas-exporter/templates/service.yaml new file mode 100644 index 0000000..cd6efdc --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "aws-service-quotas-exporter.fullname" . }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 4 }} diff --git a/charts/aws-service-quotas-exporter/templates/serviceaccount.yaml b/charts/aws-service-quotas-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000..879c717 --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "aws-service-quotas-exporter.serviceAccountName" . }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml b/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000..030bd80 --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml @@ -0,0 +1,47 @@ +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} + {{- if .Values.serviceMonitor.labels }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} + name: {{ template "aws-service-quotas-exporter.fullname" . }} +{{- if .Values.serviceMonitor.namespace }} + namespace: {{ .Values.serviceMonitor.namespace }} +{{- end }} +spec: + endpoints: + - targetPort: http +{{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} +{{- end }} +{{- if .Values.serviceMonitor.telemetryPath }} + path: {{ .Values.serviceMonitor.telemetryPath }} +{{- end }} +{{- if .Values.serviceMonitor.timeout }} + scrapeTimeout: {{ .Values.serviceMonitor.timeout }} +{{- end }} +{{- if .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ toYaml .Values.serviceMonitor.metricRelabelings | indent 4 }} +{{- end }} +{{- if .Values.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.serviceMonitor.relabelings | indent 4 }} +{{- end }} + jobLabel: {{ template "aws-service-quotas-exporter.fullname" . }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 6 }} +{{- if .Values.serviceMonitor.targetLabels }} + targetLabels: +{{- range .Values.serviceMonitor.targetLabels }} + - {{ . }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/aws-service-quotas-exporter/values.yaml b/charts/aws-service-quotas-exporter/values.yaml new file mode 100644 index 0000000..bf66cea --- /dev/null +++ b/charts/aws-service-quotas-exporter/values.yaml @@ -0,0 +1,122 @@ +# Default values for aws-service-quotas-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/lablabs/aws-service-quotas-exporter + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 8080 + + +env: + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +livenessProbe: + httpGet: + path: / + port: http +readinessProbe: + httpGet: + path: / + port: http + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +serviceMonitor: + # When set true then use a ServiceMonitor to configure scraping + enabled: false + # Set the namespace the ServiceMonitor should be deployed, if empty namespace will be .Release.Namespace + namespace: "" + # Service monitor labels + labels: { } + # Set how frequently Prometheus should scrape + interval: 30s + # Set path to redis-exporter telemtery-path + telemetryPath: /metrics + # Set timeout for scrape + timeout: 10s + # Set relabel_configs as per https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [ ] + # Set of labels to transfer on the Kubernetes Service onto the target. + targetLabels: [ ] + metricRelabelings: [ ] + + + +exporter: + address: "0.0.0.0:8080" + log: + level: "DEBUG" + format: "json" + config: diff --git a/config/example.yaml b/config/example.yaml index 42287a0..fe3b02d 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -1,14 +1,16 @@ #quotas: # - serviceCode: "ec2" # quotaCode: "L-0263D0A3" -metrics: - - name: "route53_hosted_zone_records" - help: "Number of resource sets in hosted zone" - command: "aws route53 list-hosted-zones" - list: ".HostedZones" - value: ".ResourceRecordSetCount" - labels: - - name: "id" - jqValue: ".Id" - - name: "name" - jqValue: ".Name" \ No newline at end of file +#metrics: +# - name: "route53_hosted_zone_records" +# help: "Number of resource sets in hosted zone" +# command: "aws route53 list-hosted-zones" +# list: ".HostedZones" +# value: ".ResourceRecordSetCount" +# labels: +# - name: "id" +# jqValue: ".Id" +# - name: "name" +# jqValue: ".Name" +# - name: "quota" +# value: "L-0263D0A3" \ No newline at end of file diff --git a/internal/app/config.go b/internal/app/config.go index 8d30e96..70d230b 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -2,7 +2,7 @@ package app type Log struct { Level string `long:"level" description:"Log level" choice:"DEBUG" choice:"INFO" default:"DEBUG"` - Format string `long:"format" description:"Format of message logs" choice:"json" default:"json"` + Format string `long:"format" description:"Format of message logs" choice:"json" choice:"text" default:"json"` } type Config struct { diff --git a/internal/scrape/script/config.go b/internal/scrape/script/config.go index ef53813..d7d3dc2 100644 --- a/internal/scrape/script/config.go +++ b/internal/scrape/script/config.go @@ -39,8 +39,8 @@ func (c *Config) Validate() error { if l.Name == "" { return fmt.Errorf("attribute name for label is required") } - if l.JqValue == "" { - return fmt.Errorf("jqValue for label is required") + if l.GetValue() == "" { + return fmt.Errorf("jqValue or value for label is required") } } return nil @@ -57,4 +57,15 @@ func (c *Config) FormatEnvs() []string { type Label struct { Name string `json:"name,omitempty" yaml:"name,omitempty"` JqValue string `json:"jqValue,omitempty" yaml:"jqValue,omitempty"` + Value string `json:"value,omitempty" yaml:"value,omitempty"` +} + +func (l Label) GetValue() string { + if l.JqValue != "" { + return l.JqValue + } + if l.Value != "" { + return l.Value + } + return "" } diff --git a/internal/scrape/script/script.go b/internal/scrape/script/script.go index 23c1251..3dbb0a7 100644 --- a/internal/scrape/script/script.go +++ b/internal/scrape/script/script.go @@ -68,11 +68,17 @@ func ParseRecord(r any, c Config) (Data, error) { } labels := make(map[string]string) for _, l := range c.Labels { - lv, err := GetString(r, l.JqValue) - if err != nil { - return Data{}, err + if l.JqValue != "" { + lv, err := GetString(r, l.JqValue) + if err != nil { + return Data{}, err + } + labels[l.Name] = lv + continue + } + if l.Value != "" { + labels[l.Name] = l.Value } - labels[l.Name] = lv } return Data{ Value: v, From 9f60f0d287851097eda8d67b21d26aedf77a4a80 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 11:48:01 +0100 Subject: [PATCH 10/32] test for helm --- .github/workflows/helm-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index 7ef3cf1..985c4ab 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -1,6 +1,10 @@ name: Lint and Test Charts -on: pull_request +#on: pull_request + +on: + push: + pull_request: jobs: lint-test: From f3b134b088eb05c850dae2a4ea1b448180e3819c Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 11:58:44 +0100 Subject: [PATCH 11/32] fix helm lint --- charts/aws-service-quotas-exporter/values.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/charts/aws-service-quotas-exporter/values.yaml b/charts/aws-service-quotas-exporter/values.yaml index bf66cea..2ccf63b 100644 --- a/charts/aws-service-quotas-exporter/values.yaml +++ b/charts/aws-service-quotas-exporter/values.yaml @@ -99,7 +99,7 @@ serviceMonitor: # Set the namespace the ServiceMonitor should be deployed, if empty namespace will be .Release.Namespace namespace: "" # Service monitor labels - labels: { } + labels: {} # Set how frequently Prometheus should scrape interval: 30s # Set path to redis-exporter telemtery-path @@ -107,12 +107,10 @@ serviceMonitor: # Set timeout for scrape timeout: 10s # Set relabel_configs as per https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config - relabelings: [ ] + relabelings: [] # Set of labels to transfer on the Kubernetes Service onto the target. - targetLabels: [ ] - metricRelabelings: [ ] - - + targetLabels: [] + metricRelabelings: [] exporter: address: "0.0.0.0:8080" From fe23929a8b67572f92d1734257389631c44b3d37 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 11 Mar 2024 12:02:17 +0100 Subject: [PATCH 12/32] helm release --- .github/workflows/docker-master.yaml | 8 ++++---- .github/workflows/helm-release.yml | 18 ++++++++++-------- .github/workflows/helm-test.yml | 12 ++++++------ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index a0693c3..b90d197 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -1,10 +1,10 @@ name: Build and Publish Docker Images after push to master branch -#on: -# push: -# branches: [master] on: push: - pull_request: + branches: [master] +#on: +# push: +# pull_request: jobs: build-and-push-docker-image: diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index 9d13671..c3bf6e9 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -1,13 +1,13 @@ name: Release Charts -#on: -# push: -# branches: -# - main - on: push: - pull_request: + branches: + - main + +#on: +# push: +# pull_request: jobs: release: @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -28,7 +28,9 @@ jobs: git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - name: Install Helm - uses: azure/setup-helm@v3 + uses: azure/setup-helm@v4 + with: + version: v3.12.1 - name: Run chart-releaser uses: helm/chart-releaser-action@v1.6.0 diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-test.yml index 985c4ab..1e53fad 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-test.yml @@ -11,22 +11,22 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Helm - uses: azure/setup-helm@v3 + - name: Install Helm + uses: azure/setup-helm@v4 with: version: v3.12.1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' check-latest: true - name: Set up chart-testing - uses: helm/chart-testing-action@v2.6.0 + uses: helm/chart-testing-action@v2.6.1 - name: Run chart-testing (list-changed) id: list-changed @@ -42,7 +42,7 @@ jobs: - name: Create kind cluster if: steps.list-changed.outputs.changed == 'true' - uses: helm/kind-action@v1.8.0 + uses: helm/kind-action@v1.9.0 - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' From a1f270035b421ffaef5e4c95a0609ab152a00eff Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 12 Mar 2024 09:44:14 +0100 Subject: [PATCH 13/32] testing pipeline --- .github/workflows/docker-master.yaml | 59 ++++++++----------- .github/workflows/docker-release.yaml | 54 ++++++++--------- .../{go-build.yml => go-binary-release.yml} | 2 +- .github/workflows/go-test.yml | 32 ++++++++++ .github/workflows/golangci-lint.yml | 2 +- .../{helm-test.yml => helm-lint-test.yml} | 18 +++--- .github/workflows/helm-release.yml | 5 +- .../aws-service-quotas-exporter-values.yaml | 2 +- config/example.yaml | 34 ++++++----- 9 files changed, 114 insertions(+), 94 deletions(-) rename .github/workflows/{go-build.yml => go-binary-release.yml} (94%) create mode 100644 .github/workflows/go-test.yml rename .github/workflows/{helm-test.yml => helm-lint-test.yml} (83%) diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index b90d197..e35114f 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -1,59 +1,50 @@ -name: Build and Publish Docker Images after push to master branch +name: Build and publish latest Docker images + on: push: - branches: [master] -#on: -# push: -# pull_request: + tags: + - '*' jobs: - build-and-push-docker-image: - name: Build Docker image and push to repositories with tag latest - runs-on: ubuntu-latest + build-latest-image: + runs-on: ubuntu-22.04 steps: - - - name: Checkout code + - name: Checkout code uses: actions/checkout@v4 - - - name: Set up QEMU + + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ghcr.io/lablabs/aws-service-quotas-exporter - tags: type=raw,value=latest - -# - name: Login to DockerHub -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to Github Packages + - name: Login to Github Packages uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + # generate Docker tags based on the following events/attributes + tags: | + type=raw,value=latest + + - name: Build image and push to GitHub Container Registry uses: docker/build-push-action@v5 with: context: . - provenance: false file: ./Dockerfile platforms: linux/amd64,linux/arm64 - push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + push: true - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml index 6ed4021..eb53e15 100644 --- a/.github/workflows/docker-release.yaml +++ b/.github/workflows/docker-release.yaml @@ -1,58 +1,50 @@ -name: Build and Publish Docker Images after create release tag +name: Build and publish Docker images on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' + - '*' jobs: - build-and-push-docker-image: - name: Build Docker image and push to repositories with tag - runs-on: ubuntu-latest + build-image: + runs-on: ubuntu-22.04 steps: - - - name: Checkout code + - name: Checkout code uses: actions/checkout@v4 - - - name: Set up QEMU + + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ghcr.io/lablabs/aws-service-quotas-exporter - tags: type=ref,event=tag - # - name: Login to DockerHub - # uses: docker/login-action@v3 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - - - - name: Login to Github Packages + - name: Login to Github Packages uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + # generate Docker tags based on the following events/attributes + tags: | + type=raw,value=${{ github.ref_name }} + + - name: Build image and push to GitHub Container Registry uses: docker/build-push-action@v5 with: context: . - provenance: false file: ./Dockerfile platforms: linux/amd64,linux/arm64 - push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + push: true - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.github/workflows/go-build.yml b/.github/workflows/go-binary-release.yml similarity index 94% rename from .github/workflows/go-build.yml rename to .github/workflows/go-binary-release.yml index ff063bd..5e2322b 100644 --- a/.github/workflows/go-build.yml +++ b/.github/workflows/go-binary-release.yml @@ -1,4 +1,4 @@ -name: Generate release-artifacts +name: Generate release binary artifacts on: release: types: diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml new file mode 100644 index 0000000..bc822b4 --- /dev/null +++ b/.github/workflows/go-test.yml @@ -0,0 +1,32 @@ +name: Go tests + +on: + push: + pull_request: +permissions: + contents: read + +env: + GO_VERSION: '1.22.0' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + - name: Get dependencies + run: | + go get -v -t -d ./... + - name: Build + env: + GOPROXY: "https://proxy.golang.org" + run: go build . + - name: Test + env: + GOPROXY: "https://proxy.golang.org" + run: go test -v . \ No newline at end of file diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index a1f9078..9c6a9e2 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -1,4 +1,4 @@ -name: golangci-lint +name: Golang lint on: push: pull_request: diff --git a/.github/workflows/helm-test.yml b/.github/workflows/helm-lint-test.yml similarity index 83% rename from .github/workflows/helm-test.yml rename to .github/workflows/helm-lint-test.yml index 1e53fad..a3186d6 100644 --- a/.github/workflows/helm-test.yml +++ b/.github/workflows/helm-lint-test.yml @@ -1,28 +1,28 @@ name: Lint and Test Charts -#on: pull_request - on: - push: pull_request: +env: + HELM_VERSION: 3.14.0 + jobs: lint-test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install Helm + - name: Set up Helm uses: azure/setup-helm@v4 with: - version: v3.12.1 + version: ${{ env.HELM_VERSION }} - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' check-latest: true - name: Set up chart-testing @@ -38,7 +38,7 @@ jobs: - name: Run chart-testing (lint) if: steps.list-changed.outputs.changed == 'true' - run: ct lint --target-branch ${{ github.event.repository.default_branch }} + run: ct lint --target-branch ${{ github.event.repository.default_branch }} --validate-maintainers=false - name: Create kind cluster if: steps.list-changed.outputs.changed == 'true' @@ -46,4 +46,4 @@ jobs: - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' - run: ct install --target-branch ${{ github.event.repository.default_branch }} \ No newline at end of file + run: ct install --target-branch ${{ github.event.repository.default_branch }} diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index c3bf6e9..a44221d 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -9,6 +9,9 @@ on: # push: # pull_request: +env: + HELM_VERSION: 3.14.0 + jobs: release: # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions @@ -30,7 +33,7 @@ jobs: - name: Install Helm uses: azure/setup-helm@v4 with: - version: v3.12.1 + version: ${{ env.HELM_VERSION }} - name: Run chart-releaser uses: helm/chart-releaser-action@v1.6.0 diff --git a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml index 4b6c8bf..b65feaa 100644 --- a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml +++ b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml @@ -15,4 +15,4 @@ exporter: - name: "name" jqValue: ".Name" - name: "quota" - value: "L-0263D0A3" \ No newline at end of file + value: "L-0263D0A3" diff --git a/config/example.yaml b/config/example.yaml index fe3b02d..687e31e 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -1,16 +1,18 @@ -#quotas: -# - serviceCode: "ec2" -# quotaCode: "L-0263D0A3" -#metrics: -# - name: "route53_hosted_zone_records" -# help: "Number of resource sets in hosted zone" -# command: "aws route53 list-hosted-zones" -# list: ".HostedZones" -# value: ".ResourceRecordSetCount" -# labels: -# - name: "id" -# jqValue: ".Id" -# - name: "name" -# jqValue: ".Name" -# - name: "quota" -# value: "L-0263D0A3" \ No newline at end of file +quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" +metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + command: "aws route53 list-hosted-zones" + list: ".HostedZones" + value: ".ResourceRecordSetCount" + labels: + - name: "id" + jqValue: ".Id" + - name: "name" + jqValue: ".Name" + - name: "quota" + value: "L-0263D0A3" + +//pocet EC2 instances per type \ No newline at end of file From b3f1379840ef0a1717090d1d165e9c3076d8ece4 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 12 Mar 2024 10:15:03 +0100 Subject: [PATCH 14/32] pre commit hook --- .github/workflows/go-test.yml | 4 ++-- .pre-commit-config.yaml | 25 +++++++++++++++++++++++++ .tool-versions | 1 + 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index bc822b4..7237034 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - name: Get dependencies @@ -29,4 +29,4 @@ jobs: - name: Test env: GOPROXY: "https://proxy.golang.org" - run: go test -v . \ No newline at end of file + run: go test -v . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..108a5cf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: trailing-whitespace + - id: check-merge-conflict + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + - id: detect-private-key + - id: end-of-file-fixer + + - repo: https://github.com/gruntwork-io/pre-commit + rev: v0.1.17 + hooks: + - id: helmlint + + - repo: https://github.com/norwoodj/helm-docs + rev: v1.13.0 + hooks: + - id: helm-docs + args: + - --chart-search-root=charts + - id: helm-docs-built + args: + - --chart-search-root=charts diff --git a/.tool-versions b/.tool-versions index e99a6ab..556137a 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ helm 3.14.2 awscli 2.7.14 +pre-commit 2.20.0 From d0e72dde5c503f75fadbbff9580cc8ef005757fd Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 12 Mar 2024 10:42:42 +0100 Subject: [PATCH 15/32] test flow --- .github/workflows/docker-master.yaml | 7 ++++--- .github/workflows/go-test.yml | 4 ---- .github/workflows/helm-lint-test.yml | 15 +++++++++++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml index e35114f..dbf040f 100644 --- a/.github/workflows/docker-master.yaml +++ b/.github/workflows/docker-master.yaml @@ -1,9 +1,10 @@ name: Build and publish latest Docker images on: + workflow_dispatch: push: - tags: - - '*' + branches: + - main jobs: build-latest-image: @@ -47,4 +48,4 @@ jobs: push: true - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 7237034..94f6182 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -22,10 +22,6 @@ jobs: - name: Get dependencies run: | go get -v -t -d ./... - - name: Build - env: - GOPROXY: "https://proxy.golang.org" - run: go build . - name: Test env: GOPROXY: "https://proxy.golang.org" diff --git a/.github/workflows/helm-lint-test.yml b/.github/workflows/helm-lint-test.yml index a3186d6..f25038c 100644 --- a/.github/workflows/helm-lint-test.yml +++ b/.github/workflows/helm-lint-test.yml @@ -1,7 +1,6 @@ name: Lint and Test Charts -on: - pull_request: +on: pull_request env: HELM_VERSION: 3.14.0 @@ -47,3 +46,15 @@ jobs: - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' run: ct install --target-branch ${{ github.event.repository.default_branch }} + + linter-artifacthub: + runs-on: ubuntu-22.04 + container: + image: artifacthub/ah + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run ah lint + working-directory: ./charts/ + run: ah lint From fba7f3098571c3720f623cec973e4c2ee92c253c Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 12 Mar 2024 10:45:28 +0100 Subject: [PATCH 16/32] go tests --- .github/workflows/go-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 94f6182..50a82f7 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -25,4 +25,4 @@ jobs: - name: Test env: GOPROXY: "https://proxy.golang.org" - run: go test -v . + run: go test -v ./... From 01133004c6db3d6747262d46f9736565ef2b24ed Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 12 Mar 2024 10:55:26 +0100 Subject: [PATCH 17/32] ubuntu 22:04 --- .github/workflows/go-binary-release.yml | 4 ++-- .github/workflows/go-test.yml | 2 +- .github/workflows/golangci-lint.yml | 6 +++--- .github/workflows/helm-release.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/go-binary-release.yml b/.github/workflows/go-binary-release.yml index 5e2322b..91987b3 100644 --- a/.github/workflows/go-binary-release.yml +++ b/.github/workflows/go-binary-release.yml @@ -6,7 +6,7 @@ on: jobs: generate: name: Generate cross-platform builds - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout the repository uses: actions/checkout@v4 @@ -23,4 +23,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - args: "./dist/*" \ No newline at end of file + args: "./dist/*" diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 50a82f7..7063a44 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -11,7 +11,7 @@ env: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 9c6a9e2..11b857d 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -7,11 +7,11 @@ permissions: jobs: golangci: - name: GO lang CI linter - runs-on: ubuntu-latest + name: Golang CI linter + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v4 with: - version: v1.56.2 \ No newline at end of file + version: v1.56.2 diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml index a44221d..41303b9 100644 --- a/.github/workflows/helm-release.yml +++ b/.github/workflows/helm-release.yml @@ -18,7 +18,7 @@ jobs: # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token permissions: contents: write - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -38,4 +38,4 @@ jobs: - name: Run chart-releaser uses: helm/chart-releaser-action@v1.6.0 env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From aedf9ae5b24387444cdf3bb3fb0f3ed5026b609e Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 12 Mar 2024 11:57:52 +0100 Subject: [PATCH 18/32] fix map --- charts/aws-service-quotas-exporter/templates/scrape-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/aws-service-quotas-exporter/templates/scrape-config.yaml b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml index de70ab0..39ebc46 100644 --- a/charts/aws-service-quotas-exporter/templates/scrape-config.yaml +++ b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml @@ -6,5 +6,5 @@ metadata: labels: {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} data: - scrape.yaml: + scrape.yaml: | {{- .Values.exporter.config | toYaml | nindent 4 }} From aa26feb51d21173e69af3bfb6116a703e9a0ebcd Mon Sep 17 00:00:00 2001 From: rafajpet Date: Thu, 21 Mar 2024 11:18:23 +0100 Subject: [PATCH 19/32] clean & build docker image --- .gitignore | 4 +- Dockerfile | 43 ++++--- README.md | 6 +- config/example.yaml | 13 +-- go.mod | 17 ++- go.sum | 44 ++++---- internal/app/application.go | 10 +- internal/app/config.go | 2 +- internal/exporter/exporter.go | 29 ++++- internal/exporter/exporter_test.go | 13 +-- internal/scrape/quotas/collector.go | 84 ++++++++++---- internal/scrape/quotas/config.go | 8 +- internal/scrape/script/collector.go | 78 +++++++++---- internal/scrape/script/collector_test.go | 40 +++++++ internal/scrape/script/config.go | 76 +++++-------- internal/scrape/script/metric.go | 68 +++++++++++ internal/scrape/script/metric_test.go | 61 ++++++++++ internal/scrape/script/script.go | 137 +++++------------------ internal/scrape/script/script_test.go | 91 ++++----------- pkg/jqdata/parse.go | 40 +++++++ pkg/jqdata/parse_test.go | 59 ++++++++++ test/configs/eip.yaml | 9 +- 22 files changed, 563 insertions(+), 369 deletions(-) create mode 100644 internal/scrape/script/collector_test.go create mode 100644 internal/scrape/script/metric.go create mode 100644 internal/scrape/script/metric_test.go create mode 100644 pkg/jqdata/parse.go create mode 100644 pkg/jqdata/parse_test.go diff --git a/.gitignore b/.gitignore index ec29974..2f222d3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,6 @@ go.work .env .envrc -.idea \ No newline at end of file +.idea + +coverage.out diff --git a/Dockerfile b/Dockerfile index 3b65c27..0f6d327 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,25 @@ -FROM golang:1.22-alpine3.19 as builder +ARG ALPINE_VERSION=3.19 +FROM python:3.10-alpine${ALPINE_VERSION} as builder_aws_cli + +ARG AWS_CLI_VERSION=2.15.19 +RUN apk add --no-cache git unzip groff build-base libffi-dev cmake +RUN git clone --single-branch --depth 1 -b ${AWS_CLI_VERSION} https://github.com/aws/aws-cli.git + +WORKDIR aws-cli +RUN ./configure --with-install-type=portable-exe --with-download-deps +RUN make +RUN make install + +# reduce image size: remove autocomplete and examples +RUN rm -rf \ + /usr/local/lib/aws-cli/aws_completer \ + /usr/local/lib/aws-cli/awscli/data/ac.index \ + /usr/local/lib/aws-cli/awscli/examples +RUN find /usr/local/lib/aws-cli/awscli/data -name completions-1*.json -delete +RUN find /usr/local/lib/aws-cli/awscli/botocore/data -name examples-1.json -delete +RUN (cd /usr/local/lib/aws-cli; for a in *.so*; do test -f /lib/$a && rm $a; done) + +FROM golang:1.22-alpine${ALPINE_VERSION} as builder_golang ARG GOOS=linux ARG GOARCH=amd64 @@ -17,23 +38,13 @@ RUN cd cmd/exporter && \ CGO_ENABLED=0 \ go build -o /aws-service-quotas-exporter . -FROM alpine:3.19.1 as security +FROM alpine:${ALPINE_VERSION} -RUN apk add -U --no-cache \ - tzdata \ - ca-certificates +RUN apk update && apk add jq -RUN addgroup -S nonroot \ - && adduser -S nonroot -G nonroot +COPY --from=builder_aws_cli /usr/local/lib/aws-cli/ /usr/local/lib/aws-cli/ +RUN ln -s /usr/local/lib/aws-cli/aws /usr/local/bin/aws -FROM scratch - -COPY --from=security /usr/share/zoneinfo /usr/share/zoneinfo -COPY --from=security /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=security /etc/passwd /etc/passwd -COPY --from=security /etc/group /etc/group - -COPY --from=builder /aws-service-quotas-exporter . +COPY --from=builder_golang /aws-service-quotas-exporter . ENTRYPOINT ["/aws-service-quotas-exporter"] - diff --git a/README.md b/README.md index bb6a45e..5b46a96 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # aws-service-quotas-exporter -AWS Quotas utilisation prometheus exporter is Prometheus exporter allows you to easily export real usage of AWS resources -and monitor/alert usage of over exited of those resources with respect to applied Quotas. +AWS service quotas exporter exposes actual quotas for your AWS accounts and allow you to scrape actual +usage of AWS resources. Base on those two types of data, you can easily build +alert rules to prevent case when you are not able to provision another AWS resources due to reach the limit of AWS quota +## AWS Co diff --git a/config/example.yaml b/config/example.yaml index 687e31e..6ec5e14 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -4,15 +4,4 @@ quotas: metrics: - name: "route53_hosted_zone_records" help: "Number of resource sets in hosted zone" - command: "aws route53 list-hosted-zones" - list: ".HostedZones" - value: ".ResourceRecordSetCount" - labels: - - name: "id" - jqValue: ".Id" - - name: "name" - jqValue: ".Name" - - name: "quota" - value: "L-0263D0A3" - -//pocet EC2 instances per type \ No newline at end of file + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),\\(.ResourceRecordSetCount)\"\'" diff --git a/go.mod b/go.mod index 116ea0b..2adff70 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module github.com/lablabs/aws-service-quotas-exporter go 1.22.0 require ( - github.com/aws/aws-sdk-go-v2 v1.25.2 + github.com/aws/aws-sdk-go-v2 v1.26.0 github.com/aws/aws-sdk-go-v2/config v1.27.4 - github.com/aws/aws-sdk-go-v2/service/cloudcontrol v1.18.1 github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.1 + github.com/itchyny/gojq v0.12.14 + github.com/jessevdk/go-flags v1.5.0 + github.com/prometheus/client_golang v1.18.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 golang.org/x/sync v0.6.0 @@ -16,8 +18,8 @@ require ( require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect @@ -28,13 +30,10 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-test/deep v1.1.0 // indirect - github.com/jessevdk/go-flags v1.5.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/itchyny/timefmt-go v0.1.5 // indirect + github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect - github.com/ohler55/ojg v1.21.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/go.sum b/go.sum index 6b6385f..9267523 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,17 @@ -github.com/aws/aws-sdk-go-v2 v1.25.2 h1:/uiG1avJRgLGiQM9X3qJM8+Qa6KRGK5rRPuXE0HUM+w= -github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo= +github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= +github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= github.com/aws/aws-sdk-go-v2/config v1.27.4 h1:AhfWb5ZwimdsYTgP7Od8E9L1u4sKmDW2ZVeLcf2O42M= github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g= github.com/aws/aws-sdk-go-v2/credentials v1.17.4 h1:h5Vztbd8qLppiPwX+y0Q6WiwMZgpd9keKe2EAENgAuI= github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2 h1:bNo4LagzUKbjdxE0tIcR9pMzLR2U/Tgie1Hq1HQ3iH8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2 h1:EtOU5jsPdIQNP+6Q2C5e3d65NKT1PeCiQk+9OdzO12Q= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/service/cloudcontrol v1.18.1 h1:v7LcvMEl5uRm3SOYY+JtXbs8F16S8kfJZdONU+KADoE= -github.com/aws/aws-sdk-go-v2/service/cloudcontrol v1.18.1/go.mod h1:fBKrOkINcm/C9bXnnO5NwwIT7tGIaUb/LRXHNpwXpUM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 h1:5ffmXjPtwRExp1zc7gENLgCPyHFbhEPwVTkTiH9niSk= @@ -32,25 +30,26 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= -github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc= +github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= +github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= -github.com/ohler55/ojg v1.21.3 h1:0smW0EKpyPBBIpTKhM+UbCDeQFbR0oEUxym+rFv2Y/8= -github.com/ohler55/ojg v1.21.3/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -61,6 +60,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -70,7 +71,6 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -78,11 +78,9 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/application.go b/internal/app/application.go index 74a8e28..8fa3abb 100644 --- a/internal/app/application.go +++ b/internal/app/application.go @@ -38,23 +38,15 @@ func NewApplication(log *logrus.Logger, cfg Config) (*Application, error) { if err != nil { return nil, err } - err = qcl.Register(registry) - if err != nil { - return nil, err - } cls = append(cls, qcl) scl, err := script.NewCollector(log, scCfg.Metrics, PrometheusNamespace) if err != nil { return nil, err } - err = scl.Register(registry) - if err != nil { - return nil, err - } cls = append(cls, scl) - exp, err := exporter.NewExporter(log, cls, exporterOptions(scCfg)...) + exp, err := exporter.NewExporter(log, cls, registry, exporterOptions(scCfg)...) if err != nil { return nil, err } diff --git a/internal/app/config.go b/internal/app/config.go index 70d230b..72f5581 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -8,5 +8,5 @@ type Log struct { type Config struct { Address string `long:"address" description:"Http address" default:"0.0.0.0:8080"` Log Log `group:"log" namespace:"log"` - Config string `long:"config" required:"true" description:"Path to scraper scrape file"` + Config string `long:"config" required:"true" description:"Path to metric scrape config file"` } diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index 43c239f..b724714 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -10,8 +10,8 @@ import ( ) type Collector interface { - Register(r *prometheus.Registry) error - Collect(ctx context.Context, g *errgroup.Group) + Register(ctx context.Context, r *prometheus.Registry) error + Collect(ctx context.Context) error } const ( @@ -19,7 +19,7 @@ const ( defaultCollectorTimeout = time.Second * 5 ) -func NewExporter(log *logrus.Logger, cls []Collector, options ...Option) (*Exporter, error) { +func NewExporter(log *logrus.Logger, cls []Collector, r *prometheus.Registry, options ...Option) (*Exporter, error) { cfg := config{ interval: defaultScrapeInterval, timeout: defaultCollectorTimeout, @@ -33,6 +33,7 @@ func NewExporter(log *logrus.Logger, cls []Collector, options ...Option) (*Expor log: log, cfg: &cfg, cls: cls, + r: r, } return &e, nil } @@ -41,10 +42,11 @@ type Exporter struct { log *logrus.Logger cfg *config cls []Collector + r *prometheus.Registry } func (e *Exporter) Run(ctx context.Context) error { - err := e.scrape(ctx) + err := e.register(ctx) if err != nil { return err } @@ -67,12 +69,27 @@ end: } func (e *Exporter) scrape(ctx context.Context) error { - e.log.Debugf("start scraping metrics %v with timeout: %v", time.Now().Format(time.RFC3339), e.cfg.timeout) + e.log.Debugf("scrape metrics") ctx, cancel := context.WithTimeout(ctx, e.cfg.timeout) defer cancel() g, ctx := errgroup.WithContext(ctx) for _, c := range e.cls { - c.Collect(ctx, g) + g.Go(func() error { + return c.Collect(ctx) + }) + } + err := g.Wait() + return err +} + +func (e *Exporter) register(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, e.cfg.timeout) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + for _, c := range e.cls { + g.Go(func() error { + return c.Register(ctx, e.r) + }) } err := g.Wait() return err diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go index 22d455d..4b5c44f 100644 --- a/internal/exporter/exporter_test.go +++ b/internal/exporter/exporter_test.go @@ -8,7 +8,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "golang.org/x/sync/errgroup" "testing" "time" ) @@ -52,7 +51,7 @@ func TestExporter_Run(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e, err := exporter.NewExporter(tt.fields.log, tt.fields.cls) + e, err := exporter.NewExporter(tt.fields.log, tt.fields.cls, prometheus.NewRegistry()) assert.NoError(t, err) ctx, cancel := tt.fields.ctx() defer cancel() @@ -68,12 +67,10 @@ type testCollector struct { err error } -func (t *testCollector) Register(_ *prometheus.Registry) error { - return nil +func (t *testCollector) Register(_ context.Context, _ *prometheus.Registry) error { + return t.err } -func (t *testCollector) Collect(_ context.Context, g *errgroup.Group) { - g.Go(func() error { - return t.err - }) +func (t *testCollector) Collect(_ context.Context) error { + return t.err } diff --git a/internal/scrape/quotas/collector.go b/internal/scrape/quotas/collector.go index 29ce414..41253bd 100644 --- a/internal/scrape/quotas/collector.go +++ b/internal/scrape/quotas/collector.go @@ -8,6 +8,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + "sync" ) const ( @@ -21,45 +22,84 @@ type Quota interface { } func NewCollector(log *logrus.Logger, cfg []Config, ns string, qcl Quota) (*Collector, error) { - gvq := prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: ns, - Name: "quota", - Help: "AWS service quota", - }, []string{name, code, serviceCode}) cl := Collector{ - log: log, - qcl: qcl, - gvq: gvq, - cfg: cfg, + log: log, + qcl: qcl, + cfg: cfg, + ns: ns, + tasks: make([]task, 0), } return &cl, nil } type Collector struct { - log *logrus.Logger - qcl Quota - gvq *prometheus.GaugeVec - cfg []Config + log *logrus.Logger + qcl Quota + once sync.Once + err error + ns string + cfg []Config + tasks []task + tasksLock sync.Mutex } -func (c *Collector) Register(r *prometheus.Registry) error { - r.MustRegister(c.gvq) - return nil +func (c *Collector) Register(ctx context.Context, r *prometheus.Registry) error { + c.once.Do(func() { + c.log.Debugf("start registering quota metrics") + gvq := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: c.ns, + Name: "quota", + Help: "AWS service quota", + }, []string{name, code, serviceCode}) + r.MustRegister(gvq) + g, ctx := errgroup.WithContext(ctx) + for _, cf := range c.cfg { + g.Go(func() error { + res, err := c.qcl.GetQuota(ctx, cf.ServiceCode, cf.QuotaCode, quota.WithRegion(cf.Region)) + if err != nil { + return err + } + t := task{ + m: gvq, + cfg: cf, + } + c.addTask(t) + setMetric(t.m, res) + return nil + }) + } + c.err = g.Wait() + }) + return c.err } -func (c *Collector) Collect(ctx context.Context, g *errgroup.Group) { - for _, q := range c.cfg { - g.Go(c.run(ctx, q)) +func (c *Collector) Collect(ctx context.Context) error { + g, ctx := errgroup.WithContext(ctx) + for _, t := range c.tasks { + g.Go(t.run(ctx, c.qcl)) } + err := g.Wait() + return err +} + +func (c *Collector) addTask(t task) { + c.tasksLock.Lock() + defer c.tasksLock.Unlock() + c.tasks = append(c.tasks, t) +} + +type task struct { + m *prometheus.GaugeVec + cfg Config } -func (c *Collector) run(ctx context.Context, q Config) func() error { +func (t task) run(ctx context.Context, c Quota) func() error { return func() error { - res, err := c.qcl.GetQuota(ctx, q.ServiceCode, q.QuotaCode, quota.WithRegion(q.Region)) + res, err := c.GetQuota(ctx, t.cfg.ServiceCode, t.cfg.QuotaCode, quota.WithRegion(t.cfg.Region)) if err != nil { return err } - setMetric(c.gvq, res) + setMetric(t.m, res) return nil } } diff --git a/internal/scrape/quotas/config.go b/internal/scrape/quotas/config.go index 5f853ae..7a3a141 100644 --- a/internal/scrape/quotas/config.go +++ b/internal/scrape/quotas/config.go @@ -2,14 +2,12 @@ package quotas import ( "fmt" - "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" ) type Config struct { - ServiceCode string `json:"serviceCode,omitempty" yaml:"serviceCode,omitempty"` - QuotaCode string `json:"quotaCode,omitempty" yaml:"quotaCode,omitempty"` - Region string `json:"region,omitempty" yaml:"region,omitempty"` - Usage script.Config `json:"usage,omitempty" yaml:"usage,omitempty"` + ServiceCode string `json:"serviceCode,omitempty" yaml:"serviceCode,omitempty"` + QuotaCode string `json:"quotaCode,omitempty" yaml:"quotaCode,omitempty"` + Region string `json:"region,omitempty" yaml:"region,omitempty"` } func (c Config) Validate() error { diff --git a/internal/scrape/script/collector.go b/internal/scrape/script/collector.go index b58c595..6a36039 100644 --- a/internal/scrape/script/collector.go +++ b/internal/scrape/script/collector.go @@ -5,46 +5,78 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" + "sync" ) func NewCollector(log *logrus.Logger, cfg []Config, ns string) (*Collector, error) { - - tks := make([]task, 0) - for _, c := range cfg { - m := prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: ns, - Name: c.Name, - Help: c.Help, - }, c.LabelNames()) - tks = append(tks, task{ - m: m, - cfg: c, - }) - } - cl := Collector{ log: log, - tasks: tks, + ns: ns, + cfg: cfg, + tasks: make([]task, 0), } return &cl, nil } type Collector struct { - log *logrus.Logger - tasks []task + log *logrus.Logger + once sync.Once + err error + ns string + cfg []Config + tasks []task + tasksLock sync.Mutex } -func (c *Collector) Register(r *prometheus.Registry) error { - for _, t := range c.tasks { - r.MustRegister(t.m) - } - return nil +func (c *Collector) Register(ctx context.Context, r *prometheus.Registry) error { + c.once.Do(func() { + c.log.Debugf("start registering script metrics") + g, ctx := errgroup.WithContext(ctx) + for _, cf := range c.cfg { + g.Go(func() error { + data, err := Run(ctx, cf) + if err != nil { + c.log.Errorf("unable to run command: %s, %v", cf.Script, err) + return err + } + if len(data) > 0 { + lbs := data[0].LabelNames() + m := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: c.ns, + Name: cf.Name, + Help: cf.Help, + }, lbs) + r.MustRegister(m) + t := task{ + m: m, + cfg: cf, + } + c.addTask(t) + for _, d := range data { + t.m.With(d.Labels).Set(d.Value) + } + } + return nil + }) + } + c.err = g.Wait() + }) + return c.err } -func (c *Collector) Collect(ctx context.Context, g *errgroup.Group) { +func (c *Collector) Collect(ctx context.Context) error { + g, ctx := errgroup.WithContext(ctx) for _, t := range c.tasks { g.Go(t.run(ctx)) } + err := g.Wait() + return err +} + +func (c *Collector) addTask(t task) { + c.tasksLock.Lock() + defer c.tasksLock.Unlock() + c.tasks = append(c.tasks, t) } type task struct { diff --git a/internal/scrape/script/collector_test.go b/internal/scrape/script/collector_test.go new file mode 100644 index 0000000..efe4b10 --- /dev/null +++ b/internal/scrape/script/collector_test.go @@ -0,0 +1,40 @@ +package script_test + +import ( + "context" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/lablabs/aws-service-quotas-exporter/test" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestNewCollector(t *testing.T) { + + cfg := []script.Config{ + { + Name: "metric_a", + Help: "metric b", + Script: "echo \"name=n,cluster=1,1\"", + }, + { + Name: "metric_b", + Help: "metric b", + Script: "echo \"name=n,cluster=2,1\"", + }, + } + + cl, err := script.NewCollector(test.DefaultLogger(), cfg, "ns") + assert.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + r := prometheus.NewRegistry() + err = cl.Register(ctx, r) + assert.NoError(t, err) + err = cl.Register(ctx, r) + assert.NoError(t, err) + err = cl.Collect(ctx) + assert.NoError(t, err) + +} diff --git a/internal/scrape/script/config.go b/internal/scrape/script/config.go index d7d3dc2..9d60976 100644 --- a/internal/scrape/script/config.go +++ b/internal/scrape/script/config.go @@ -3,49 +3,47 @@ package script import "fmt" type Config struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Help string `json:"help,omitempty" yaml:"help,omitempty"` - Command string `json:"command,omitempty" yaml:"command,omitempty"` - Envs []Env `json:"env,omitempty" yaml:"envs,omitempty"` - List string `json:"list,omitempty" yaml:"list,omitempty"` - Value string `json:"value,omitempty" yaml:"value,omitempty"` - Labels []Label `json:"labels,omitempty" yaml:"labels,omitempty"` -} - -type Env struct { - Name string `json:"name,omitempty" yaml:"name"` - Value string `json:"value,omitempty" yaml:"value"` -} - -func (c *Config) LabelNames() []string { - l := make([]string, 0) - for _, c := range c.Labels { - l = append(l, c.Name) - } - return l + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Help string `json:"help,omitempty" yaml:"help,omitempty"` + Script string `json:"script,omitempty" yaml:"script"` + Envs []Env `json:"env,omitempty" yaml:"envs,omitempty"` } func (c *Config) Validate() error { if c.Name == "" { - return fmt.Errorf("name attribute is required") + return fmt.Errorf("name of metric is required") } - if c.Command == "" { - return fmt.Errorf("command attribute is required") + if c.Help == "" { + return fmt.Errorf("help of metric is required") } - if c.Value == "" { - return fmt.Errorf("value attribute is required") + if c.Script == "" { + return fmt.Errorf("script is required") } - for _, l := range c.Labels { - if l.Name == "" { - return fmt.Errorf("attribute name for label is required") - } - if l.GetValue() == "" { - return fmt.Errorf("jqValue or value for label is required") + if c.Envs != nil { + for _, e := range c.Envs { + if err := e.Validate(); err != nil { + return err + } } } return nil } +type Env struct { + Name string `json:"name,omitempty" yaml:"name"` + Value string `json:"value,omitempty" yaml:"value"` +} + +func (e *Env) Validate() error { + if e.Name == "" { + return fmt.Errorf("name for env is required") + } + if e.Value == "" { + return fmt.Errorf("value for env is required") + } + return nil +} + func (c *Config) FormatEnvs() []string { evs := make([]string, 0) for _, e := range c.Envs { @@ -53,19 +51,3 @@ func (c *Config) FormatEnvs() []string { } return evs } - -type Label struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - JqValue string `json:"jqValue,omitempty" yaml:"jqValue,omitempty"` - Value string `json:"value,omitempty" yaml:"value,omitempty"` -} - -func (l Label) GetValue() string { - if l.JqValue != "" { - return l.JqValue - } - if l.Value != "" { - return l.Value - } - return "" -} diff --git a/internal/scrape/script/metric.go b/internal/scrape/script/metric.go new file mode 100644 index 0000000..435b760 --- /dev/null +++ b/internal/scrape/script/metric.go @@ -0,0 +1,68 @@ +package script + +import ( + "bufio" + "fmt" + "io" + "regexp" + "strconv" + "strings" +) + +const metricRegex = `,` + +func ParseStdout(r io.Reader) ([]Data, error) { + out := make([]Data, 0) + pr := NewParser() + scanner := bufio.NewScanner(r) + for scanner.Scan() { + mt, err := pr.ParseMetric(scanner.Text()) + if err != nil { + return nil, fmt.Errorf("unable to parse metric: %w", err) + } + out = append(out, mt) + } + return out, nil +} + +func NewParser() Parser { + re := regexp.MustCompile(metricRegex) + p := Parser{re: re} + return p +} + +type Parser struct { + re *regexp.Regexp +} + +func (p Parser) ParseMetric(l string) (Data, error) { + lbs := make(map[string]string) + var v float64 + mtcs := p.re.Split(l, -1) + for i, mt := range mtcs { + if i == len(mtcs)-1 { + vl, err := strconv.ParseFloat(mt, 64) + if err != nil { + return Data{}, fmt.Errorf("invalid value of metric, %s is not float64, %v", mt, err) + } + v = vl + continue + } + spl := strings.Index(mt, "=") + lbs[mt[:spl]] = cleanBracket(mt[spl+1:]) + } + return Data{ + Value: v, + Labels: lbs, + }, nil +} + +func cleanBracket(input string) string { + if strings.HasPrefix(input, `"`) || strings.HasPrefix(input, "'") { + input = input[1:] + } + if strings.HasSuffix(input, `"`) || strings.HasSuffix(input, "'") { + input = input[:len(input)-1] + } + return input +} diff --git a/internal/scrape/script/metric_test.go b/internal/scrape/script/metric_test.go new file mode 100644 index 0000000..a45fc93 --- /dev/null +++ b/internal/scrape/script/metric_test.go @@ -0,0 +1,61 @@ +package script_test + +import ( + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/stretchr/testify/assert" + "reflect" + "testing" +) + +func TestParser_ParseMetric(t *testing.T) { + tests := []struct { + name string + metric string + want script.Data + wantErr bool + }{ + { + name: "Parse Metric OK", + metric: "a=1,b=2,c='3',d='t e a',y=\"aa bbb ccc\",4", + want: script.Data{ + Value: 4, + Labels: map[string]string{ + "a": "1", + "b": "2", + "c": "3", + "d": "t e a", + "y": "aa bbb ccc", + }, + }, + wantErr: false, + }, + { + name: "Value not valid float64", + metric: "a=1,b=2,c='3',d='t e a',y=\"aa bbb ccc\",not_valid", + want: script.Data{ + Value: 4, + Labels: map[string]string{ + "a": "1", + "b": "2", + "c": "3", + "d": "t e a", + "y": "aa bbb ccc", + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := script.NewParser() + got, err := p.ParseMetric(tt.metric) + if tt.wantErr { + assert.Errorf(t, err, "ParseMetric() expect error") + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseMetric() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/scrape/script/script.go b/internal/scrape/script/script.go index 3dbb0a7..2cacfe8 100644 --- a/internal/scrape/script/script.go +++ b/internal/scrape/script/script.go @@ -3,9 +3,8 @@ package script import ( "bytes" "context" - "encoding/json" "fmt" - "github.com/ohler55/ojg/jp" + "io" "os" "os/exec" "strings" @@ -16,129 +15,49 @@ type Data struct { Labels map[string]string } -func Run(ctx context.Context, cfg Config) ([]Data, error) { +func (d Data) LabelNames() []string { + if d.Labels == nil { + return []string{} + } + r := make([]string, 0) + for k, _ := range d.Labels { + r = append(r, k) + } + return r +} - cs := strings.Split(cfg.Command, " ") - prg := cs[0] - args := cs[1:] - cmd := exec.CommandContext(ctx, prg, args...) - var stdout, stderr bytes.Buffer +func Run(ctx context.Context, cfg Config) ([]Data, error) { + cmd := exec.CommandContext(ctx, "bash", "-c", cfg.Script) envs := make([]string, 0) envs = append(envs, os.Environ()...) envs = append(envs, cfg.FormatEnvs()...) cmd.Env = envs - cmd.Stderr = &stderr - cmd.Stdout = &stdout - err := cmd.Run() - if err != nil { - return nil, fmt.Errorf("script error: %w, std err: %s", err, stderr.String()) - } - data, err := ParseJSON(stdout.Bytes()) - if err != nil { - return nil, fmt.Errorf("unable to parse response from command: %v", cfg.Command) - } - - result := make([]Data, 0) - if cfg.List != "" { - items, err := GetArray(data, cfg.List) - if err != nil { - return nil, fmt.Errorf("unable to parse list items jq: %w", err) - } - for _, it := range items { - r, err := ParseRecord(it, cfg) - if err != nil { - return nil, err - } - result = append(result, r) - } - return result, nil - } - r, err := ParseRecord(data, cfg) + stderr, err := cmd.StderrPipe() if err != nil { return nil, err } - result = append(result, r) - return result, nil -} - -func ParseRecord(r any, c Config) (Data, error) { - v, err := GetFloat64(r, c.Value) + stdout := &bytes.Buffer{} + cmd.Stdout = stdout + err = cmd.Run() if err != nil { - return Data{}, err - } - labels := make(map[string]string) - for _, l := range c.Labels { - if l.JqValue != "" { - lv, err := GetString(r, l.JqValue) - if err != nil { - return Data{}, err - } - labels[l.Name] = lv - continue - } - if l.Value != "" { - labels[l.Name] = l.Value + errString, err := errorString(stderr) + if err != nil { + return nil, err } + return nil, fmt.Errorf("script error: %w, std err: %s", err, errString) } - return Data{ - Value: v, - Labels: labels, - }, nil -} - -func ParseJSON(data []byte) (map[string]interface{}, error) { - var out map[string]interface{} - err := json.Unmarshal(data, &out) - if err != nil { - return nil, fmt.Errorf("unable parse to json: %w", err) - } - return out, nil -} - -func GetFloat64(data any, query string) (float64, error) { - res, err := ParseJq(data, query) + data, err := ParseStdout(stdout) if err != nil { - return 0, err - } - v, ok := res[0].(float64) - if !ok { - return 0, fmt.Errorf("invalid data. jq expression must return valid float64") + return nil, fmt.Errorf("unable to parse response from command: %v", cfg.Script) } - return v, nil + return data, nil } -func GetString(data any, query string) (string, error) { - res, err := ParseJq(data, query) +func errorString(r io.Reader) (string, error) { + b := strings.Builder{} + _, err := io.Copy(&b, r) if err != nil { return "", err } - v, ok := res[0].(string) - if !ok { - return "", fmt.Errorf("invalid data. jq expression must return valid string") - } - return v, nil -} - -func GetArray(data any, query string) ([]any, error) { - res, err := ParseJq(data, query) - if err != nil { - return nil, err - } - v, ok := res[0].([]interface{}) - if !ok { - return v, fmt.Errorf("invalid data. jq expression must return valid string") - } - return v, nil -} - -func ParseJq(data any, qs string) ([]any, error) { - q, err := jp.ParseString(qs) - if err != nil { - return nil, fmt.Errorf("unable to parse jq selector: %w", err) - } - result := q.Get(data) - if len(result) != 1 { - return nil, fmt.Errorf("empty data returned from jq expression: %s", qs) - } - return result, nil + return b.String(), nil } diff --git a/internal/scrape/script/script_test.go b/internal/scrape/script/script_test.go index 907f629..12bab15 100644 --- a/internal/scrape/script/script_test.go +++ b/internal/scrape/script/script_test.go @@ -1,99 +1,54 @@ -package script +package script_test import ( "context" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" "reflect" "testing" - "time" ) -func TestScrapper_Run(t *testing.T) { - type fields struct { - cfg Config - } +func TestRun(t *testing.T) { tests := []struct { name string - fields fields - want []Data + args script.Config + want []script.Data wantErr bool }{ { - name: "Scrape items OK", - fields: fields{ - cfg: Config{ - Command: "echo {\"items\":[{\"v\": 1,\"name\":\"a\"},{\"v\":2,\"name\":\"b\"}]}", - List: ".items", - Value: ".v", - Labels: []Label{ - { - Name: "name", - JqValue: ".name", - }, - }, - }, + name: "Parsing command OK", + args: script.Config{ + Name: "metric_1", + Help: "", + Script: "echo \"region=eu-central-1,cluster=eks-dev-1,type=dev,2\"", }, - want: []Data{ - { - Value: 1, - Labels: map[string]string{ - "name": "a", - }, - }, + want: []script.Data{ { Value: 2, Labels: map[string]string{ - "name": "b", - }, - }, - }, - wantErr: false, - }, - { - name: "Scrape one value OK", - fields: fields{ - cfg: Config{ - Command: "echo {\"v\": 1,\"name\":\"a\"}", - Value: ".v", - Labels: []Label{ - { - Name: "name", - JqValue: ".name", - }, - }, - }, - }, - want: []Data{ - { - Value: 1, - Labels: map[string]string{ - "name": "a", + "region": "eu-central-1", + "cluster": "eks-dev-1", + "type": "dev", }, }, }, wantErr: false, }, { - name: "Scrape cmd Json Error", - fields: fields{ - cfg: Config{ - Command: "echo not json", - Value: ".v", - Labels: []Label{ - { - Name: "name", - JqValue: ".name", - }, - }, - }, + name: "Invalid command", + args: script.Config{ + Name: "metric_1", + Help: "", + Script: "invalid command", }, + want: nil, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - got, err := Run(ctx, tt.fields.cfg) - cancel() + ctx, cancelCtx := context.WithCancel(context.Background()) + defer cancelCtx() + got, err := script.Run(ctx, tt.args) if (err != nil) != tt.wantErr { t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/jqdata/parse.go b/pkg/jqdata/parse.go new file mode 100644 index 0000000..fae19e2 --- /dev/null +++ b/pkg/jqdata/parse.go @@ -0,0 +1,40 @@ +package jqdata + +import ( + "context" + "encoding/json" + "fmt" + "github.com/itchyny/gojq" +) + +type JsonData struct { + d map[string]any +} + +func ParseRawJSON(data []byte) (JsonData, error) { + d := make(map[string]any) + err := json.Unmarshal(data, &d) + if err != nil { + return JsonData{}, fmt.Errorf("unable parse JSON: %w", err) + } + return JsonData{d: d}, nil +} + +func (j JsonData) Query(ctx context.Context, q string) (any, error) { + qr, err := gojq.Parse(q) + if err != nil { + return nil, err + } + it := qr.RunWithContext(ctx, j.d) + for { + v, ok := it.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return nil, err + } + return v, nil + } + return nil, fmt.Errorf("empty data for query: %v", q) +} diff --git a/pkg/jqdata/parse_test.go b/pkg/jqdata/parse_test.go new file mode 100644 index 0000000..54c02c9 --- /dev/null +++ b/pkg/jqdata/parse_test.go @@ -0,0 +1,59 @@ +package jqdata + +import ( + "context" + "encoding/json" + "github.com/stretchr/testify/assert" + "reflect" + "testing" + "time" +) + +func TestJsonData_Query(t *testing.T) { + data := map[string]any{ + "v": 1, + "s": "s", + "l": []string{"1", "2", "3"}, + } + j, err := json.Marshal(data) + assert.NoError(t, err) + type args struct { + q string + } + tests := []struct { + name string + j func() JsonData + args args + want any + wantErr bool + }{ + { + name: "Query OK", + j: func() JsonData { + jd, err := ParseRawJSON(j) + assert.NoError(t, err) + return jd + }, + args: args{ + q: ".l | length", + }, + want: 3, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + jd := tt.j() + got, err := jd.Query(ctx, tt.args.q) + if (err != nil) != tt.wantErr { + t.Errorf("Query() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Query() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/test/configs/eip.yaml b/test/configs/eip.yaml index a3a98ac..4d53e32 100644 --- a/test/configs/eip.yaml +++ b/test/configs/eip.yaml @@ -4,11 +4,4 @@ quotas: metrics: - name: "route53_hosted_zone_records" help: "Number of resource sets in hosted zone" - command: "aws route53 list-hosted-zones" - list: ".HostedZones" - value: ".ResourceRecordSetCount" - labels: - - name: "id" - jqValue: ".Id" - - name: "name" - jqValue: ".Name" \ No newline at end of file + script: "aws route53 list-hosted-zones | jq -r '.HostedZones[] | \"id=\(.Id),name=\(.Name),\(.ResourceRecordSetCount)\"'" From 5370beb2827a25da392c881f575e29d870d99b06 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Thu, 21 Mar 2024 11:35:56 +0100 Subject: [PATCH 20/32] list --- internal/exporter/exporter.go | 6 ++++-- internal/scrape/quotas/collector.go | 8 +++++--- internal/scrape/script/collector.go | 11 ++++++----- internal/scrape/script/script.go | 2 +- pkg/jqdata/parse.go | 10 +++++----- pkg/jqdata/parse_test.go | 4 ++-- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index b724714..0590096 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -74,8 +74,9 @@ func (e *Exporter) scrape(ctx context.Context) error { defer cancel() g, ctx := errgroup.WithContext(ctx) for _, c := range e.cls { + cl := c g.Go(func() error { - return c.Collect(ctx) + return cl.Collect(ctx) }) } err := g.Wait() @@ -87,8 +88,9 @@ func (e *Exporter) register(ctx context.Context) error { defer cancel() g, ctx := errgroup.WithContext(ctx) for _, c := range e.cls { + cl := c g.Go(func() error { - return c.Register(ctx, e.r) + return cl.Register(ctx, e.r) }) } err := g.Wait() diff --git a/internal/scrape/quotas/collector.go b/internal/scrape/quotas/collector.go index 41253bd..eba575d 100644 --- a/internal/scrape/quotas/collector.go +++ b/internal/scrape/quotas/collector.go @@ -54,14 +54,15 @@ func (c *Collector) Register(ctx context.Context, r *prometheus.Registry) error r.MustRegister(gvq) g, ctx := errgroup.WithContext(ctx) for _, cf := range c.cfg { + qc := cf g.Go(func() error { - res, err := c.qcl.GetQuota(ctx, cf.ServiceCode, cf.QuotaCode, quota.WithRegion(cf.Region)) + res, err := c.qcl.GetQuota(ctx, qc.ServiceCode, qc.QuotaCode, quota.WithRegion(qc.Region)) if err != nil { return err } t := task{ m: gvq, - cfg: cf, + cfg: qc, } c.addTask(t) setMetric(t.m, res) @@ -76,7 +77,8 @@ func (c *Collector) Register(ctx context.Context, r *prometheus.Registry) error func (c *Collector) Collect(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) for _, t := range c.tasks { - g.Go(t.run(ctx, c.qcl)) + ts := t + g.Go(ts.run(ctx, c.qcl)) } err := g.Wait() return err diff --git a/internal/scrape/script/collector.go b/internal/scrape/script/collector.go index 6a36039..6be2b8a 100644 --- a/internal/scrape/script/collector.go +++ b/internal/scrape/script/collector.go @@ -33,23 +33,24 @@ func (c *Collector) Register(ctx context.Context, r *prometheus.Registry) error c.log.Debugf("start registering script metrics") g, ctx := errgroup.WithContext(ctx) for _, cf := range c.cfg { + config := cf g.Go(func() error { - data, err := Run(ctx, cf) + data, err := Run(ctx, config) if err != nil { - c.log.Errorf("unable to run command: %s, %v", cf.Script, err) + c.log.Errorf("unable to run command: %s, %v", config.Script, err) return err } if len(data) > 0 { lbs := data[0].LabelNames() m := prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: c.ns, - Name: cf.Name, - Help: cf.Help, + Name: config.Name, + Help: config.Help, }, lbs) r.MustRegister(m) t := task{ m: m, - cfg: cf, + cfg: config, } c.addTask(t) for _, d := range data { diff --git a/internal/scrape/script/script.go b/internal/scrape/script/script.go index 2cacfe8..ef98ace 100644 --- a/internal/scrape/script/script.go +++ b/internal/scrape/script/script.go @@ -20,7 +20,7 @@ func (d Data) LabelNames() []string { return []string{} } r := make([]string, 0) - for k, _ := range d.Labels { + for k := range d.Labels { r = append(r, k) } return r diff --git a/pkg/jqdata/parse.go b/pkg/jqdata/parse.go index fae19e2..82bdb1c 100644 --- a/pkg/jqdata/parse.go +++ b/pkg/jqdata/parse.go @@ -7,20 +7,20 @@ import ( "github.com/itchyny/gojq" ) -type JsonData struct { +type JSONData struct { d map[string]any } -func ParseRawJSON(data []byte) (JsonData, error) { +func ParseRawJSON(data []byte) (JSONData, error) { d := make(map[string]any) err := json.Unmarshal(data, &d) if err != nil { - return JsonData{}, fmt.Errorf("unable parse JSON: %w", err) + return JSONData{}, fmt.Errorf("unable parse JSON: %w", err) } - return JsonData{d: d}, nil + return JSONData{d: d}, nil } -func (j JsonData) Query(ctx context.Context, q string) (any, error) { +func (j JSONData) Query(ctx context.Context, q string) (any, error) { qr, err := gojq.Parse(q) if err != nil { return nil, err diff --git a/pkg/jqdata/parse_test.go b/pkg/jqdata/parse_test.go index 54c02c9..4906671 100644 --- a/pkg/jqdata/parse_test.go +++ b/pkg/jqdata/parse_test.go @@ -22,14 +22,14 @@ func TestJsonData_Query(t *testing.T) { } tests := []struct { name string - j func() JsonData + j func() JSONData args args want any wantErr bool }{ { name: "Query OK", - j: func() JsonData { + j: func() JSONData { jd, err := ParseRawJSON(j) assert.NoError(t, err) return jd From c4c0b80d6542b404ec33fedff434af91bad81796 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Thu, 21 Mar 2024 11:45:03 +0100 Subject: [PATCH 21/32] pre commit hook --- .github/workflows/golangci-lint.yml | 2 +- .github/workflows/helm-lint-test.yml | 120 +++++++++++++-------------- .pre-commit-config.yaml | 29 ++++--- 3 files changed, 78 insertions(+), 73 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 11b857d..94e4a86 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -14,4 +14,4 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v4 with: - version: v1.56.2 + version: v1.57.1 diff --git a/.github/workflows/helm-lint-test.yml b/.github/workflows/helm-lint-test.yml index f25038c..eaab8b5 100644 --- a/.github/workflows/helm-lint-test.yml +++ b/.github/workflows/helm-lint-test.yml @@ -1,60 +1,60 @@ -name: Lint and Test Charts - -on: pull_request - -env: - HELM_VERSION: 3.14.0 - -jobs: - lint-test: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Helm - uses: azure/setup-helm@v4 - with: - version: ${{ env.HELM_VERSION }} - - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - check-latest: true - - - name: Set up chart-testing - uses: helm/chart-testing-action@v2.6.1 - - - name: Run chart-testing (list-changed) - id: list-changed - run: | - changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) - if [[ -n "$changed" ]]; then - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - - - name: Run chart-testing (lint) - if: steps.list-changed.outputs.changed == 'true' - run: ct lint --target-branch ${{ github.event.repository.default_branch }} --validate-maintainers=false - - - name: Create kind cluster - if: steps.list-changed.outputs.changed == 'true' - uses: helm/kind-action@v1.9.0 - - - name: Run chart-testing (install) - if: steps.list-changed.outputs.changed == 'true' - run: ct install --target-branch ${{ github.event.repository.default_branch }} - - linter-artifacthub: - runs-on: ubuntu-22.04 - container: - image: artifacthub/ah - options: --user root - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Run ah lint - working-directory: ./charts/ - run: ah lint +#name: Lint and Test Charts +# +#on: pull_request +# +#env: +# HELM_VERSION: 3.14.0 +# +#jobs: +# lint-test: +# runs-on: ubuntu-22.04 +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# +# - name: Set up Helm +# uses: azure/setup-helm@v4 +# with: +# version: ${{ env.HELM_VERSION }} +# +# - uses: actions/setup-python@v5 +# with: +# python-version: '3.11' +# check-latest: true +# +# - name: Set up chart-testing +# uses: helm/chart-testing-action@v2.6.1 +# +# - name: Run chart-testing (list-changed) +# id: list-changed +# run: | +# changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) +# if [[ -n "$changed" ]]; then +# echo "changed=true" >> "$GITHUB_OUTPUT" +# fi +# +# - name: Run chart-testing (lint) +# if: steps.list-changed.outputs.changed == 'true' +# run: ct lint --target-branch ${{ github.event.repository.default_branch }} --validate-maintainers=false +# +# - name: Create kind cluster +# if: steps.list-changed.outputs.changed == 'true' +# uses: helm/kind-action@v1.9.0 +# +# - name: Run chart-testing (install) +# if: steps.list-changed.outputs.changed == 'true' +# run: ct install --target-branch ${{ github.event.repository.default_branch }} +# +# linter-artifacthub: +# runs-on: ubuntu-22.04 +# container: +# image: artifacthub/ah +# options: --user root +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 +# - name: Run ah lint +# working-directory: ./charts/ +# run: ah lint diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 108a5cf..30fa62a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,17 +9,22 @@ repos: - id: detect-private-key - id: end-of-file-fixer - - repo: https://github.com/gruntwork-io/pre-commit - rev: v0.1.17 + - repo: https://github.com/golangci/golangci-lint + rev: v1.57.1 hooks: - - id: helmlint + - id: golangci-lint - - repo: https://github.com/norwoodj/helm-docs - rev: v1.13.0 - hooks: - - id: helm-docs - args: - - --chart-search-root=charts - - id: helm-docs-built - args: - - --chart-search-root=charts +# - repo: https://github.com/gruntwork-io/pre-commit +# rev: v0.1.17 +# hooks: +# - id: helmlint +# +# - repo: https://github.com/norwoodj/helm-docs +# rev: v1.13.0 +# hooks: +# - id: helm-docs +# args: +# - --chart-search-root=charts +# - id: helm-docs-built +# args: +# - --chart-search-root=charts From dde29667bb62a915b1532d11e5d5293bbc00d2e6 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Fri, 22 Mar 2024 11:20:44 +0100 Subject: [PATCH 22/32] change example --- .gitignore | 2 ++ config/example.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2f222d3..bd2f25d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ go.work .idea coverage.out + +bin/* diff --git a/config/example.yaml b/config/example.yaml index 6ec5e14..6914499 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -4,4 +4,4 @@ quotas: metrics: - name: "route53_hosted_zone_records" help: "Number of resource sets in hosted zone" - script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),\\(.ResourceRecordSetCount)\"\'" + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" From d86244f8bc9068753a90cee7f733aa23a41e3367 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 25 Mar 2024 06:47:18 +0100 Subject: [PATCH 23/32] worflows change + config --- .github/workflows/docker-master.yaml | 51 ---------------- .../{docker-release.yaml => docker.yaml} | 28 +++++++-- .github/workflows/go-binary-release.yml | 44 ++++++++------ .github/workflows/helm-lint-test.yml | 60 ------------------- .goreleaser.yaml | 12 ++++ .tool-versions | 1 + .../aws-service-quotas-exporter-values.yaml | 28 ++++----- .../aws-service-quotas-exporter/values.yaml | 5 ++ config/example.yaml | 3 + internal/app/application.go | 6 +- internal/scrape/config.go | 16 ++++- internal/scrape/config_test.go | 51 ++++++++++++++++ 12 files changed, 148 insertions(+), 157 deletions(-) delete mode 100644 .github/workflows/docker-master.yaml rename .github/workflows/{docker-release.yaml => docker.yaml} (67%) delete mode 100644 .github/workflows/helm-lint-test.yml create mode 100644 .goreleaser.yaml create mode 100644 internal/scrape/config_test.go diff --git a/.github/workflows/docker-master.yaml b/.github/workflows/docker-master.yaml deleted file mode 100644 index dbf040f..0000000 --- a/.github/workflows/docker-master.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: Build and publish latest Docker images - -on: - workflow_dispatch: - push: - branches: - - main - -jobs: - build-latest-image: - runs-on: ubuntu-22.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Github Packages - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ghcr.io/lablabs/aws-service-quotas-exporter - # generate Docker tags based on the following events/attributes - tags: | - type=raw,value=latest - - - name: Build image and push to GitHub Container Registry - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - push: true - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker.yaml similarity index 67% rename from .github/workflows/docker-release.yaml rename to .github/workflows/docker.yaml index eb53e15..3da146d 100644 --- a/.github/workflows/docker-release.yaml +++ b/.github/workflows/docker.yaml @@ -1,9 +1,18 @@ name: Build and publish Docker images +#on: +# push: +# branches: +# - main +# - master +# tags: +# - 'v*' +# release: +# types: [published] + on: + pull_request: push: - tags: - - '*' jobs: build-image: @@ -26,7 +35,18 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta + - name: Docker meta for latest + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + tags: | + type=raw,value=latest + + - name: Docker meta for tag + if: startsWith(github.ref, 'refs/tags/') id: meta uses: docker/metadata-action@v5 with: @@ -47,4 +67,4 @@ jobs: push: true - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/go-binary-release.yml b/.github/workflows/go-binary-release.yml index 91987b3..c3ea79e 100644 --- a/.github/workflows/go-binary-release.yml +++ b/.github/workflows/go-binary-release.yml @@ -1,26 +1,34 @@ name: Generate release binary artifacts +#on: +# release: +# types: +# - created + on: - release: - types: - - created + pull_request: + push: + +permissions: + contents: write + jobs: - generate: - name: Generate cross-platform builds - runs-on: ubuntu-22.04 + goreleaser: + runs-on: ubuntu-latest steps: - - name: Checkout the repository + - + name: Checkout uses: actions/checkout@v4 - - name: Generate build files - uses: thatisuday/go-cross-build@v1 with: - platforms: 'linux/amd64, darwin/amd64, linux/arm64' - package: '' - name: 'aws-service-quotas-exporter' - compress: 'false' - dest: 'dist' - - name: Copy build-artifacts - uses: skx/github-action-publish-binaries@master + fetch-depth: 0 + - + name: Set up Go + uses: actions/setup-go@v5 + - + name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - args: "./dist/*" diff --git a/.github/workflows/helm-lint-test.yml b/.github/workflows/helm-lint-test.yml deleted file mode 100644 index eaab8b5..0000000 --- a/.github/workflows/helm-lint-test.yml +++ /dev/null @@ -1,60 +0,0 @@ -#name: Lint and Test Charts -# -#on: pull_request -# -#env: -# HELM_VERSION: 3.14.0 -# -#jobs: -# lint-test: -# runs-on: ubuntu-22.04 -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# -# - name: Set up Helm -# uses: azure/setup-helm@v4 -# with: -# version: ${{ env.HELM_VERSION }} -# -# - uses: actions/setup-python@v5 -# with: -# python-version: '3.11' -# check-latest: true -# -# - name: Set up chart-testing -# uses: helm/chart-testing-action@v2.6.1 -# -# - name: Run chart-testing (list-changed) -# id: list-changed -# run: | -# changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) -# if [[ -n "$changed" ]]; then -# echo "changed=true" >> "$GITHUB_OUTPUT" -# fi -# -# - name: Run chart-testing (lint) -# if: steps.list-changed.outputs.changed == 'true' -# run: ct lint --target-branch ${{ github.event.repository.default_branch }} --validate-maintainers=false -# -# - name: Create kind cluster -# if: steps.list-changed.outputs.changed == 'true' -# uses: helm/kind-action@v1.9.0 -# -# - name: Run chart-testing (install) -# if: steps.list-changed.outputs.changed == 'true' -# run: ct install --target-branch ${{ github.event.repository.default_branch }} -# -# linter-artifacthub: -# runs-on: ubuntu-22.04 -# container: -# image: artifacthub/ah -# options: --user root -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 -# - name: Run ah lint -# working-directory: ./charts/ -# run: ah lint diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..4e717a7 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,12 @@ +builds: + - id: "exporter" + main: ./cmd/exporter + binary: exporter + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 + mod_timestamp: "{{ .CommitTimestamp }}" diff --git a/.tool-versions b/.tool-versions index 556137a..e20dbc3 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,4 @@ helm 3.14.2 awscli 2.7.14 pre-commit 2.20.0 +golang 1.22.0 diff --git a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml index b65feaa..2663af4 100644 --- a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml +++ b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml @@ -1,18 +1,10 @@ -exporter: - config: - quotas: - - serviceCode: "ec2" - quotaCode: "L-0263D0A3" - metrics: - - name: "route53_hosted_zone_records" - help: "Number of resource sets in hosted zone" - command: "aws route53 list-hosted-zones" - list: ".HostedZones" - value: ".ResourceRecordSetCount" - labels: - - name: "id" - jqValue: ".Id" - - name: "name" - jqValue: ".Name" - - name: "quota" - value: "L-0263D0A3" +scrape: + interval: "60s" + timeout: "5s" +quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" +metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" diff --git a/charts/aws-service-quotas-exporter/values.yaml b/charts/aws-service-quotas-exporter/values.yaml index 2ccf63b..f39aa5b 100644 --- a/charts/aws-service-quotas-exporter/values.yaml +++ b/charts/aws-service-quotas-exporter/values.yaml @@ -45,6 +45,8 @@ service: env: +# - name: "AWS_REGION" +# value: "eu-central-1" resources: {} # We usually recommend not to specify default resources and to leave this as a conscious @@ -118,3 +120,6 @@ exporter: level: "DEBUG" format: "json" config: +# scrape: +# quotas: [] +# metrics: [] diff --git a/config/example.yaml b/config/example.yaml index 6914499..2663af4 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -1,3 +1,6 @@ +scrape: + interval: "60s" + timeout: "5s" quotas: - serviceCode: "ec2" quotaCode: "L-0263D0A3" diff --git a/internal/app/application.go b/internal/app/application.go index 8fa3abb..23ed600 100644 --- a/internal/app/application.go +++ b/internal/app/application.go @@ -15,7 +15,7 @@ import ( ) const ( - PrometheusNamespace = "quota_exporter" + PrometheusNamespace = "aws_quota_exporter" ) func NewApplication(log *logrus.Logger, cfg Config) (*Application, error) { @@ -84,7 +84,7 @@ func (a *Application) Run(ctx context.Context) error { func exporterOptions(cfg *scrape.Config) []exporter.Option { return []exporter.Option{ - exporter.WithInterval(cfg.Global.Interval), - exporter.WithTimeout(cfg.Global.Timeout), + exporter.WithInterval(cfg.Scrape.Interval), + exporter.WithTimeout(cfg.Scrape.Timeout), } } diff --git a/internal/scrape/config.go b/internal/scrape/config.go index 655296d..0bc1b0f 100644 --- a/internal/scrape/config.go +++ b/internal/scrape/config.go @@ -8,13 +8,23 @@ import ( "time" ) -type Global struct { +type Scrape struct { Interval time.Duration `json:"interval,omitempty" yaml:"interval,omitempty"` Timeout time.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` } +func (s *Scrape) Validate() error { + if s.Interval != 0 && s.Interval < time.Minute { + return fmt.Errorf("scrape.interval is not valid. Minimal value is 60s") + } + if s.Timeout != 0 && s.Timeout < (time.Second*5) { + return fmt.Errorf("scrape.timeout is not valid. Minimal value is 5s") + } + return nil +} + type Config struct { - Global Global `json:"global,omitempty" yaml:"global,omitempty"` + Scrape Scrape `json:"scrape,omitempty" yaml:"scrape,omitempty"` Quotas []quotas.Config `json:"quotas,omitempty" yaml:"quotas,omitempty"` Metrics []script.Config `json:"metrics,omitempty" yaml:"metrics,omitempty"` } @@ -30,7 +40,7 @@ func (c *Config) Validate() error { return err } } - return nil + return c.Scrape.Validate() } func LoadAndValidateConfig(path string) (*Config, error) { diff --git a/internal/scrape/config_test.go b/internal/scrape/config_test.go new file mode 100644 index 0000000..9ededa3 --- /dev/null +++ b/internal/scrape/config_test.go @@ -0,0 +1,51 @@ +package scrape + +import ( + "testing" + "time" +) + +func TestScrape_Validate(t *testing.T) { + type fields struct { + Interval time.Duration + Timeout time.Duration + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "Scrape config not set", + fields: fields{}, + wantErr: false, + }, + { + name: "Scrape config OK", + fields: fields{ + Interval: time.Minute + time.Second*5, + Timeout: time.Second * 6, + }, + wantErr: false, + }, + { + name: "Scrape config error", + fields: fields{ + Interval: time.Minute - time.Second, + Timeout: time.Second * 4, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Scrape{ + Interval: tt.fields.Interval, + Timeout: tt.fields.Timeout, + } + if err := s.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From 3a4b2cd387d6dab11c58dca80f1a8897717ff02a Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 25 Mar 2024 06:53:05 +0100 Subject: [PATCH 24/32] docker flow --- .github/workflows/docker.yaml | 48 ++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 3da146d..723cff9 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -15,8 +15,9 @@ on: push: jobs: - build-image: + latest: runs-on: ubuntu-22.04 + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -36,7 +37,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Docker meta for latest - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} id: meta uses: docker/metadata-action@v5 with: @@ -47,7 +47,7 @@ jobs: - name: Docker meta for tag if: startsWith(github.ref, 'refs/tags/') - id: meta + id: meta-tag uses: docker/metadata-action@v5 with: images: | @@ -68,3 +68,45 @@ jobs: - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} + tag: + runs-on: ubuntu-22.04 + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Github Packages + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta for latest + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + tags: | + type=raw,value=latest + + - name: Build image and push to GitHub Container Registry + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} From 6bc4586652dbf6c4a5e5b9c9fe9d727e3a64c237 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 25 Mar 2024 07:21:43 +0100 Subject: [PATCH 25/32] workflow v2 --- .github/workflows/docker.yaml | 2 +- .github/workflows/go-binary-release.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 723cff9..a7bcf1e 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -70,7 +70,7 @@ jobs: run: echo ${{ steps.docker_build.outputs.digest }} tag: runs-on: ubuntu-22.04 - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + if: startsWith(github.ref, 'refs/tags/') steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/go-binary-release.yml b/.github/workflows/go-binary-release.yml index c3ea79e..3c5a719 100644 --- a/.github/workflows/go-binary-release.yml +++ b/.github/workflows/go-binary-release.yml @@ -25,6 +25,7 @@ jobs: uses: actions/setup-go@v5 - name: Run GoReleaser + if: startsWith(github.ref, 'refs/tags/') uses: goreleaser/goreleaser-action@v5 with: distribution: goreleaser @@ -32,3 +33,13 @@ jobs: args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - + name: Run GoReleaser snapshot + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean --snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From ef39c8cfafdcc48d9eecc7b1aa6b02d67d79e8d6 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 25 Mar 2024 09:14:31 +0100 Subject: [PATCH 26/32] change trigger for docker and go bin --- .github/workflows/docker.yaml | 18 +++++++----------- .github/workflows/go-binary-release.yml | 11 ++++++----- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index a7bcf1e..fce4d6e 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,18 +1,14 @@ name: Build and publish Docker images -#on: -# push: -# branches: -# - main -# - master -# tags: -# - 'v*' -# release: -# types: [published] - on: - pull_request: push: + branches: + - main + tags: + - 'v*' + release: + types: [published] + jobs: latest: diff --git a/.github/workflows/go-binary-release.yml b/.github/workflows/go-binary-release.yml index 3c5a719..eea2002 100644 --- a/.github/workflows/go-binary-release.yml +++ b/.github/workflows/go-binary-release.yml @@ -1,12 +1,13 @@ name: Generate release binary artifacts -#on: -# release: -# types: -# - created on: - pull_request: push: + branches: + - main + tags: + - 'v*' + release: + types: [published] permissions: contents: write From b9a8b2367c65f0125ab2656f784e41fffb7976ec Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 25 Mar 2024 09:20:16 +0100 Subject: [PATCH 27/32] add example with private zone flag --- test/configs/eip.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/configs/eip.yaml b/test/configs/eip.yaml index 4d53e32..6914499 100644 --- a/test/configs/eip.yaml +++ b/test/configs/eip.yaml @@ -4,4 +4,4 @@ quotas: metrics: - name: "route53_hosted_zone_records" help: "Number of resource sets in hosted zone" - script: "aws route53 list-hosted-zones | jq -r '.HostedZones[] | \"id=\(.Id),name=\(.Name),\(.ResourceRecordSetCount)\"'" + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" From 87f6030a65d7f14f3e908f511140b986a69d0108 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Mon, 25 Mar 2024 09:23:59 +0100 Subject: [PATCH 28/32] MR comments --- .../aws-service-quotas-exporter-values.yaml | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml index 2663af4..fbe8465 100644 --- a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml +++ b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml @@ -1,10 +1,16 @@ -scrape: - interval: "60s" - timeout: "5s" -quotas: - - serviceCode: "ec2" - quotaCode: "L-0263D0A3" -metrics: - - name: "route53_hosted_zone_records" - help: "Number of resource sets in hosted zone" - script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" +exporter: + address: "0.0.0.0:8080" + log: + level: "DEBUG" + format: "json" + config: + scrape: + interval: "60s" + timeout: "5s" + quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" + metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" From 126185b69c559b2a7fb91f68b8c00e48245047f9 Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 26 Mar 2024 07:22:52 +0100 Subject: [PATCH 29/32] rename eip to route53.yaml --- test/config.go | 2 +- test/configs/{eip.yaml => route53.yaml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename test/configs/{eip.yaml => route53.yaml} (100%) diff --git a/test/config.go b/test/config.go index 40c119a..ad3e4f9 100644 --- a/test/config.go +++ b/test/config.go @@ -7,7 +7,7 @@ import ( "testing" ) -//go:embed configs/eip.yaml +//go:embed configs/route53.yaml var configEip string func TmpConfigMetricFile(t *testing.T) string { diff --git a/test/configs/eip.yaml b/test/configs/route53.yaml similarity index 100% rename from test/configs/eip.yaml rename to test/configs/route53.yaml From 494ed41689e50dde741f4aa73f953e2e35fbd5ec Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 26 Mar 2024 12:55:45 +0100 Subject: [PATCH 30/32] MR comments --- Dockerfile | 2 +- .../templates/deployment.yaml | 2 - .../templates/scrape-config.yaml | 4 +- .../templates/servicemonitor.yaml | 8 ++-- .../aws-service-quotas-exporter/values.yaml | 44 +++++++------------ internal/http/http.go | 10 +++++ 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0f6d327..c5b22fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN cd cmd/exporter && \ FROM alpine:${ALPINE_VERSION} -RUN apk update && apk add jq +RUN apk update && apk add jq bash COPY --from=builder_aws_cli /usr/local/lib/aws-cli/ /usr/local/lib/aws-cli/ RUN ln -s /usr/local/lib/aws-cli/aws /usr/local/bin/aws diff --git a/charts/aws-service-quotas-exporter/templates/deployment.yaml b/charts/aws-service-quotas-exporter/templates/deployment.yaml index a1c28d1..ad9c010 100644 --- a/charts/aws-service-quotas-exporter/templates/deployment.yaml +++ b/charts/aws-service-quotas-exporter/templates/deployment.yaml @@ -5,9 +5,7 @@ metadata: labels: {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} spec: - {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} - {{- end }} selector: matchLabels: {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 6 }} diff --git a/charts/aws-service-quotas-exporter/templates/scrape-config.yaml b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml index 39ebc46..c2fd92f 100644 --- a/charts/aws-service-quotas-exporter/templates/scrape-config.yaml +++ b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml @@ -6,5 +6,7 @@ metadata: labels: {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} data: + {{- if .Values.exporter.config }} scrape.yaml: | - {{- .Values.exporter.config | toYaml | nindent 4 }} + {{- toYaml .Values.exporter.config | nindent 4 }} + {{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml b/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml index 030bd80..9a6c673 100644 --- a/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml +++ b/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml @@ -17,8 +17,8 @@ spec: {{- if .Values.serviceMonitor.interval }} interval: {{ .Values.serviceMonitor.interval }} {{- end }} -{{- if .Values.serviceMonitor.telemetryPath }} - path: {{ .Values.serviceMonitor.telemetryPath }} +{{- if .Values.serviceMonitor.path }} + path: {{ .Values.serviceMonitor.path }} {{- end }} {{- if .Values.serviceMonitor.timeout }} scrapeTimeout: {{ .Values.serviceMonitor.timeout }} @@ -34,7 +34,7 @@ spec: jobLabel: {{ template "aws-service-quotas-exporter.fullname" . }} namespaceSelector: matchNames: - - {{ .Release.Namespace }} + - {{ .Values.serviceMonitor.namespace | default .Release.Namespace }} selector: matchLabels: {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 6 }} @@ -44,4 +44,4 @@ spec: - {{ . }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/aws-service-quotas-exporter/values.yaml b/charts/aws-service-quotas-exporter/values.yaml index f39aa5b..a64bdb9 100644 --- a/charts/aws-service-quotas-exporter/values.yaml +++ b/charts/aws-service-quotas-exporter/values.yaml @@ -31,13 +31,12 @@ podLabels: {} podSecurityContext: {} # fsGroup: 2000 -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 +securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + add: ["NET_ADMIN", "NET_RAW"] service: type: ClusterIP @@ -48,34 +47,25 @@ env: # - name: "AWS_REGION" # value: "eu-central-1" -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. +resources: + # This was measured for 5 quotas and 5 metrics script + requests: + cpu: 400m + memory: 200Mi # limits: # cpu: 100m # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + livenessProbe: httpGet: - path: / + path: /liveness port: http readinessProbe: httpGet: - path: / + path: /readiness port: http -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - # Additional volumes on the output Deployment definition. volumes: [] # - name: foo @@ -104,8 +94,8 @@ serviceMonitor: labels: {} # Set how frequently Prometheus should scrape interval: 30s - # Set path to redis-exporter telemtery-path - telemetryPath: /metrics + # Set path to metrics endpoint + path: /metrics # Set timeout for scrape timeout: 10s # Set relabel_configs as per https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config @@ -119,7 +109,7 @@ exporter: log: level: "DEBUG" format: "json" - config: + config: {} # scrape: # quotas: [] # metrics: [] diff --git a/internal/http/http.go b/internal/http/http.go index 9688a6a..9bf75ab 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -28,6 +28,7 @@ func NewHTTP(log *logrus.Logger, address string, registry *prometheus.Registry) s: &s, } RegisterMetricEndpoint(handler, registry) + RegisterStatusEndpoint(handler, &h) return &h, nil } @@ -61,8 +62,17 @@ func (h *HTTP) Run(ctx context.Context) error { return nil } +func (h *HTTP) Status(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) +} + func RegisterMetricEndpoint(mux *http.ServeMux, registry *prometheus.Registry) { registry.MustRegister(collectors.NewGoCollector()) registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{Registry: registry})) } + +func RegisterStatusEndpoint(mux *http.ServeMux, h *HTTP) { + mux.HandleFunc("/liveness", h.Status) + mux.HandleFunc("/readiness", h.Status) +} From 457faa1030739c9c4e1fe5d6a7c8f172ee08829a Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 26 Mar 2024 12:59:57 +0100 Subject: [PATCH 31/32] fix for context cancelation --- pkg/service/mng.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/service/mng.go b/pkg/service/mng.go index d73abad..993d5e6 100644 --- a/pkg/service/mng.go +++ b/pkg/service/mng.go @@ -2,6 +2,7 @@ package service import ( "context" + "errors" "golang.org/x/sync/errgroup" ) @@ -31,7 +32,7 @@ func (m *Manager) StartAndWait(ctx context.Context) error { service := s group.Go(func() error { err := service.Run(ctx) - if err != nil { + if err != nil && !errors.Is(err, context.Canceled) { return err } <-ctx.Done() From f4fd8a7f443db31794f8c6f2c924a8451949341b Mon Sep 17 00:00:00 2001 From: rafajpet Date: Tue, 26 Mar 2024 14:02:11 +0100 Subject: [PATCH 32/32] capability drop all --- charts/aws-service-quotas-exporter/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/aws-service-quotas-exporter/values.yaml b/charts/aws-service-quotas-exporter/values.yaml index a64bdb9..6775122 100644 --- a/charts/aws-service-quotas-exporter/values.yaml +++ b/charts/aws-service-quotas-exporter/values.yaml @@ -36,7 +36,8 @@ securityContext: runAsNonRoot: true runAsUser: 1000 capabilities: - add: ["NET_ADMIN", "NET_RAW"] + drop: + - ALL service: type: ClusterIP