diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000..fce4d6e --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,108 @@ +name: Build and publish Docker images + +on: + push: + branches: + - main + tags: + - 'v*' + release: + types: [published] + + +jobs: + latest: + runs-on: ubuntu-22.04 + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Github Packages + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta for latest + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + tags: | + type=raw,value=latest + + - name: Docker meta for tag + if: startsWith(github.ref, 'refs/tags/') + id: meta-tag + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + # generate Docker tags based on the following events/attributes + tags: | + type=raw,value=${{ github.ref_name }} + + - name: Build image and push to GitHub Container Registry + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} + tag: + runs-on: ubuntu-22.04 + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Github Packages + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta for latest + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/lablabs/aws-service-quotas-exporter + tags: | + type=raw,value=latest + + - name: Build image and push to GitHub Container Registry + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/go-binary-release.yml b/.github/workflows/go-binary-release.yml new file mode 100644 index 0000000..eea2002 --- /dev/null +++ b/.github/workflows/go-binary-release.yml @@ -0,0 +1,46 @@ +name: Generate release binary artifacts + +on: + push: + branches: + - main + tags: + - 'v*' + release: + types: [published] + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - + name: Set up Go + uses: actions/setup-go@v5 + - + name: Run GoReleaser + if: startsWith(github.ref, 'refs/tags/') + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - + name: Run GoReleaser snapshot + if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + uses: goreleaser/goreleaser-action@v5 + with: + distribution: goreleaser + version: latest + args: release --clean --snapshot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml new file mode 100644 index 0000000..7063a44 --- /dev/null +++ b/.github/workflows/go-test.yml @@ -0,0 +1,28 @@ +name: Go tests + +on: + push: + pull_request: +permissions: + contents: read + +env: + GO_VERSION: '1.22.0' + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Get dependencies + run: | + go get -v -t -d ./... + - name: Test + env: + GOPROXY: "https://proxy.golang.org" + run: go test -v ./... diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..94e4a86 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,17 @@ +name: Golang lint +on: + push: + pull_request: +permissions: + contents: read + +jobs: + golangci: + name: Golang CI linter + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: v1.57.1 diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml new file mode 100644 index 0000000..41303b9 --- /dev/null +++ b/.github/workflows/helm-release.yml @@ -0,0 +1,41 @@ +name: Release Charts + +on: + push: + branches: + - main + +#on: +# push: +# pull_request: + +env: + HELM_VERSION: 3.14.0 + +jobs: + release: + # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions + # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token + permissions: + contents: write + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: ${{ env.HELM_VERSION }} + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.6.0 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitignore b/.gitignore index 3b735ec..bd2f25d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,12 @@ # Go workspace file go.work + +.env +.envrc + +.idea + +coverage.out + +bin/* diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..8032a42 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,14 @@ +linters: + enable: + - megacheck + - gofmt + - govet + - revive + +linters-settings: + staticcheck: + checks: + - '-SA5008' + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..4e717a7 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,12 @@ +builds: + - id: "exporter" + main: ./cmd/exporter + binary: exporter + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 + mod_timestamp: "{{ .CommitTimestamp }}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..30fa62a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: trailing-whitespace + - id: check-merge-conflict + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + - id: detect-private-key + - id: end-of-file-fixer + + - repo: https://github.com/golangci/golangci-lint + rev: v1.57.1 + hooks: + - id: golangci-lint + +# - repo: https://github.com/gruntwork-io/pre-commit +# rev: v0.1.17 +# hooks: +# - id: helmlint +# +# - repo: https://github.com/norwoodj/helm-docs +# rev: v1.13.0 +# hooks: +# - id: helm-docs +# args: +# - --chart-search-root=charts +# - id: helm-docs-built +# args: +# - --chart-search-root=charts diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..e20dbc3 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,4 @@ +helm 3.14.2 +awscli 2.7.14 +pre-commit 2.20.0 +golang 1.22.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c5b22fd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +ARG ALPINE_VERSION=3.19 +FROM python:3.10-alpine${ALPINE_VERSION} as builder_aws_cli + +ARG AWS_CLI_VERSION=2.15.19 +RUN apk add --no-cache git unzip groff build-base libffi-dev cmake +RUN git clone --single-branch --depth 1 -b ${AWS_CLI_VERSION} https://github.com/aws/aws-cli.git + +WORKDIR aws-cli +RUN ./configure --with-install-type=portable-exe --with-download-deps +RUN make +RUN make install + +# reduce image size: remove autocomplete and examples +RUN rm -rf \ + /usr/local/lib/aws-cli/aws_completer \ + /usr/local/lib/aws-cli/awscli/data/ac.index \ + /usr/local/lib/aws-cli/awscli/examples +RUN find /usr/local/lib/aws-cli/awscli/data -name completions-1*.json -delete +RUN find /usr/local/lib/aws-cli/awscli/botocore/data -name examples-1.json -delete +RUN (cd /usr/local/lib/aws-cli; for a in *.so*; do test -f /lib/$a && rm $a; done) + +FROM golang:1.22-alpine${ALPINE_VERSION} as builder_golang + +ARG GOOS=linux +ARG GOARCH=amd64 + +RUN apk --update add ca-certificates + +WORKDIR $GOPATH/src/github.com/lablabs/aws-service-quotas-exporter +COPY go.mod go.sum ./ +COPY . . +RUN go mod download +RUN go mod vendor +RUN go mod verify + +RUN cd cmd/exporter && \ + GOOS=$GOOS GOARCH=$GOARCH \ + CGO_ENABLED=0 \ + go build -o /aws-service-quotas-exporter . + +FROM alpine:${ALPINE_VERSION} + +RUN apk update && apk add jq bash + +COPY --from=builder_aws_cli /usr/local/lib/aws-cli/ /usr/local/lib/aws-cli/ +RUN ln -s /usr/local/lib/aws-cli/aws /usr/local/bin/aws + +COPY --from=builder_golang /aws-service-quotas-exporter . + +ENTRYPOINT ["/aws-service-quotas-exporter"] diff --git a/README.md b/README.md index 002342f..5b46a96 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,7 @@ # aws-service-quotas-exporter -AWS Quotas utilisation prometheus exporter + +AWS service quotas exporter exposes actual quotas for your AWS accounts and allow you to scrape actual +usage of AWS resources. Base on those two types of data, you can easily build +alert rules to prevent case when you are not able to provision another AWS resources due to reach the limit of AWS quota + +## AWS Co diff --git a/charts/aws-service-quotas-exporter/.helmignore b/charts/aws-service-quotas-exporter/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/aws-service-quotas-exporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/aws-service-quotas-exporter/Chart.yaml b/charts/aws-service-quotas-exporter/Chart.yaml new file mode 100644 index 0000000..0a3b98b --- /dev/null +++ b/charts/aws-service-quotas-exporter/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: aws-service-quotas-exporter +description: A Helm chart for Kubernetes + +type: application + +version: 0.0.1 + +appVersion: "v0.0.1" diff --git a/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml new file mode 100644 index 0000000..fbe8465 --- /dev/null +++ b/charts/aws-service-quotas-exporter/ci/aws-service-quotas-exporter-values.yaml @@ -0,0 +1,16 @@ +exporter: + address: "0.0.0.0:8080" + log: + level: "DEBUG" + format: "json" + config: + scrape: + interval: "60s" + timeout: "5s" + quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" + metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" diff --git a/charts/aws-service-quotas-exporter/templates/_helpers.tpl b/charts/aws-service-quotas-exporter/templates/_helpers.tpl new file mode 100644 index 0000000..f2a103d --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-service-quotas-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-service-quotas-exporter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-service-quotas-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "aws-service-quotas-exporter.labels" -}} +helm.sh/chart: {{ include "aws-service-quotas-exporter.chart" . }} +{{ include "aws-service-quotas-exporter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "aws-service-quotas-exporter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aws-service-quotas-exporter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "aws-service-quotas-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "aws-service-quotas-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/deployment.yaml b/charts/aws-service-quotas-exporter/templates/deployment.yaml new file mode 100644 index 0000000..ad9c010 --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/deployment.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "aws-service-quotas-exporter.fullname" . }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "aws-service-quotas-exporter.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "--log.level={{ .Values.exporter.log.level }}" + - "--log.format={{ .Values.exporter.log.format }}" + - "--config=/etc/exporter/scrape.yaml" + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: scrape-cfg + mountPath: /etc/exporter + readOnly: true + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: scrape-cfg + configMap: + name: {{ printf "%s-cfg" (include "aws-service-quotas-exporter.fullname" .) }} + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/scrape-config.yaml b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml new file mode 100644 index 0000000..c2fd92f --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/scrape-config.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-cfg" (include "aws-service-quotas-exporter.fullname" .) }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} +data: + {{- if .Values.exporter.config }} + scrape.yaml: | + {{- toYaml .Values.exporter.config | nindent 4 }} + {{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/service.yaml b/charts/aws-service-quotas-exporter/templates/service.yaml new file mode 100644 index 0000000..cd6efdc --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "aws-service-quotas-exporter.fullname" . }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 4 }} diff --git a/charts/aws-service-quotas-exporter/templates/serviceaccount.yaml b/charts/aws-service-quotas-exporter/templates/serviceaccount.yaml new file mode 100644 index 0000000..879c717 --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "aws-service-quotas-exporter.serviceAccountName" . }} + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml b/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000..9a6c673 --- /dev/null +++ b/charts/aws-service-quotas-exporter/templates/servicemonitor.yaml @@ -0,0 +1,47 @@ +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + {{- include "aws-service-quotas-exporter.labels" . | nindent 4 }} + {{- if .Values.serviceMonitor.labels }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} + name: {{ template "aws-service-quotas-exporter.fullname" . }} +{{- if .Values.serviceMonitor.namespace }} + namespace: {{ .Values.serviceMonitor.namespace }} +{{- end }} +spec: + endpoints: + - targetPort: http +{{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} +{{- end }} +{{- if .Values.serviceMonitor.path }} + path: {{ .Values.serviceMonitor.path }} +{{- end }} +{{- if .Values.serviceMonitor.timeout }} + scrapeTimeout: {{ .Values.serviceMonitor.timeout }} +{{- end }} +{{- if .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ toYaml .Values.serviceMonitor.metricRelabelings | indent 4 }} +{{- end }} +{{- if .Values.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.serviceMonitor.relabelings | indent 4 }} +{{- end }} + jobLabel: {{ template "aws-service-quotas-exporter.fullname" . }} + namespaceSelector: + matchNames: + - {{ .Values.serviceMonitor.namespace | default .Release.Namespace }} + selector: + matchLabels: + {{- include "aws-service-quotas-exporter.selectorLabels" . | nindent 6 }} +{{- if .Values.serviceMonitor.targetLabels }} + targetLabels: +{{- range .Values.serviceMonitor.targetLabels }} + - {{ . }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/aws-service-quotas-exporter/values.yaml b/charts/aws-service-quotas-exporter/values.yaml new file mode 100644 index 0000000..6775122 --- /dev/null +++ b/charts/aws-service-quotas-exporter/values.yaml @@ -0,0 +1,116 @@ +# Default values for aws-service-quotas-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/lablabs/aws-service-quotas-exporter + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + +service: + type: ClusterIP + port: 8080 + + +env: +# - name: "AWS_REGION" +# value: "eu-central-1" + +resources: + # This was measured for 5 quotas and 5 metrics script + requests: + cpu: 400m + memory: 200Mi + # limits: + # cpu: 100m + # memory: 128Mi + + +livenessProbe: + httpGet: + path: /liveness + port: http +readinessProbe: + httpGet: + path: /readiness + port: http + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +serviceMonitor: + # When set true then use a ServiceMonitor to configure scraping + enabled: false + # Set the namespace the ServiceMonitor should be deployed, if empty namespace will be .Release.Namespace + namespace: "" + # Service monitor labels + labels: {} + # Set how frequently Prometheus should scrape + interval: 30s + # Set path to metrics endpoint + path: /metrics + # Set timeout for scrape + timeout: 10s + # Set relabel_configs as per https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + # Set of labels to transfer on the Kubernetes Service onto the target. + targetLabels: [] + metricRelabelings: [] + +exporter: + address: "0.0.0.0:8080" + log: + level: "DEBUG" + format: "json" + config: {} +# scrape: +# quotas: [] +# metrics: [] diff --git a/cmd/exporter/main.go b/cmd/exporter/main.go new file mode 100644 index 0000000..fadce8d --- /dev/null +++ b/cmd/exporter/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "github.com/lablabs/aws-service-quotas-exporter/internal/app" + "github.com/lablabs/aws-service-quotas-exporter/pkg/flags" + log "github.com/lablabs/aws-service-quotas-exporter/pkg/log" + "github.com/lablabs/aws-service-quotas-exporter/pkg/service" + "os" +) + +func main() { + cfg := app.Config{} + flags.ParseOrFail(&cfg, os.Args) + logger := log.NewLoggerOrFail(cfg.Log.Format, cfg.Log.Level) + app, err := app.NewApplication(logger, cfg) + if err != nil { + logger.Fatal(err) + } + ctx := service.SignContext() + if err := app.Run(ctx); err != nil { + logger.Fatal(err) + } +} diff --git a/config/example.yaml b/config/example.yaml new file mode 100644 index 0000000..2663af4 --- /dev/null +++ b/config/example.yaml @@ -0,0 +1,10 @@ +scrape: + interval: "60s" + timeout: "5s" +quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" +metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2adff70 --- /dev/null +++ b/go.mod @@ -0,0 +1,42 @@ +module github.com/lablabs/aws-service-quotas-exporter + +go 1.22.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.26.0 + github.com/aws/aws-sdk-go-v2/config v1.27.4 + github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.1 + github.com/itchyny/gojq v0.12.14 + github.com/jessevdk/go-flags v1.5.0 + github.com/prometheus/client_golang v1.18.0 + github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.4 + golang.org/x/sync v0.6.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.17.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 // indirect + github.com/aws/smithy-go v1.20.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/itchyny/timefmt-go v0.1.5 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + golang.org/x/sys v0.15.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9267523 --- /dev/null +++ b/go.sum @@ -0,0 +1,86 @@ +github.com/aws/aws-sdk-go-v2 v1.26.0 h1:/Ce4OCiM3EkpW7Y+xUnfAFpchU78K7/Ug01sZni9PgA= +github.com/aws/aws-sdk-go-v2 v1.26.0/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= +github.com/aws/aws-sdk-go-v2/config v1.27.4 h1:AhfWb5ZwimdsYTgP7Od8E9L1u4sKmDW2ZVeLcf2O42M= +github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.4 h1:h5Vztbd8qLppiPwX+y0Q6WiwMZgpd9keKe2EAENgAuI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2 h1:AK0J8iYBFeUk2Ax7O8YpLtFsfhdOByh2QIkHmigpRYk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4/go.mod h1:WjpDrhWisWOIoS9n3nk67A3Ll1vfULJ9Kq6h29HTD48= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2 h1:5ffmXjPtwRExp1zc7gENLgCPyHFbhEPwVTkTiH9niSk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.1 h1:CCd+AWX79LVGzTXkgW9fBaPRFiC+J67zGuMcsER8Q1g= +github.com/aws/aws-sdk-go-v2/service/servicequotas v1.21.1/go.mod h1:+M0h5pY1hwymLXfxTAiAB0D87KxkvGrqmDz0gQbrm4A= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.1 h1:utEGkfdQ4L6YW/ietH7111ZYglLJvS+sLriHJ1NBJEQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1 h1:9/GylMS45hGGFCcMrUZDVayQE1jYSIN6da9jo7RAYIw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.1 h1:3I2cBEYgKhrWlwyZgfpSO2BpaMY1LHPqXYk/QGlu2ew= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA= +github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= +github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc= +github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= +github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/application.go b/internal/app/application.go new file mode 100644 index 0000000..23ed600 --- /dev/null +++ b/internal/app/application.go @@ -0,0 +1,90 @@ +package app + +import ( + "context" + "fmt" + "github.com/lablabs/aws-service-quotas-exporter/internal/exporter" + "github.com/lablabs/aws-service-quotas-exporter/internal/http" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/quotas" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/lablabs/aws-service-quotas-exporter/pkg/quota" + "github.com/lablabs/aws-service-quotas-exporter/pkg/service" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" +) + +const ( + PrometheusNamespace = "aws_quota_exporter" +) + +func NewApplication(log *logrus.Logger, cfg Config) (*Application, error) { + mng, err := service.NewManager() + if err != nil { + return nil, err + } + scCfg, err := scrape.LoadAndValidateConfig(cfg.Config) + if err != nil { + return nil, fmt.Errorf("unable to configure application: %w", err) + } + + registry := prometheus.NewRegistry() + cls := make([]exporter.Collector, 0) + client, err := quota.NewClient(log) + if err != nil { + return nil, err + } + qcl, err := quotas.NewCollector(log, scCfg.Quotas, PrometheusNamespace, client) + if err != nil { + return nil, err + } + cls = append(cls, qcl) + + scl, err := script.NewCollector(log, scCfg.Metrics, PrometheusNamespace) + if err != nil { + return nil, err + } + cls = append(cls, scl) + + exp, err := exporter.NewExporter(log, cls, registry, exporterOptions(scCfg)...) + if err != nil { + return nil, err + } + mng.Add(exp) + + http, err := http.NewHTTP(log, cfg.Address, registry) + if err != nil { + return nil, err + } + mng.Add(http) + a := Application{ + log: log, + cfg: cfg, + mng: mng, + } + return &a, nil +} + +type Application struct { + log *logrus.Logger + cfg Config + mng *service.Manager +} + +func (a *Application) Run(ctx context.Context) error { + a.log.Infof("exporter is starting") + err := a.mng.StartAndWait(ctx) + if err != nil { + return err + } + <-ctx.Done() + a.log.Infof("exporter exit OK") + return nil +} + +func exporterOptions(cfg *scrape.Config) []exporter.Option { + return []exporter.Option{ + exporter.WithInterval(cfg.Scrape.Interval), + exporter.WithTimeout(cfg.Scrape.Timeout), + } +} diff --git a/internal/app/config.go b/internal/app/config.go new file mode 100644 index 0000000..72f5581 --- /dev/null +++ b/internal/app/config.go @@ -0,0 +1,12 @@ +package app + +type Log struct { + Level string `long:"level" description:"Log level" choice:"DEBUG" choice:"INFO" default:"DEBUG"` + Format string `long:"format" description:"Format of message logs" choice:"json" choice:"text" default:"json"` +} + +type Config struct { + Address string `long:"address" description:"Http address" default:"0.0.0.0:8080"` + Log Log `group:"log" namespace:"log"` + Config string `long:"config" required:"true" description:"Path to metric scrape config file"` +} diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go new file mode 100644 index 0000000..0590096 --- /dev/null +++ b/internal/exporter/exporter.go @@ -0,0 +1,98 @@ +package exporter + +import ( + "context" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "time" +) + +type Collector interface { + Register(ctx context.Context, r *prometheus.Registry) error + Collect(ctx context.Context) error +} + +const ( + defaultScrapeInterval = time.Second * 60 + defaultCollectorTimeout = time.Second * 5 +) + +func NewExporter(log *logrus.Logger, cls []Collector, r *prometheus.Registry, options ...Option) (*Exporter, error) { + cfg := config{ + interval: defaultScrapeInterval, + timeout: defaultCollectorTimeout, + } + for _, o := range options { + if err := o(&cfg); err != nil { + return nil, fmt.Errorf("unable to configure exporter: %w", err) + } + } + e := Exporter{ + log: log, + cfg: &cfg, + cls: cls, + r: r, + } + return &e, nil +} + +type Exporter struct { + log *logrus.Logger + cfg *config + cls []Collector + r *prometheus.Registry +} + +func (e *Exporter) Run(ctx context.Context) error { + err := e.register(ctx) + if err != nil { + return err + } + ticker := time.NewTicker(e.cfg.interval) + e.log.Debugf("scrape metrics every: %v", e.cfg.interval) + defer ticker.Stop() +end: + for { + select { + case <-ctx.Done(): + break end + case <-ticker.C: + err := e.scrape(ctx) + if err != nil { + e.log.Errorf("unable to scrape metric: %v", err) + } + } + } + return nil +} + +func (e *Exporter) scrape(ctx context.Context) error { + e.log.Debugf("scrape metrics") + ctx, cancel := context.WithTimeout(ctx, e.cfg.timeout) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + for _, c := range e.cls { + cl := c + g.Go(func() error { + return cl.Collect(ctx) + }) + } + err := g.Wait() + return err +} + +func (e *Exporter) register(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, e.cfg.timeout) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + for _, c := range e.cls { + cl := c + g.Go(func() error { + return cl.Register(ctx, e.r) + }) + } + err := g.Wait() + return err +} diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go new file mode 100644 index 0000000..4b5c44f --- /dev/null +++ b/internal/exporter/exporter_test.go @@ -0,0 +1,76 @@ +package exporter_test + +import ( + "context" + "errors" + "github.com/lablabs/aws-service-quotas-exporter/internal/exporter" + "github.com/lablabs/aws-service-quotas-exporter/test" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestExporter_Run(t *testing.T) { + type fields struct { + log *logrus.Logger + cls []exporter.Collector + ops []exporter.Option + ctx func() (context.Context, func()) + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "Exporter OK", + fields: fields{ + log: test.DefaultLogger(), + cls: []exporter.Collector{&testCollector{}}, + ops: []exporter.Option{}, + ctx: func() (context.Context, func()) { + return context.WithTimeout(context.Background(), time.Second*1) + }, + }, + wantErr: false, + }, + { + name: "Exporter timeout", + fields: fields{ + log: test.DefaultLogger(), + cls: []exporter.Collector{&testCollector{err: errors.New("timeout")}}, + ops: []exporter.Option{}, + ctx: func() (context.Context, func()) { + return context.WithTimeout(context.Background(), time.Second*1) + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e, err := exporter.NewExporter(tt.fields.log, tt.fields.cls, prometheus.NewRegistry()) + assert.NoError(t, err) + ctx, cancel := tt.fields.ctx() + defer cancel() + if err := e.Run(ctx); (err != nil) != tt.wantErr { + t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + } + + }) + } +} + +type testCollector struct { + err error +} + +func (t *testCollector) Register(_ context.Context, _ *prometheus.Registry) error { + return t.err +} + +func (t *testCollector) Collect(_ context.Context) error { + return t.err +} diff --git a/internal/exporter/options.go b/internal/exporter/options.go new file mode 100644 index 0000000..8705cfe --- /dev/null +++ b/internal/exporter/options.go @@ -0,0 +1,28 @@ +package exporter + +import "time" + +type config struct { + interval time.Duration + timeout time.Duration +} + +type Option func(c *config) error + +func WithInterval(i time.Duration) Option { + return func(c *config) error { + if i.Nanoseconds() != 0 { + c.interval = i + } + return nil + } +} + +func WithTimeout(t time.Duration) Option { + return func(c *config) error { + if t.Nanoseconds() != 0 { + c.timeout = t + } + return nil + } +} diff --git a/internal/http/http.go b/internal/http/http.go new file mode 100644 index 0000000..9bf75ab --- /dev/null +++ b/internal/http/http.go @@ -0,0 +1,78 @@ +package http + +import ( + "context" + "errors" + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "net" + "net/http" +) + +func NewHTTP(log *logrus.Logger, address string, registry *prometheus.Registry) (*HTTP, error) { + ln, err := net.Listen("tcp", address) + if err != nil { + return nil, err + } + handler := http.NewServeMux() + s := http.Server{ + Handler: handler, + } + h := HTTP{ + log: log, + ln: ln, + s: &s, + } + RegisterMetricEndpoint(handler, registry) + RegisterStatusEndpoint(handler, &h) + return &h, nil +} + +type HTTP struct { + log *logrus.Logger + ln net.Listener + s *http.Server +} + +func (h *HTTP) Run(ctx context.Context) error { + h.log.Infof("start http endpoint: %s", h.ln.Addr()) + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + err := h.s.Serve(h.ln) + if !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("unable to serve http enpoint: %w", err) + } + return nil + }) + <-ctx.Done() + h.log.Debugf("http endpoint exiting...") + err := h.s.Shutdown(ctx) + if err != nil { + return err + } + err = g.Wait() + if err != nil { + return err + } + h.log.Infof("http endpoint exit OK") + return nil +} + +func (h *HTTP) Status(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func RegisterMetricEndpoint(mux *http.ServeMux, registry *prometheus.Registry) { + registry.MustRegister(collectors.NewGoCollector()) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{Registry: registry})) +} + +func RegisterStatusEndpoint(mux *http.ServeMux, h *HTTP) { + mux.HandleFunc("/liveness", h.Status) + mux.HandleFunc("/readiness", h.Status) +} diff --git a/internal/http/http_test.go b/internal/http/http_test.go new file mode 100644 index 0000000..367c283 --- /dev/null +++ b/internal/http/http_test.go @@ -0,0 +1,43 @@ +package http_test + +import ( + "context" + httpApi "github.com/lablabs/aws-service-quotas-exporter/internal/http" + "github.com/lablabs/aws-service-quotas-exporter/test" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "net/http" + "sync" + "testing" +) + +const ( + address = "0.0.0.0:8080" +) + +func TestNewHttp(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + closeHTTP := StartHTTP(ctx, t) + defer func() { + cancel() + closeHTTP() + }() + resp, err := http.Get("http://" + address + "/metrics") + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +func StartHTTP(ctx context.Context, t *testing.T) func() { + http, err := httpApi.NewHTTP(test.DefaultLogger(), address, prometheus.NewRegistry()) + assert.NoError(t, err) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := http.Run(ctx) + assert.NoError(t, err) + }() + return func() { + wg.Wait() + } +} diff --git a/internal/scrape/config.go b/internal/scrape/config.go new file mode 100644 index 0000000..0bc1b0f --- /dev/null +++ b/internal/scrape/config.go @@ -0,0 +1,61 @@ +package scrape + +import ( + "fmt" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/quotas" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/lablabs/aws-service-quotas-exporter/pkg/config" + "time" +) + +type Scrape struct { + Interval time.Duration `json:"interval,omitempty" yaml:"interval,omitempty"` + Timeout time.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` +} + +func (s *Scrape) Validate() error { + if s.Interval != 0 && s.Interval < time.Minute { + return fmt.Errorf("scrape.interval is not valid. Minimal value is 60s") + } + if s.Timeout != 0 && s.Timeout < (time.Second*5) { + return fmt.Errorf("scrape.timeout is not valid. Minimal value is 5s") + } + return nil +} + +type Config struct { + Scrape Scrape `json:"scrape,omitempty" yaml:"scrape,omitempty"` + Quotas []quotas.Config `json:"quotas,omitempty" yaml:"quotas,omitempty"` + Metrics []script.Config `json:"metrics,omitempty" yaml:"metrics,omitempty"` +} + +func (c *Config) Validate() error { + for _, q := range c.Quotas { + if err := q.Validate(); err != nil { + return err + } + } + for _, m := range c.Metrics { + if err := m.Validate(); err != nil { + return err + } + } + return c.Scrape.Validate() +} + +func LoadAndValidateConfig(path string) (*Config, error) { + cfg := Config{} + err := config.ParseYamlFromFile(path, &cfg) + if err != nil { + return nil, fmt.Errorf("unable to parse metric scrape from file: %w", err) + } + err = cfg.Validate() + if err != nil { + return nil, err + } + return &cfg, nil +} + +type Validator interface { + Validate() error +} diff --git a/internal/scrape/config_test.go b/internal/scrape/config_test.go new file mode 100644 index 0000000..9ededa3 --- /dev/null +++ b/internal/scrape/config_test.go @@ -0,0 +1,51 @@ +package scrape + +import ( + "testing" + "time" +) + +func TestScrape_Validate(t *testing.T) { + type fields struct { + Interval time.Duration + Timeout time.Duration + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "Scrape config not set", + fields: fields{}, + wantErr: false, + }, + { + name: "Scrape config OK", + fields: fields{ + Interval: time.Minute + time.Second*5, + Timeout: time.Second * 6, + }, + wantErr: false, + }, + { + name: "Scrape config error", + fields: fields{ + Interval: time.Minute - time.Second, + Timeout: time.Second * 4, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Scrape{ + Interval: tt.fields.Interval, + Timeout: tt.fields.Timeout, + } + if err := s.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/scrape/quotas/collector.go b/internal/scrape/quotas/collector.go new file mode 100644 index 0000000..eba575d --- /dev/null +++ b/internal/scrape/quotas/collector.go @@ -0,0 +1,115 @@ +package quotas + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/servicequotas/types" + "github.com/lablabs/aws-service-quotas-exporter/pkg/quota" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "sync" +) + +const ( + name = "name" + code = "code" + serviceCode = "service_code" +) + +type Quota interface { + GetQuota(ctx context.Context, serviceCode string, quotaCode string, options ...quota.Option) (*types.ServiceQuota, error) +} + +func NewCollector(log *logrus.Logger, cfg []Config, ns string, qcl Quota) (*Collector, error) { + cl := Collector{ + log: log, + qcl: qcl, + cfg: cfg, + ns: ns, + tasks: make([]task, 0), + } + return &cl, nil +} + +type Collector struct { + log *logrus.Logger + qcl Quota + once sync.Once + err error + ns string + cfg []Config + tasks []task + tasksLock sync.Mutex +} + +func (c *Collector) Register(ctx context.Context, r *prometheus.Registry) error { + c.once.Do(func() { + c.log.Debugf("start registering quota metrics") + gvq := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: c.ns, + Name: "quota", + Help: "AWS service quota", + }, []string{name, code, serviceCode}) + r.MustRegister(gvq) + g, ctx := errgroup.WithContext(ctx) + for _, cf := range c.cfg { + qc := cf + g.Go(func() error { + res, err := c.qcl.GetQuota(ctx, qc.ServiceCode, qc.QuotaCode, quota.WithRegion(qc.Region)) + if err != nil { + return err + } + t := task{ + m: gvq, + cfg: qc, + } + c.addTask(t) + setMetric(t.m, res) + return nil + }) + } + c.err = g.Wait() + }) + return c.err +} + +func (c *Collector) Collect(ctx context.Context) error { + g, ctx := errgroup.WithContext(ctx) + for _, t := range c.tasks { + ts := t + g.Go(ts.run(ctx, c.qcl)) + } + err := g.Wait() + return err +} + +func (c *Collector) addTask(t task) { + c.tasksLock.Lock() + defer c.tasksLock.Unlock() + c.tasks = append(c.tasks, t) +} + +type task struct { + m *prometheus.GaugeVec + cfg Config +} + +func (t task) run(ctx context.Context, c Quota) func() error { + return func() error { + res, err := c.GetQuota(ctx, t.cfg.ServiceCode, t.cfg.QuotaCode, quota.WithRegion(t.cfg.Region)) + if err != nil { + return err + } + setMetric(t.m, res) + return nil + } +} + +func setMetric(gc *prometheus.GaugeVec, q *types.ServiceQuota) { + gc.With(prometheus.Labels{ + name: aws.ToString(q.QuotaName), + code: aws.ToString(q.QuotaCode), + serviceCode: aws.ToString(q.ServiceCode), + }).Set(aws.ToFloat64(q.Value)) +} diff --git a/internal/scrape/quotas/config.go b/internal/scrape/quotas/config.go new file mode 100644 index 0000000..7a3a141 --- /dev/null +++ b/internal/scrape/quotas/config.go @@ -0,0 +1,21 @@ +package quotas + +import ( + "fmt" +) + +type Config struct { + ServiceCode string `json:"serviceCode,omitempty" yaml:"serviceCode,omitempty"` + QuotaCode string `json:"quotaCode,omitempty" yaml:"quotaCode,omitempty"` + Region string `json:"region,omitempty" yaml:"region,omitempty"` +} + +func (c Config) Validate() error { + if c.ServiceCode == "" { + return fmt.Errorf("serviceCode must not be empty") + } + if c.QuotaCode == "" { + return fmt.Errorf("quotaCode must not be empty") + } + return nil +} diff --git a/internal/scrape/script/collector.go b/internal/scrape/script/collector.go new file mode 100644 index 0000000..6be2b8a --- /dev/null +++ b/internal/scrape/script/collector.go @@ -0,0 +1,99 @@ +package script + +import ( + "context" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "sync" +) + +func NewCollector(log *logrus.Logger, cfg []Config, ns string) (*Collector, error) { + cl := Collector{ + log: log, + ns: ns, + cfg: cfg, + tasks: make([]task, 0), + } + return &cl, nil +} + +type Collector struct { + log *logrus.Logger + once sync.Once + err error + ns string + cfg []Config + tasks []task + tasksLock sync.Mutex +} + +func (c *Collector) Register(ctx context.Context, r *prometheus.Registry) error { + c.once.Do(func() { + c.log.Debugf("start registering script metrics") + g, ctx := errgroup.WithContext(ctx) + for _, cf := range c.cfg { + config := cf + g.Go(func() error { + data, err := Run(ctx, config) + if err != nil { + c.log.Errorf("unable to run command: %s, %v", config.Script, err) + return err + } + if len(data) > 0 { + lbs := data[0].LabelNames() + m := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: c.ns, + Name: config.Name, + Help: config.Help, + }, lbs) + r.MustRegister(m) + t := task{ + m: m, + cfg: config, + } + c.addTask(t) + for _, d := range data { + t.m.With(d.Labels).Set(d.Value) + } + } + return nil + }) + } + c.err = g.Wait() + }) + return c.err +} + +func (c *Collector) Collect(ctx context.Context) error { + g, ctx := errgroup.WithContext(ctx) + for _, t := range c.tasks { + g.Go(t.run(ctx)) + } + err := g.Wait() + return err +} + +func (c *Collector) addTask(t task) { + c.tasksLock.Lock() + defer c.tasksLock.Unlock() + c.tasks = append(c.tasks, t) +} + +type task struct { + m *prometheus.GaugeVec + cfg Config +} + +func (t task) run(ctx context.Context) func() error { + return func() error { + data, err := Run(ctx, t.cfg) + if err != nil { + return err + } + for _, d := range data { + t.m.With(d.Labels).Set(d.Value) + } + return nil + } +} diff --git a/internal/scrape/script/collector_test.go b/internal/scrape/script/collector_test.go new file mode 100644 index 0000000..efe4b10 --- /dev/null +++ b/internal/scrape/script/collector_test.go @@ -0,0 +1,40 @@ +package script_test + +import ( + "context" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/lablabs/aws-service-quotas-exporter/test" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func TestNewCollector(t *testing.T) { + + cfg := []script.Config{ + { + Name: "metric_a", + Help: "metric b", + Script: "echo \"name=n,cluster=1,1\"", + }, + { + Name: "metric_b", + Help: "metric b", + Script: "echo \"name=n,cluster=2,1\"", + }, + } + + cl, err := script.NewCollector(test.DefaultLogger(), cfg, "ns") + assert.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + r := prometheus.NewRegistry() + err = cl.Register(ctx, r) + assert.NoError(t, err) + err = cl.Register(ctx, r) + assert.NoError(t, err) + err = cl.Collect(ctx) + assert.NoError(t, err) + +} diff --git a/internal/scrape/script/config.go b/internal/scrape/script/config.go new file mode 100644 index 0000000..9d60976 --- /dev/null +++ b/internal/scrape/script/config.go @@ -0,0 +1,53 @@ +package script + +import "fmt" + +type Config struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Help string `json:"help,omitempty" yaml:"help,omitempty"` + Script string `json:"script,omitempty" yaml:"script"` + Envs []Env `json:"env,omitempty" yaml:"envs,omitempty"` +} + +func (c *Config) Validate() error { + if c.Name == "" { + return fmt.Errorf("name of metric is required") + } + if c.Help == "" { + return fmt.Errorf("help of metric is required") + } + if c.Script == "" { + return fmt.Errorf("script is required") + } + if c.Envs != nil { + for _, e := range c.Envs { + if err := e.Validate(); err != nil { + return err + } + } + } + return nil +} + +type Env struct { + Name string `json:"name,omitempty" yaml:"name"` + Value string `json:"value,omitempty" yaml:"value"` +} + +func (e *Env) Validate() error { + if e.Name == "" { + return fmt.Errorf("name for env is required") + } + if e.Value == "" { + return fmt.Errorf("value for env is required") + } + return nil +} + +func (c *Config) FormatEnvs() []string { + evs := make([]string, 0) + for _, e := range c.Envs { + evs = append(evs, fmt.Sprintf("%s=%s", e.Name, e.Value)) + } + return evs +} diff --git a/internal/scrape/script/metric.go b/internal/scrape/script/metric.go new file mode 100644 index 0000000..435b760 --- /dev/null +++ b/internal/scrape/script/metric.go @@ -0,0 +1,68 @@ +package script + +import ( + "bufio" + "fmt" + "io" + "regexp" + "strconv" + "strings" +) + +const metricRegex = `,` + +func ParseStdout(r io.Reader) ([]Data, error) { + out := make([]Data, 0) + pr := NewParser() + scanner := bufio.NewScanner(r) + for scanner.Scan() { + mt, err := pr.ParseMetric(scanner.Text()) + if err != nil { + return nil, fmt.Errorf("unable to parse metric: %w", err) + } + out = append(out, mt) + } + return out, nil +} + +func NewParser() Parser { + re := regexp.MustCompile(metricRegex) + p := Parser{re: re} + return p +} + +type Parser struct { + re *regexp.Regexp +} + +func (p Parser) ParseMetric(l string) (Data, error) { + lbs := make(map[string]string) + var v float64 + mtcs := p.re.Split(l, -1) + for i, mt := range mtcs { + if i == len(mtcs)-1 { + vl, err := strconv.ParseFloat(mt, 64) + if err != nil { + return Data{}, fmt.Errorf("invalid value of metric, %s is not float64, %v", mt, err) + } + v = vl + continue + } + spl := strings.Index(mt, "=") + lbs[mt[:spl]] = cleanBracket(mt[spl+1:]) + } + return Data{ + Value: v, + Labels: lbs, + }, nil +} + +func cleanBracket(input string) string { + if strings.HasPrefix(input, `"`) || strings.HasPrefix(input, "'") { + input = input[1:] + } + if strings.HasSuffix(input, `"`) || strings.HasSuffix(input, "'") { + input = input[:len(input)-1] + } + return input +} diff --git a/internal/scrape/script/metric_test.go b/internal/scrape/script/metric_test.go new file mode 100644 index 0000000..a45fc93 --- /dev/null +++ b/internal/scrape/script/metric_test.go @@ -0,0 +1,61 @@ +package script_test + +import ( + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "github.com/stretchr/testify/assert" + "reflect" + "testing" +) + +func TestParser_ParseMetric(t *testing.T) { + tests := []struct { + name string + metric string + want script.Data + wantErr bool + }{ + { + name: "Parse Metric OK", + metric: "a=1,b=2,c='3',d='t e a',y=\"aa bbb ccc\",4", + want: script.Data{ + Value: 4, + Labels: map[string]string{ + "a": "1", + "b": "2", + "c": "3", + "d": "t e a", + "y": "aa bbb ccc", + }, + }, + wantErr: false, + }, + { + name: "Value not valid float64", + metric: "a=1,b=2,c='3',d='t e a',y=\"aa bbb ccc\",not_valid", + want: script.Data{ + Value: 4, + Labels: map[string]string{ + "a": "1", + "b": "2", + "c": "3", + "d": "t e a", + "y": "aa bbb ccc", + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := script.NewParser() + got, err := p.ParseMetric(tt.metric) + if tt.wantErr { + assert.Errorf(t, err, "ParseMetric() expect error") + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseMetric() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/scrape/script/script.go b/internal/scrape/script/script.go new file mode 100644 index 0000000..ef98ace --- /dev/null +++ b/internal/scrape/script/script.go @@ -0,0 +1,63 @@ +package script + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" +) + +type Data struct { + Value float64 + Labels map[string]string +} + +func (d Data) LabelNames() []string { + if d.Labels == nil { + return []string{} + } + r := make([]string, 0) + for k := range d.Labels { + r = append(r, k) + } + return r +} + +func Run(ctx context.Context, cfg Config) ([]Data, error) { + cmd := exec.CommandContext(ctx, "bash", "-c", cfg.Script) + envs := make([]string, 0) + envs = append(envs, os.Environ()...) + envs = append(envs, cfg.FormatEnvs()...) + cmd.Env = envs + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + stdout := &bytes.Buffer{} + cmd.Stdout = stdout + err = cmd.Run() + if err != nil { + errString, err := errorString(stderr) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("script error: %w, std err: %s", err, errString) + } + data, err := ParseStdout(stdout) + if err != nil { + return nil, fmt.Errorf("unable to parse response from command: %v", cfg.Script) + } + return data, nil +} + +func errorString(r io.Reader) (string, error) { + b := strings.Builder{} + _, err := io.Copy(&b, r) + if err != nil { + return "", err + } + return b.String(), nil +} diff --git a/internal/scrape/script/script_test.go b/internal/scrape/script/script_test.go new file mode 100644 index 0000000..12bab15 --- /dev/null +++ b/internal/scrape/script/script_test.go @@ -0,0 +1,61 @@ +package script_test + +import ( + "context" + "github.com/lablabs/aws-service-quotas-exporter/internal/scrape/script" + "reflect" + "testing" +) + +func TestRun(t *testing.T) { + tests := []struct { + name string + args script.Config + want []script.Data + wantErr bool + }{ + { + name: "Parsing command OK", + args: script.Config{ + Name: "metric_1", + Help: "", + Script: "echo \"region=eu-central-1,cluster=eks-dev-1,type=dev,2\"", + }, + want: []script.Data{ + { + Value: 2, + Labels: map[string]string{ + "region": "eu-central-1", + "cluster": "eks-dev-1", + "type": "dev", + }, + }, + }, + wantErr: false, + }, + { + name: "Invalid command", + args: script.Config{ + Name: "metric_1", + Help: "", + Script: "invalid command", + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancelCtx := context.WithCancel(context.Background()) + defer cancelCtx() + got, err := script.Run(ctx, tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Run() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/config/yaml.go b/pkg/config/yaml.go new file mode 100644 index 0000000..e50586b --- /dev/null +++ b/pkg/config/yaml.go @@ -0,0 +1,24 @@ +package config + +import ( + "fmt" + "gopkg.in/yaml.v3" + "os" +) + +// ParseYaml bytes to scrape struct +func ParseYaml(data []byte, c interface{}) error { + err := yaml.Unmarshal(data, c) + if err != nil { + return fmt.Errorf("invalid yaml config: %w", err) + } + return nil +} + +func ParseYamlFromFile(path string, c interface{}) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("unable to read file: %w", err) + } + return ParseYaml(data, c) +} diff --git a/pkg/flags/parse.go b/pkg/flags/parse.go new file mode 100644 index 0000000..8619bbc --- /dev/null +++ b/pkg/flags/parse.go @@ -0,0 +1,21 @@ +package flags + +import ( + "github.com/jessevdk/go-flags" + "os" +) + +func ParseFlags(config interface{}, args []string) error { + _, err := flags.NewParser(config, flags.Default).ParseArgs(args) + if err != nil { + return err + } + return nil +} + +func ParseOrFail(config interface{}, args []string) { + err := ParseFlags(config, args) + if err != nil { + os.Exit(1) + } +} diff --git a/pkg/jqdata/parse.go b/pkg/jqdata/parse.go new file mode 100644 index 0000000..82bdb1c --- /dev/null +++ b/pkg/jqdata/parse.go @@ -0,0 +1,40 @@ +package jqdata + +import ( + "context" + "encoding/json" + "fmt" + "github.com/itchyny/gojq" +) + +type JSONData struct { + d map[string]any +} + +func ParseRawJSON(data []byte) (JSONData, error) { + d := make(map[string]any) + err := json.Unmarshal(data, &d) + if err != nil { + return JSONData{}, fmt.Errorf("unable parse JSON: %w", err) + } + return JSONData{d: d}, nil +} + +func (j JSONData) Query(ctx context.Context, q string) (any, error) { + qr, err := gojq.Parse(q) + if err != nil { + return nil, err + } + it := qr.RunWithContext(ctx, j.d) + for { + v, ok := it.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return nil, err + } + return v, nil + } + return nil, fmt.Errorf("empty data for query: %v", q) +} diff --git a/pkg/jqdata/parse_test.go b/pkg/jqdata/parse_test.go new file mode 100644 index 0000000..4906671 --- /dev/null +++ b/pkg/jqdata/parse_test.go @@ -0,0 +1,59 @@ +package jqdata + +import ( + "context" + "encoding/json" + "github.com/stretchr/testify/assert" + "reflect" + "testing" + "time" +) + +func TestJsonData_Query(t *testing.T) { + data := map[string]any{ + "v": 1, + "s": "s", + "l": []string{"1", "2", "3"}, + } + j, err := json.Marshal(data) + assert.NoError(t, err) + type args struct { + q string + } + tests := []struct { + name string + j func() JSONData + args args + want any + wantErr bool + }{ + { + name: "Query OK", + j: func() JSONData { + jd, err := ParseRawJSON(j) + assert.NoError(t, err) + return jd + }, + args: args{ + q: ".l | length", + }, + want: 3, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Hour) + defer cancel() + jd := tt.j() + got, err := jd.Query(ctx, tt.args.q) + if (err != nil) != tt.wantErr { + t.Errorf("Query() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Query() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..a72c0b4 --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,39 @@ +package log + +import ( + log "github.com/sirupsen/logrus" + "os" +) + +func DefaultLoggerOrFail() *log.Logger { + logger := NewLoggerOrFail("json", "DEBUG") + return logger +} + +func NewLoggerOrFail(format string, level string) *log.Logger { + log, err := NewLogger(format, level) + if err != nil { + panic(err) + } + return log +} + +func NewLogger(format string, logLevel string) (*log.Logger, error) { + logger := log.New() + level, err := log.ParseLevel(logLevel) + if err != nil { + return nil, err + } + switch format { + case "json": + logger.SetFormatter(&log.JSONFormatter{}) + case "text": + logger.SetFormatter(&log.TextFormatter{ + DisableColors: true, + FullTimestamp: true, + }) + } + logger.SetOutput(os.Stdout) + logger.SetLevel(level) + return logger, nil +} diff --git a/pkg/quota/client.go b/pkg/quota/client.go new file mode 100644 index 0000000..65e9287 --- /dev/null +++ b/pkg/quota/client.go @@ -0,0 +1,61 @@ +package quota + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/servicequotas" + "github.com/aws/aws-sdk-go-v2/service/servicequotas/types" + "github.com/sirupsen/logrus" +) + +func NewClient(log *logrus.Logger) (*Client, error) { + + cfg, err := awsConfig.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, err + } + squ := servicequotas.NewFromConfig(cfg) + c := Client{ + log: log, + squ: squ, + } + return &c, nil +} + +type Client struct { + log *logrus.Logger + squ *servicequotas.Client +} + +func (c *Client) GetQuota(ctx context.Context, serviceCode string, quotaCode string, options ...Option) (*types.ServiceQuota, error) { + res, err := c.squ.GetServiceQuota(ctx, &servicequotas.GetServiceQuotaInput{ + QuotaCode: aws.String(quotaCode), + ServiceCode: aws.String(serviceCode), + }, buildOptions(options...)) + if err != nil { + return nil, fmt.Errorf("unable to get quota with service: %s, code: %s, %w", serviceCode, quotaCode, err) + } + return res.Quota, nil +} + +func (c *Client) GetQuotas(ctx context.Context, serviceCode string, options ...Option) ([]types.ServiceQuota, error) { + qs := make([]types.ServiceQuota, 0) + var token *string + for { + res, err := c.squ.ListServiceQuotas(ctx, &servicequotas.ListServiceQuotasInput{ + ServiceCode: aws.String(serviceCode), + NextToken: token, + }, buildOptions(options...)) + if err != nil { + return nil, err + } + qs = append(qs, res.Quotas...) + if res.NextToken == nil { + break + } + token = res.NextToken + } + return qs, nil +} diff --git a/pkg/quota/options.go b/pkg/quota/options.go new file mode 100644 index 0000000..34844c9 --- /dev/null +++ b/pkg/quota/options.go @@ -0,0 +1,28 @@ +package quota + +import "github.com/aws/aws-sdk-go-v2/service/servicequotas" + +type Options struct { + *servicequotas.Options +} + +type Option func(c *Options) + +func WithRegion(region string) Option { + return func(c *Options) { + if region != "" { + c.Region = region + } + } +} + +func buildOptions(option ...Option) func(*servicequotas.Options) { + return func(awsSq *servicequotas.Options) { + op := Options{ + awsSq, + } + for _, o := range option { + o(&op) + } + } +} diff --git a/pkg/service/ctx.go b/pkg/service/ctx.go new file mode 100644 index 0000000..0532216 --- /dev/null +++ b/pkg/service/ctx.go @@ -0,0 +1,13 @@ +package service + +import ( + "context" + "os" + "os/signal" + "syscall" +) + +func SignContext() context.Context { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) + return ctx +} diff --git a/pkg/service/mng.go b/pkg/service/mng.go new file mode 100644 index 0000000..993d5e6 --- /dev/null +++ b/pkg/service/mng.go @@ -0,0 +1,43 @@ +package service + +import ( + "context" + "errors" + "golang.org/x/sync/errgroup" +) + +func NewManager() (*Manager, error) { + m := Manager{ + services: make([]Starter, 0), + } + return &m, nil +} + +type Starter interface { + // Run controller and block until finish + Run(ctx context.Context) error +} + +type Manager struct { + services []Starter +} + +func (m *Manager) Add(s Starter) { + m.services = append(m.services, s) +} + +func (m *Manager) StartAndWait(ctx context.Context) error { + group, ctx := errgroup.WithContext(ctx) + for _, s := range m.services { + service := s + group.Go(func() error { + err := service.Run(ctx) + if err != nil && !errors.Is(err, context.Canceled) { + return err + } + <-ctx.Done() + return err + }) + } + return group.Wait() +} diff --git a/test/config.go b/test/config.go new file mode 100644 index 0000000..ad3e4f9 --- /dev/null +++ b/test/config.go @@ -0,0 +1,19 @@ +package test + +import ( + _ "embed" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +//go:embed configs/route53.yaml +var configEip string + +func TmpConfigMetricFile(t *testing.T) string { + f, err := os.CreateTemp(os.TempDir(), "exporter") + assert.NoError(t, err) + _, err = f.WriteString(configEip) + assert.NoError(t, err) + return f.Name() +} diff --git a/test/configs/route53.yaml b/test/configs/route53.yaml new file mode 100644 index 0000000..6914499 --- /dev/null +++ b/test/configs/route53.yaml @@ -0,0 +1,7 @@ +quotas: + - serviceCode: "ec2" + quotaCode: "L-0263D0A3" +metrics: + - name: "route53_hosted_zone_records" + help: "Number of resource sets in hosted zone" + script: "aws route53 list-hosted-zones | jq -r \'.HostedZones[] | \"id=\\(.Id),name=\\(.Name),private=\\(.Config.PrivateZone),\\(.ResourceRecordSetCount)\"\'" diff --git a/test/log.go b/test/log.go new file mode 100644 index 0000000..361c9e6 --- /dev/null +++ b/test/log.go @@ -0,0 +1,16 @@ +package test + +import ( + logUtil "github.com/lablabs/aws-service-quotas-exporter/pkg/log" + log "github.com/sirupsen/logrus" +) + +const ( + debugLevel = "DEBUG" + format = "json " +) + +func DefaultLogger() *log.Logger { + logger := logUtil.NewLoggerOrFail(format, debugLevel) + return logger +}