Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(promtail-mixin): improve standalone usage #8534

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat(promtail-mixin): improve standalone usage #8534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c475091
feat(promtail-mixin): adjustable name & selectors
Zaunei Nov 17, 2023
7430939
chore(promtail-mixin): match loki-mixin file structure
Zaunei Nov 17, 2023
d163387
feat(makefile): compile promtail-mixin
Zaunei May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 34 additions & 12 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,32 +253,54 @@ clients/cmd/promtail/promtail-debug:
CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_DEBUG_GO_FLAGS) --tags=$(PROMTAIL_GO_TAGS) -o $@ ./$(@D)

#########
# Mixin #
# Loki Mixin #
#########

MIXIN_PATH := production/loki-mixin
MIXIN_OUT_PATH := production/loki-mixin-compiled
MIXIN_OUT_PATH_SSD := production/loki-mixin-compiled-ssd
MIXIN_PATH_LOKI := production/loki-mixin
MIXIN_OUT_LOKI_PATH := production/loki-mixin-compiled
MIXIN_OUT_LOKI_PATH_SSD := production/loki-mixin-compiled-ssd

loki-mixin: ## compile the loki mixin
ifeq ($(BUILD_IN_CONTAINER),true)
$(SUDO) docker run $(RM) $(TTY) -i \
-v $(shell pwd):/src/loki$(MOUNT_FLAGS) \
$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@;
else
@rm -rf $(MIXIN_OUT_PATH) && mkdir $(MIXIN_OUT_PATH)
@cd $(MIXIN_PATH) && jb install
@mixtool generate all --output-alerts $(MIXIN_OUT_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_PATH)/rules.yaml --directory $(MIXIN_OUT_PATH)/dashboards ${MIXIN_PATH}/mixin.libsonnet
@rm -rf $(MIXIN_OUT_LOKI_PATH) && mkdir $(MIXIN_OUT_LOKI_PATH)
@cd $(MIXIN_PATH_LOKI) && jb install
@mixtool generate all --output-alerts $(MIXIN_OUT_LOKI_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_LOKI_PATH)/rules.yaml --directory $(MIXIN_OUT_LOKI_PATH)/dashboards ${MIXIN_PATH_LOKI}/mixin.libsonnet

@rm -rf $(MIXIN_OUT_PATH_SSD) && mkdir $(MIXIN_OUT_PATH_SSD)
@cd $(MIXIN_PATH) && jb install
@mixtool generate all --output-alerts $(MIXIN_OUT_PATH_SSD)/alerts.yaml --output-rules $(MIXIN_OUT_PATH_SSD)/rules.yaml --directory $(MIXIN_OUT_PATH_SSD)/dashboards ${MIXIN_PATH}/mixin-ssd.libsonnet
@rm -rf $(MIXIN_OUT_LOKI_PATH_SSD) && mkdir $(MIXIN_OUT_LOKI_PATH_SSD)
@cd $(MIXIN_PATH_LOKI) && jb install
@mixtool generate all --output-alerts $(MIXIN_OUT_LOKI_PATH_SSD)/alerts.yaml --output-rules $(MIXIN_OUT_LOKI_PATH_SSD)/rules.yaml --directory $(MIXIN_OUT_LOKI_PATH_SSD)/dashboards ${MIXIN_PATH_LOKI}/mixin-ssd.libsonnet
endif

loki-mixin-check: loki-mixin ## check the loki mixin is up to date
@echo "Checking diff"
@git diff --exit-code -- $(MIXIN_OUT_PATH) || (echo "Please build mixin by running 'make loki-mixin'" && false)
@git diff --exit-code -- $(MIXIN_OUT_PATH_SSD) || (echo "Please build mixin by running 'make loki-mixin'" && false)
@git diff --exit-code -- $(MIXIN_OUT_LOKI_PATH) || (echo "Please build mixin by running 'make loki-mixin'" && false)
@git diff --exit-code -- $(MIXIN_OUT_LOKI_PATH_SSD) || (echo "Please build mixin by running 'make loki-mixin'" && false)

#########
# Promtail Mixin #
#########

MIXIN_PATH_PROMTAIL := production/promtail-mixin
MIXIN_OUT_PROMTAIL_PATH := production/promtail-mixin-compiled

promtail-mixin:
ifeq ($(BUILD_IN_CONTAINER),true)
$(SUDO) docker run $(RM) $(TTY) -i \
-v $(shell pwd):/src/loki$(MOUNT_FLAGS) \
$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@;
else
@rm -rf $(MIXIN_OUT_PROMTAIL_PATH) && mkdir $(MIXIN_OUT_PROMTAIL_PATH)
@cd $(MIXIN_PATH_PROMTAIL) && jb install
@mixtool generate all --output-alerts $(MIXIN_OUT_PROMTAIL_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_PROMTAIL_PATH)/rules.yaml --directory $(MIXIN_OUT_PROMTAIL_PATH)/dashboards ${MIXIN_PATH_PROMTAIL}/mixin.libsonnet
endif

promtail-mixin-check: promtail-mixin
@echo "Checking diff"
@git diff --exit-code -- $(MIXIN_OUT_PROMTAIL_PATH) || (echo "Please build mixin by running 'make promtail-mixin'" && false)

###############
# Migrate #
Expand Down
36 changes: 36 additions & 0 deletions production/promtail-mixin-compiled/alerts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
groups:
- name: promtail_alerts
rules:
- alert: PromtailRequestsErrors
annotations:
description: |
{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
summary: Promtail request error rate is high.
expr: |
100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance)
/
sum(rate(promtail_request_duration_seconds_count[1m])) by (namespace, job, route, instance)
> 10
for: 15m
labels:
severity: critical
- alert: PromtailRequestLatency
annotations:
description: |
{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
summary: Promtail request latency P99 is high.
expr: |
job_status_code_namespace:promtail_request_duration_seconds:99quantile > 1
for: 15m
labels:
severity: critical
- alert: PromtailFileMissing
annotations:
description: |
{{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} matches the glob but is not being tailed.
summary: Promtail cannot find a file it should be tailing.
expr: |
promtail_file_bytes_total unless promtail_read_bytes_total
for: 15m
labels:
severity: warning
Loading
Loading