fix: 5xx alarm (#166)

* fix: 5xx alarm * fix: var * fix: alert syntax * chore: reduce threshold
reown-com · Jan 19, 2024 · e98c652 · e98c652
1 parent abbc120
commit e98c652
Showing 1 changed file with 32 additions and 1 deletion.
diff --git a/terraform/monitoring/panels/lb/error_5xx.libsonnet b/terraform/monitoring/panels/lb/error_5xx.libsonnet
@@ -4,7 +4,7 @@ local defaults  = import '../../grafonnet-lib/defaults.libsonnet';
 local panels    = grafana.panels;
 local targets   = grafana.targets;
 
-local threshold = 100;
+local threshold = 5;
 
 local _configuration = defaults.configuration.timeseries
   .withSoftLimit(
@@ -29,6 +29,37 @@ local _configuration = defaults.configuration.timeseries
       value = threshold,
     )
 
+    .setAlert(
+      grafana.alert.new(
+        namespace     = vars.namespace,
+        name          = "%(env)s - 5XX alert"     % { env: grafana.utils.strings.capitalize(vars.environment) },
+        message       = '%(env)s - Notify - 5XX alert'  % { env: grafana.utils.strings.capitalize(vars.environment) },
+        notifications = vars.notifications,
+        noDataState   = 'no_data',
+        period        = '0m',
+        conditions    = [
+          grafana.alertCondition.new(
+            evaluatorParams = [ 5 ],
+            evaluatorType   = 'gt',
+            operatorType    = 'or',
+            queryRefId      = 'ELB',
+            queryTimeStart  = '5m',
+            queryTimeEnd    = 'now',
+            reducerType     = grafana.alert_reducers.Avg
+          ),
+          grafana.alertCondition.new(
+            evaluatorParams = [ threshold ],
+            evaluatorType   = 'gt',
+            operatorType    = 'or',
+            queryRefId      = 'Target',
+            queryTimeStart  = '5m',
+            queryTimeEnd    = 'now',
+            reducerType     = grafana.alert_reducers.Avg
+          ),
+        ],
+      )
+    )
+
     .addTarget(targets.cloudwatch(
       alias       = 'ELB',
       datasource  = ds.cloudwatch,