Skip to content

Commit

Permalink
PANDARIA: support unified alert template
Browse files Browse the repository at this point in the history
  • Loading branch information
@GGGitBoy @guangbochen
GGGitBoy authored and guangbochen committed Sep 10, 2020
1 parent 1161a59 commit b33a499
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 125 deletions.
12 changes: 11 additions & 1 deletion pkg/apis/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,17 @@ func sendAlert(req *restful.Request, resp *restful.Response) {
return
}

msg, err := tmpl.ExecuteTextString(td)
tpl, err := options.GetTemplate()
if err != nil {
log.Errorf("get template err:%v", err)
err = resp.WriteErrorString(400, err.Error())
if err != nil {
log.Errorf("failed to write error string err:%v", err)
}
return
}

msg, err := tmpl.ExecuteTextString(td, tpl)
if err != nil {
log.Errorf("tmpl parse err: %v", err)
err = resp.WriteErrorString(500, err.Error())
Expand Down
18 changes: 18 additions & 0 deletions pkg/options/option.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package options
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
Expand Down Expand Up @@ -43,6 +45,22 @@ func Init(configPath string) {
go viper.WatchConfig()
}

func GetTemplate() (string, error) {
mut.RLock()
defer mut.RUnlock()
file, err := os.Open("/etc/webhook-receiver/tmpl/notification.tmpl")
if err != nil {
return "", fmt.Errorf("open notification.tmpl file err:%v", err)
}
defer file.Close()

fileData, err := ioutil.ReadAll(file)
if err != nil {
return "", fmt.Errorf("read notification.tmpl file err:%v", err)
}
return string(fileData), nil
}

func GetReceiverAndSender(receiverName string) (providers.Receiver, providers.Sender, error) {
mut.RLock()
defer mut.RUnlock()
Expand Down
154 changes: 30 additions & 124 deletions pkg/tmpl/tmpl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,139 +2,45 @@ package tmpl

import (
"bytes"
"regexp"
"strings"
"text/template"
)

var tpl *template.Template
func ExecuteTextString(data interface{}, notificationTmpl string) (string, error) {
if notificationTmpl == "" {
return "", nil
}

func init() {
var err error
tpl, err = template.New("").Option("missingkey=zero").Parse(NotificationTmpl)
tmpl := template.New("").Option("missingkey=zero")
tmpl.Funcs(template.FuncMap(DefaultFuncs))
tpl, err := tmpl.Parse(notificationTmpl)
if err != nil {
panic(err)
return "", err
}
}

const (
NotificationTmpl = `
{{- if eq .Status "resolved" -}}
[Resolved]
{{- end -}}
{{- if eq .CommonLabels.alert_type "event" -}}
{{ .CommonLabels.event_type}} event of {{.GroupLabels.resource_kind}} occurred
{{- else if eq .CommonLabels.alert_type "systemService" -}}
The system component {{ .GroupLabels.component_name}} is not running
{{- else if eq .CommonLabels.alert_type "nodeHealthy" -}}
The kubelet on the node {{ .GroupLabels.node_name}} is not healthy
{{- else if eq .CommonLabels.alert_type "nodeCPU" -}}
The CPU usage on the node {{ .GroupLabels.node_name}} is over {{ .CommonLabels.cpu_threshold}}%
{{- else if eq .CommonLabels.alert_type "nodeMemory" -}}
The memory usage on the node {{ .GroupLabels.node_name}} is over {{ .CommonLabels.mem_threshold}}%
{{- else if eq .CommonLabels.alert_type "podNotScheduled" -}}
The Pod {{ if .GroupLabels.namespace}}{{.GroupLabels.namespace}}:{{end}}{{.GroupLabels.pod_name}} is not scheduled
{{- else if eq .CommonLabels.alert_type "podNotRunning" -}}
The Pod {{ if .GroupLabels.namespace}}{{.GroupLabels.namespace}}:{{end}}{{.GroupLabels.pod_name}} is not running
{{- else if eq .CommonLabels.alert_type "podRestarts" -}}
The Pod {{ if .GroupLabels.namespace}}{{.GroupLabels.namespace}}:{{end}}{{.GroupLabels.pod_name}} restarts {{ .CommonLabels.restart_times}} times in {{ .CommonLabels.restart_interval}} sec
{{- else if eq .CommonLabels.alert_type "workload" -}}
The workload {{ if .GroupLabels.workload_namespace}}{{.GroupLabels.workload_namespace}}:{{end}}{{.GroupLabels.workload_name}} has available replicas less than {{ .CommonLabels.available_percentage}}%
{{- else if eq .CommonLabels.alert_type "metric" -}}
The metric {{ .CommonLabels.alert_name}} crossed the threshold
{{ end -}}
{{- if eq .Status "resolved" -}}
{{ range .Alerts.Resolved }}
{{ template "__text_single" . }}
{{ end -}}
{{- else}}
{{ range .Alerts.Firing }}
{{ template "__text_single" . }}
{{ end -}}
{{ end -}}
{{- define "__text_single" -}}
Server URL: {{ .Labels.server_url}}
Alert Name: {{ .Labels.alert_name}}
Severity: {{ .Labels.severity}}
Cluster Name: {{.Labels.cluster_name}}
{{- if .Labels.node_ip }}
Node IP: {{ .Labels.node_ip}}{{ end -}}
{{- if .Labels.pod_ip }}
Pod IP: {{ .Labels.pod_ip}}{{ end -}}
{{- if eq .Labels.alert_type "event" }}
{{- if .Labels.workload_name }}
Workload Name: {{.Labels.workload_name}}{{ end }}
Target: {{ if .Labels.target_namespace -}}{{.Labels.target_namespace}}:{{ end -}}{{.Labels.target_name}}
Count: {{ .Labels.event_count}}
Event Message: {{ .Labels.event_message}}
First Seen: {{ .Labels.event_firstseen}}
Last Seen: {{ .Labels.event_lastseen}}
{{- else if eq .Labels.alert_type "nodeCPU" }}
Used CPU: {{ .Labels.used_cpu}} m
Total CPU: {{ .Labels.total_cpu}} m
{{- else if eq .Labels.alert_type "nodeMemory" }}
Used Memory: {{ .Labels.used_mem}}
Total Memory: {{ .Labels.total_mem}}
{{- else if eq .Labels.alert_type "podRestarts" }}
Project Name: {{ .Labels.project_name}}
Namespace: {{ .Labels.namespace}}
{{- if .Labels.workload_name }}
Workload Name: {{.Labels.workload_name}}
{{ end -}}
Container Name: {{ .Labels.container_name}}
{{- else if eq .Labels.alert_type "podNotRunning" }}
Project Name: {{ .Labels.project_name}}
Namespace: {{ .Labels.namespace}}
{{- if .Labels.workload_name }}
Workload Name: {{.Labels.workload_name}}
{{ end -}}
Container Name: {{ .Labels.container_name}}
{{- else if eq .Labels.alert_type "podNotScheduled" }}
Project Name: {{ .Labels.project_name}}
Namespace: {{ .Labels.namespace}}
Pod Name: {{ .Labels.pod_name}}
{{- if .Labels.workload_name }}
Workload Name: {{.Labels.workload_name}}
{{ end -}}
{{- else if eq .Labels.alert_type "workload" }}
Project Name: {{ .Labels.project_name}}
Available Replicas: {{ .Labels.available_replicas}}
Desired Replicas: {{ .Labels.desired_replicas}}
{{- else if eq .Labels.alert_type "metric" }}
{{- if .Labels.namespace }}
Namespace: {{ .Labels.namespace}}{{ end }}
{{- if .Labels.project_name }}
Project Name: {{ .Labels.project_name}}{{ end }}
{{- if .Labels.pod_name }}
Pod Name: {{ .Labels.pod_name}}{{ else if .Labels.pod -}}Pod Name: {{ .Labels.pod}}{{ end }}
Expression: {{ .Labels.expression}}
{{- if .Labels.threshold_value }}
Description: Threshold Crossed: datapoint value {{ .Annotations.current_value}} was {{ .Labels.comparison}} to the threshold ({{ .Labels.threshold_value}}) for ({{ .Labels.duration}})
{{- else}}
Description: The configured event happened for ({{ .Labels.duration}}): expression matched, datapoint value is {{ .Annotations.current_value}}
{{ end -}}
{{ end -}}
{{- if .Labels.logs }}
Logs: {{ .Labels.logs}}
{{ end -}}
{{ end -}}
`
)

func ExecuteTextString(data interface{}) (string, error) {
buf := bytes.Buffer{}
if err := tpl.Execute(&buf, data); err != nil {
if err := tpl.ExecuteTemplate(&buf, "title.text.list", data); err != nil {
return "", err
}

return buf.String(), nil
}

type FuncMap map[string]interface{}

var DefaultFuncs = FuncMap{
"toUpper": strings.ToUpper,
"toLower": strings.ToLower,
"title": strings.Title,
"join": func(sep string, s []string) string {
return strings.Join(s, sep)
},
"match": regexp.MatchString,
"reReplaceAll": func(pattern, repl, text string) string {
re := regexp.MustCompile(pattern)
return re.ReplaceAllString(text, repl)
},
"stringSlice": func(s ...string) []string {
return s
},
}

0 comments on commit b33a499

Please sign in to comment.