From 8883a861f7ed7173e528274482fe53d52ed7021c Mon Sep 17 00:00:00 2001 From: Kedar Vijay Kulkarni Date: Tue, 9 Nov 2021 18:48:07 -0500 Subject: [PATCH] Adding some code around slack notification, still work remaining to do Signed-off-by: Kedar Vijay Kulkarni --- .gitignore | 2 +- README.md | 3 ++- cmd/analyze/analyze.go | 12 +++++++++--- cmd/notify/notifications.go | 30 ++++++++++++++++++++++++++++++ config/slack.yaml.template | 3 +++ go.mod | 2 ++ go.sum | 7 +++++++ main.go | 4 ++++ 8 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 cmd/notify/notifications.go create mode 100644 config/slack.yaml.template diff --git a/.gitignore b/.gitignore index ae1b1f9..4fd957c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -config/prometheus.yaml \ No newline at end of file +config/*.yaml \ No newline at end of file diff --git a/README.md b/README.md index 468c1b2..b70a432 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,8 @@ This tool allows OpenShift users to run a watcher for Prometheus queries and def * [x] File logging the output * [x] Print output to screen even when logging enabled - simultaneously * [x] Let user decide query frequency -* [ ] Notify/Do Something when results don't match conditions +* [x] Slack Notification +* [ ] Notify/Do Something(e.g. Pause/Kill benchmark jobs to preserve cluster) when results don't match conditions * [ ] Spawn goroutines to keep running queries and evaluating results to handle scale - e.g. when we have very large number of queries in the yaml file, we can divide and concurrently run queries diff --git a/cmd/analyze/analyze.go b/cmd/analyze/analyze.go index 0249243..82f67a8 100644 --- a/cmd/analyze/analyze.go +++ b/cmd/analyze/analyze.go @@ -6,8 +6,10 @@ import ( "log" "math/rand" "strconv" + "strings" "time" + notify "github.com/kedark3/cpa/cmd/notify" prometheus "github.com/kedark3/cpa/cmd/prometheus" exutil "github.com/openshift/openshift-tests/test/extended/util" @@ -107,9 +109,13 @@ func Notify(c chan string) { for { select { case msg := <-c: - log.Println("***************************************") - log.Println("Received following on the channel:", msg) - log.Println("***************************************") + msgFmt := fmt.Sprintf(` +%s +Received following on the channel: %s +%[1]s + `, strings.Repeat("~", 80), msg) + fmt.Println(msgFmt) + notify.SlackNotify(msg) default: fmt.Printf("\r%s Please Wait. No new message received on the channel....", waitChars[rand.Intn(4)]) time.Sleep(time.Millisecond * 500) diff --git a/cmd/notify/notifications.go b/cmd/notify/notifications.go new file mode 100644 index 0000000..63aa37e --- /dev/null +++ b/cmd/notify/notifications.go @@ -0,0 +1,30 @@ +package Notify + +import ( + "fmt" + + "github.com/slack-go/slack" +) + +type SlackConfig struct { + UserID string `json:"userID"` + ChannelID string `json:"channelID"` + SlackToken string `json:"slackToken"` +} + +// TODO Add function to Read Slack Config + +func (s SlackConfig) SlackNotify(message string) { + api := slack.New(s.SlackToken, slack.OptionDebug(true)) + msgText := slack.NewTextBlockObject("mrkdwn", fmt.Sprintf("Hi <@%s>, following query failed:%s", s.UserID, message), false, false) + msgSection := slack.NewSectionBlock(msgText, nil, nil) + msgBlock := slack.MsgOptionBlocks( + msgSection, + ) + _, _, _, err := api.SendMessage(s.ChannelID, msgBlock) + if err != nil { + fmt.Printf("%s\n", err) + return + } + +} diff --git a/config/slack.yaml.template b/config/slack.yaml.template new file mode 100644 index 0000000..02c8e1b --- /dev/null +++ b/config/slack.yaml.template @@ -0,0 +1,3 @@ +slackToken: xoxb- +channelID: C0XXXXXXXXX +userID: U0XXXXXXXXX \ No newline at end of file diff --git a/go.mod b/go.mod index 0e19ca8..2d8af20 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,9 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.16.0 github.com/openshift/openshift-tests v0.0.0-20210916082130-4fca21c38ee6 + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.6.0 + github.com/slack-go/slack v0.9.5 // indirect k8s.io/api v0.17.1 k8s.io/apimachinery v0.17.1 k8s.io/kubernetes v1.21.0 diff --git a/go.sum b/go.sum index 1cb171d..637e0bb 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,7 @@ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2K github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= @@ -307,6 +308,8 @@ github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= @@ -494,6 +497,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -537,6 +542,8 @@ github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjM github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/slack-go/slack v0.9.5 h1:j7uOUDowybWf9eSgZg/AbGx6J1OPJB6SE8Z5dNl6Mtw= +github.com/slack-go/slack v0.9.5/go.mod h1:wWL//kk0ho+FcQXcBTmEafUI5dz4qz5f4mMk8oIkioQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= diff --git a/main.go b/main.go index 02c70c4..2016ad8 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ func main() { QueryFrequency time.Duration `arg:"-f,--query-frequency" help:"How often do we run queries. You can pass values like 4h or 1h10m10s" default:"20s"` Timeout time.Duration `arg:"-t,--timeout" help:"Duration to run Continuous Performance Analysis. You can pass values like 4h or 1h10m10s" default:"4h"` LogOutput bool `arg:"-l,--log-output" help:"Output will be stored in a log file instead of stdout." default:"false"` + KillBenchmark string `arg:"-k,--kill-benchmark" help:"When CPA is running in parallel with benchmark job, let CPA know to kill benchmark if any query fail. E.g. -k run_nodedensity_from_git.sh Helpful to preserve cluster for further analysis." default:""` } arg.MustParse(&args) @@ -41,6 +42,9 @@ func main() { //set output of logs to f log.SetOutput(multiWriter) } + if args.KillBenchmark != "" { + // TODO Add logic to handle running benchmark processes when analyze sends notification on the channel. + } oc := exutil.NewCLI("prometheus-cpa", exutil.KubeConfigPath()) // secrets, err := oc.AdminKubeClient().CoreV1().Secrets("openshift-monitoring").List(metav1.ListOptions{})