Check db update (#94)

- add metric for time spent saving checkUpdates to DB.
- run 10 goroutines for saving checkUpdates instead of 1
- update go version and remove "eval failed messages" from logs

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/golang:1.11.10
+      - image: circleci/golang:1.13.5
     steps:
       - checkout
       - run: scripts/build.sh
@@ -13,7 +13,7 @@ jobs:
 
   test:
     docker:
-      - image: circleci/golang:1.11.10
+      - image: circleci/golang:1.13.5
     steps:
       - checkout
       - run: test -z "$(gofmt -s -l ./ | tee /dev/stderr)"
@@ -22,7 +22,7 @@ jobs:
 
   deploy:
     docker:
-      - image: circleci/golang:1.11.10
+      - image: circleci/golang:1.13.5
     steps:
       - checkout
       - attach_workspace:

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/golang/snappy v0.0.1
 	github.com/googollee/go-engine.io v0.0.0-20150203162345-597faf3df88a
 	github.com/googollee/go-socket.io v0.0.0-20150201044939-31886c563625
-	github.com/gopherjs/gopherjs v0.0.0-20160129235136-4b53e1bddba0
+	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00
 	github.com/gorilla/websocket v0.0.0-20150515162638-1551221275a7
 	github.com/graarh/golang-socketio v0.0.0-20160527125345-533dd6f9e8e9
 	github.com/grafana/grafana v4.2.0+incompatible
@@ -38,7 +38,7 @@ require (
 	github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec // indirect
 	github.com/jarcoal/httpmock v1.0.4 // indirect
 	github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03
-	github.com/jtolds/gls v4.2.0+incompatible
+	github.com/jtolds/gls v4.20.0+incompatible
 	github.com/klauspost/crc32 v0.0.0-20170210140523-1bab8b35b6bb
 	github.com/lib/pq v0.0.0-20150109111458-19eeca3e30d2
 	github.com/mattn/go-colorable v0.1.2 // indirect
@@ -56,18 +56,20 @@ require (
 	github.com/raintank/tsdb-gw v0.0.0-20190408195803-9b50e6fd3947
 	github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a
 	github.com/sirupsen/logrus v1.2.0
-	github.com/smartystreets/assertions v0.0.0-20160707190355-2063fd1cc7c9
-	github.com/smartystreets/goconvey v0.0.0-20161103171500-019319c870f8
+	github.com/smartystreets/assertions v1.1.0
+	github.com/smartystreets/goconvey v1.6.4
 	github.com/tinylib/msgp v0.0.0-20170101023110-362bfb3384d5
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect
 	github.com/weaveworks/common v0.0.0-20190714171817-ddeaa31513fd // indirect
 	github.com/yanzay/log v0.0.0-20160419144809-87352bb23506
-	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
-	golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
+	golang.org/x/sys v0.0.0-20190412213103-97732733099d
+	golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 // indirect
 	gopkg.in/ini.v1 v1.21.1
 	gopkg.in/macaron.v1 v1.1.8
 	gopkg.in/raintank/schema.v0 v0.0.0-20160713163449-b1d2969aa4a5
 	gopkg.in/raintank/schema.v1 v1.0.0-20170112123755-a323316458b5
 )
+
+go 1.13

go.sum

@@ -74,6 +74,10 @@ github.com/googollee/go-engine.io v0.0.0-20150203162345-597faf3df88a/go.mod h1:M
 github.com/googollee/go-socket.io v0.0.0-20150201044939-31886c563625 h1:sx2FVK4J9Kf1lsYsRipWlFSzhgRUqMnEa631h/WO6Ac=
 github.com/googollee/go-socket.io v0.0.0-20150201044939-31886c563625/go.mod h1:ftBGBMhSYToR5oV4ImIPKvAIsNaTkLC+tTvoNafqxlQ=
 github.com/gopherjs/gopherjs v0.0.0-20160129235136-4b53e1bddba0/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
+github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v0.0.0-20150515162638-1551221275a7 h1:eBy59eVFUXrdV3OIfS3/2Ic73/+bNys8CJe+uqQ+syU=
 github.com/gorilla/websocket v0.0.0-20150515162638-1551221275a7/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/graarh/golang-socketio v0.0.0-20160527125345-533dd6f9e8e9 h1:JcFd+pTCu6zEq8bkd+Beykt1DdcgOGS3VsLDvJFRKSA=
@@ -98,6 +102,8 @@ github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjL
 github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jtolds/gls v4.2.0+incompatible h1:f7Me2eb3Wy6xfgSj1uyi0Ho6c4NY1L4KoX7c3auAhOI=
 github.com/jtolds/gls v4.2.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/klauspost/crc32 v0.0.0-20170210140523-1bab8b35b6bb/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
 github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -162,8 +168,14 @@ github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20160707190355-2063fd1cc7c9 h1:Kg4w5zDU3jgGRghptweqKKNAADO4nqR0Grd9WomB8H4=
 github.com/smartystreets/assertions v0.0.0-20160707190355-2063fd1cc7c9/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
+github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/goconvey v0.0.0-20161103171500-019319c870f8 h1:Jv2ZbhWhqWca52WCpO48zrw3B0LA2mAiniUjeQLDFQ4=
 github.com/smartystreets/goconvey v0.0.0-20161103171500-019319c870f8/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -182,18 +194,28 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/yanzay/log v0.0.0-20160419144809-87352bb23506 h1:RPrQRi67MUoUPg/3wwL9K9vKVWAB180e2WJgxC02CJE=
 github.com/yanzay/log v0.0.0-20160419144809-87352bb23506/go.mod h1:x2hAcqPbZzQz79Fzr9xEe7BM73+UsZ3nBucx2u30oSk=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE=
 golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20150901164945-9c60d1c508f5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -202,7 +224,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53 h1:vmsb6v0zUdmUlXfwKaYrHPPRCV0lHq/IwNIf0ASGjyQ=
+golang.org/x/tools v0.0.0-20200515220128-d3bf790afa53/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.21.1 h1:yoADnB7wX6cPgdGE1GC3PGjN+uM8idmZJuU3ozHTfT8=

pkg/alerting/alerting_eval_test.go

@@ -27,7 +27,7 @@ func check(series []graphite.Series, steps, numProbes int) m.CheckEvalResult {
 		NumProbes: numProbes,
 		Steps:     steps,
 	}
-	result, err := eval(res, &healthSettings)
+	result, err := eval(res, 1, &healthSettings)
 	So(err, ShouldBeNil)
 	return result
 }

pkg/alerting/check_state.go

@@ -2,6 +2,7 @@ package alerting
 
 import (
 	"strings"
+	"time"
 
 	"github.com/raintank/worldping-api/pkg/log"
 	m "github.com/raintank/worldping-api/pkg/models"
@@ -18,8 +19,9 @@ func InitResultHandler() {
 	ResultQueue = make(chan *m.AlertingJob, 1000)
 
 	stateChanges := make(chan *m.AlertingJob, 1000)
-
-	go storeResults(stateChanges)
+	for i := 0; i < 5; i++ {
+		go storeResults(stateChanges)
+	}
 	go handleStateChange(stateChanges)
 
 }
@@ -30,7 +32,9 @@ func storeResults(stateChanges chan *m.AlertingJob) {
 		attempts := 0
 		for !saved && attempts < 3 {
 			attempts++
+			pre := time.Now()
 			change, err := sqlstore.UpdateCheckState(j)
+			executorStateDBUpdate.Value(util.Since(pre))
 			if err != nil {
 				log.Warn("failed to update checkState for checkId=%d. %s", j.Id, err)
 				continue

pkg/alerting/executor.go

@@ -18,6 +18,10 @@ import (
 	"gopkg.in/raintank/schema.v1"
 )
 
+var (
+	ErrNoData = fmt.Errorf("fatal: no data returned for job")
+)
+
 func ChanExecutor(jobQueue <-chan *m.AlertingJob, cache *lru.Cache) {
 	var wg sync.WaitGroup
 	for j := range jobQueue {
@@ -80,10 +84,9 @@ func execute(job *m.AlertingJob, cache *lru.Cache) {
 		return
 	}
 
-	newState, err := eval(res, job.HealthSettings)
+	newState, err := eval(res, job.Id, job.HealthSettings)
 	if err != nil {
 		executorAlertOutcomesErr.Inc()
-		log.Error(3, "Alerting: eval failed for job %q : %s", job, err.Error())
 		return
 	}
 	job.NewState = newState
@@ -107,10 +110,11 @@ func execute(job *m.AlertingJob, cache *lru.Cache) {
 	}
 }
 
-func eval(res graphite.Response, healthSettings *m.CheckHealthSettings) (m.CheckEvalResult, error) {
+func eval(res graphite.Response, checkId int64, healthSettings *m.CheckHealthSettings) (m.CheckEvalResult, error) {
 	if len(res) == 0 {
 		executorGraphiteEmptyResponse.Inc()
-		return m.EvalResultUnknown, fmt.Errorf("fatal: no data returned for job")
+		log.Debug("Alerting: no data returned for job checkId=%d", checkId)
+		return m.EvalResultUnknown, ErrNoData
 	}
 	badEndpoints := 0
 	endpointsWithData := 0

pkg/alerting/init.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/grafana/metrictank/stats"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/raintank/worldping-api/pkg/alerting/jobqueue"
 	"github.com/raintank/worldping-api/pkg/log"
 	"github.com/raintank/worldping-api/pkg/services"
@@ -34,6 +34,7 @@ var (
 
 	executorJobExecDelay        = stats.NewMeter32("alert-executor.job_execution_delay", true)
 	executorStateSaveDelay      = stats.NewMeter32("alert-executor.state_save_delay", true)
+	executorStateDBUpdate       = stats.NewMeter32("alert-executor.state_db_update", true)
 	executorJobQueryGraphite    = stats.NewMeter32("alert-executor.job_query_graphite", true)
 	executorGraphiteMissingVals = stats.NewCounterRate32("alert-executor.graphite-missingVals")