From 97e393d2e74dabb9c6e88e40bae892514b5542bf Mon Sep 17 00:00:00 2001 From: Samantha Date: Mon, 29 Mar 2021 12:56:54 -0700 Subject: [PATCH] boulder-observer (#5315) Add configuration driven Prometheus black box metric exporter --- cmd/boulder-observer/README.md | 216 ++++++++++++++++++++++++ cmd/boulder-observer/main.go | 35 ++++ observer/mon_conf.go | 63 +++++++ observer/mon_conf_test.go | 33 ++++ observer/monitor.go | 40 +++++ observer/obs_conf.go | 137 +++++++++++++++ observer/obs_conf_test.go | 133 +++++++++++++++ observer/observer.go | 21 +++ observer/probers/dns/dns.go | 55 ++++++ observer/probers/dns/dns_conf.go | 133 +++++++++++++++ observer/probers/dns/dns_conf_test.go | 195 +++++++++++++++++++++ observer/probers/http/http.go | 47 ++++++ observer/probers/http/http_conf.go | 78 +++++++++ observer/probers/http/http_conf_test.go | 76 +++++++++ observer/probers/mock/mock_conf.go | 38 +++++ observer/probers/mock/mock_prober.go | 26 +++ observer/probers/prober.go | 77 +++++++++ test/config-next/observer.yml | 91 ++++++++++ test/prometheus/prometheus.yml | 1 + 19 files changed, 1495 insertions(+) create mode 100644 cmd/boulder-observer/README.md create mode 100644 cmd/boulder-observer/main.go create mode 100644 observer/mon_conf.go create mode 100644 observer/mon_conf_test.go create mode 100644 observer/monitor.go create mode 100644 observer/obs_conf.go create mode 100644 observer/obs_conf_test.go create mode 100644 observer/observer.go create mode 100644 observer/probers/dns/dns.go create mode 100644 observer/probers/dns/dns_conf.go create mode 100644 observer/probers/dns/dns_conf_test.go create mode 100644 observer/probers/http/http.go create mode 100644 observer/probers/http/http_conf.go create mode 100644 observer/probers/http/http_conf_test.go create mode 100644 observer/probers/mock/mock_conf.go create mode 100644 observer/probers/mock/mock_prober.go create mode 100644 observer/probers/prober.go create mode 100644 test/config-next/observer.yml diff --git a/cmd/boulder-observer/README.md b/cmd/boulder-observer/README.md new file mode 100644 index 00000000000..c9a760e76e5 --- /dev/null +++ b/cmd/boulder-observer/README.md @@ -0,0 +1,216 @@ +# boulder-observer + +A modular configuration driven approach to black box monitoring with +Prometheus. + +* [boulder-observer](#boulder-observer) + * [Usage](#usage) + * [Options](#options) + * [Starting the boulder-observer + daemon](#starting-the-boulder-observer-daemon) + * [Configuration](#configuration) + * [Root](#root) + * [Schema](#schema) + * [Example](#example) + * [Monitors](#monitors) + * [Schema](#schema-1) + * [Example](#example-1) + * [Probers](#probers) + * [DNS](#dns) + * [Schema](#schema-2) + * [Example](#example-2) + * [HTTP](#http) + * [Schema](#schema-3) + * [Example](#example-3) + * [Metrics](#metrics) + * [obs_monitors](#obs_monitors) + * [obs_observations](#obs_observations) + * [Development](#development) + * [Starting Prometheus locally](#starting-prometheus-locally) + * [Viewing metrics locally](#viewing-metrics-locally) + +## Usage + +### Options + +```shell +$ ./boulder-observer -help + -config string + Path to boulder-observer configuration file (default "config.yml") +``` + +### Starting the boulder-observer daemon + +```shell +$ ./boulder-observer -config test/config-next/observer.yml +I152525 boulder-observer _KzylQI Versions: main=(Unspecified Unspecified) Golang=(go1.16.2) BuildHost=(Unspecified) +I152525 boulder-observer q_D84gk Initializing boulder-observer daemon from config: test/config-next/observer.yml +I152525 boulder-observer 7aq68AQ all monitors passed validation +I152527 boulder-observer yaefiAw kind=[HTTP] success=[true] duration=[0.130097] name=[https://letsencrypt.org-[200]] +I152527 boulder-observer 65CuDAA kind=[HTTP] success=[true] duration=[0.148633] name=[http://letsencrypt.org/foo-[200 404]] +I152530 boulder-observer idi4rwE kind=[DNS] success=[false] duration=[0.000093] name=[[2606:4700:4700::1111]:53-udp-A-google.com-recurse] +I152530 boulder-observer prOnrw8 kind=[DNS] success=[false] duration=[0.000242] name=[[2606:4700:4700::1111]:53-tcp-A-google.com-recurse] +I152530 boulder-observer 6uXugQw kind=[DNS] success=[true] duration=[0.022962] name=[1.1.1.1:53-udp-A-google.com-recurse] +I152530 boulder-observer to7h-wo kind=[DNS] success=[true] duration=[0.029860] name=[owen.ns.cloudflare.com:53-udp-A-letsencrypt.org-no-recurse] +I152530 boulder-observer ovDorAY kind=[DNS] success=[true] duration=[0.033820] name=[owen.ns.cloudflare.com:53-tcp-A-letsencrypt.org-no-recurse] +... +``` + +## Configuration + +Configuration is provided via a YAML file. + +### Root + +#### Schema + +`debugaddr`: The Prometheus scrape port prefixed with a single colon +(e.g. `:8040`). + +`buckets`: List of floats representing Prometheus histogram buckets (e.g +`[.001, .002, .005, .01, .02, .05, .1, .2, .5, 1, 2, 5, 10]`) + +`syslog`: Map of log levels, see schema below. + +- `stdoutlevel`: Log level for stdout, see legend below. +- `sysloglevel`:Log level for stdout, see legend below. + +`0`: *EMERG* `1`: *ALERT* `2`: *CRIT* `3`: *ERR* `4`: *WARN* `5`: +*NOTICE* `6`: *INFO* `7`: *DEBUG* + +`monitors`: List of monitors, see [monitors](#monitors) for schema. + +#### Example + +```yaml +debugaddr: :8040 +buckets: [.001, .002, .005, .01, .02, .05, .1, .2, .5, 1, 2, 5, 10] +syslog: + stdoutlevel: 6 + sysloglevel: 6 + - + ... +``` + +### Monitors + +#### Schema + +`period`: Interval between probing attempts (e.g. `1s` `1m` `1h`). + +`kind`: Kind of prober to use, see [probers](#probers) for schema. + +`settings`: Map of prober settings, see [probers](#probers) for schema. + +#### Example + +```yaml +monitors: + - + period: 5s + kind: DNS + settings: + ... +``` + +### Probers + +#### DNS + +##### Schema + +`protocol`: Protocol to use, options are: `udp` or `tcp`. + +`server`: Hostname, IPv4 address, or IPv6 address surrounded with +brackets + port of the DNS server to send the query to (e.g. +`example.com:53`, `1.1.1.1:53`, or `[2606:4700:4700::1111]:53`). + +`recurse`: Bool indicating if recursive resolution is desired. + +`query_name`: Name to query (e.g. `example.com`). + +`query_type`: Record type to query, options are: `A`, `AAAA`, `TXT`, or +`CAA`. + +##### Example + +```yaml +monitors: + - + period: 5s + kind: DNS + settings: + protocol: tcp + server: [2606:4700:4700::1111]:53 + recurse: false + query_name: letsencrypt.org + query_type: A +``` + +#### HTTP + +##### Schema + +`url`: Scheme + Hostname to send a request to (e.g. +`https://example.com`). + +`rcodes`: List of expected HTTP response codes. + +##### Example + +```yaml +monitors: + - + period: 2s + kind: HTTP + settings: + url: http://letsencrypt.org/FOO + rcodes: [200, 404] +``` + +## Metrics + +Observer provides the following metrics. + +### obs_monitors + +Count of configured monitors. + +**Labels:** + +`kind`: Kind of Prober the monitor is configured to use. + +`valid`: Bool indicating whether settings provided could be validated +for the `kind` of Prober specified. + +### obs_observations + +**Labels:** + +`name`: Name of the monitor. + +`kind`: Kind of prober the monitor is configured to use. + +`duration`: Duration of the probing in seconds. + +`success`: Bool indicating whether the result of the probe attempt was +successful. + +**Bucketed response times:** + +This is configurable, see `buckets` under [root/schema](#schema). + +## Development + +### Starting Prometheus locally + +Please note, this assumes you've installed a local Prometheus binary. + +```shell +prometheus --config.file=boulder/test/prometheus/prometheus.yml +``` + +### Viewing metrics locally + +When developing with a local Prometheus instance you can use this link +to view metrics: [link](http://0.0.0.0:9090) \ No newline at end of file diff --git a/cmd/boulder-observer/main.go b/cmd/boulder-observer/main.go new file mode 100644 index 00000000000..a7414653b5c --- /dev/null +++ b/cmd/boulder-observer/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "flag" + "io/ioutil" + + "github.com/letsencrypt/boulder/cmd" + "github.com/letsencrypt/boulder/observer" + "gopkg.in/yaml.v2" +) + +func main() { + configPath := flag.String( + "config", "config.yml", "Path to boulder-observer configuration file") + flag.Parse() + + configYAML, err := ioutil.ReadFile(*configPath) + cmd.FailOnError(err, "failed to read config file") + + // Parse the YAML config file. + var config observer.ObsConf + err = yaml.Unmarshal(configYAML, &config) + if err != nil { + cmd.FailOnError(err, "failed to parse YAML config") + } + + // Make an `Observer` object. + observer, err := config.MakeObserver() + if err != nil { + cmd.FailOnError(err, "config failed validation") + } + + // Start the `Observer` daemon. + observer.Start() +} diff --git a/observer/mon_conf.go b/observer/mon_conf.go new file mode 100644 index 00000000000..f04535929db --- /dev/null +++ b/observer/mon_conf.go @@ -0,0 +1,63 @@ +package observer + +import ( + "errors" + "strings" + "time" + + "github.com/letsencrypt/boulder/cmd" + "github.com/letsencrypt/boulder/observer/probers" + "gopkg.in/yaml.v2" +) + +// MonConf is exported to receive YAML configuration in `ObsConf`. +type MonConf struct { + Period cmd.ConfigDuration `yaml:"period"` + Kind string `yaml:"kind"` + Settings probers.Settings `yaml:"settings"` +} + +// validatePeriod ensures the received `Period` field is at least 1µs. +func (c *MonConf) validatePeriod() error { + if c.Period.Duration < 1*time.Microsecond { + return errors.New("period must be at least 1µs") + } + return nil +} + +// unmarshalConfigurer constructs a `Configurer` by marshaling the +// value of the `Settings` field back to bytes, then passing it to the +// `UnmarshalSettings` method of the `Configurer` type specified by the +// `Kind` field. +func (c MonConf) unmarshalConfigurer() (probers.Configurer, error) { + kind := strings.Trim(strings.ToLower(c.Kind), " ") + configurer, err := probers.GetConfigurer(kind) + if err != nil { + return nil, err + } + settings, _ := yaml.Marshal(c.Settings) + configurer, err = configurer.UnmarshalSettings(settings) + if err != nil { + return nil, err + } + return configurer, nil +} + +// makeMonitor constructs a `monitor` object from the contents of the +// bound `MonConf`. If the `MonConf` cannot be validated, an error +// appropriate for end-user consumption is returned instead. +func (c MonConf) makeMonitor() (*monitor, error) { + err := c.validatePeriod() + if err != nil { + return nil, err + } + probeConf, err := c.unmarshalConfigurer() + if err != nil { + return nil, err + } + prober, err := probeConf.MakeProber() + if err != nil { + return nil, err + } + return &monitor{c.Period.Duration, prober}, nil +} diff --git a/observer/mon_conf_test.go b/observer/mon_conf_test.go new file mode 100644 index 00000000000..6bc1a605c29 --- /dev/null +++ b/observer/mon_conf_test.go @@ -0,0 +1,33 @@ +package observer + +import ( + "testing" + "time" + + "github.com/letsencrypt/boulder/cmd" +) + +func TestMonConf_validatePeriod(t *testing.T) { + type fields struct { + Period cmd.ConfigDuration + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + {"valid", fields{cmd.ConfigDuration{Duration: 1 * time.Microsecond}}, false}, + {"1 nanosecond", fields{cmd.ConfigDuration{Duration: 1 * time.Nanosecond}}, true}, + {"none supplied", fields{cmd.ConfigDuration{}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &MonConf{ + Period: tt.fields.Period, + } + if err := c.validatePeriod(); (err != nil) != tt.wantErr { + t.Errorf("MonConf.validatePeriod() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/observer/monitor.go b/observer/monitor.go new file mode 100644 index 00000000000..a8f2b3bc9ed --- /dev/null +++ b/observer/monitor.go @@ -0,0 +1,40 @@ +package observer + +import ( + "strconv" + "time" + + blog "github.com/letsencrypt/boulder/log" + "github.com/letsencrypt/boulder/observer/probers" +) + +type monitor struct { + period time.Duration + prober probers.Prober +} + +// start spins off a 'Prober' goroutine on an interval of `m.period` +// with a timeout of half `m.period` +func (m monitor) start(logger blog.Logger) { + ticker := time.NewTicker(m.period) + timeout := m.period / 2 + go func() { + for { + select { + case <-ticker.C: + // Attempt to probe the configured target. + success, dur := m.prober.Probe(timeout) + + // Produce metrics to be scraped by Prometheus. + histObservations.WithLabelValues( + m.prober.Name(), m.prober.Kind(), strconv.FormatBool(success), + ).Observe(dur.Seconds()) + + // Log the outcome of the probe attempt. + logger.Infof( + "kind=[%s] success=[%v] duration=[%f] name=[%s]", + m.prober.Kind(), success, dur.Seconds(), m.prober.Name()) + } + } + }() +} diff --git a/observer/obs_conf.go b/observer/obs_conf.go new file mode 100644 index 00000000000..b1c423d4871 --- /dev/null +++ b/observer/obs_conf.go @@ -0,0 +1,137 @@ +package observer + +import ( + "errors" + "fmt" + "net" + "strconv" + + "github.com/letsencrypt/boulder/cmd" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + countMonitors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "obs_monitors", + Help: "details of each configured monitor", + }, + []string{"kind", "valid"}, + ) + histObservations *prometheus.HistogramVec +) + +// ObsConf is exported to receive YAML configuration. +type ObsConf struct { + DebugAddr string `yaml:"debugaddr"` + Buckets []float64 `yaml:"buckets"` + Syslog cmd.SyslogConfig `yaml:"syslog"` + MonConfs []*MonConf `yaml:"monitors"` +} + +// validateSyslog ensures the the `Syslog` field received by `ObsConf` +// contains valid log levels. +func (c *ObsConf) validateSyslog() error { + syslog, stdout := c.Syslog.SyslogLevel, c.Syslog.StdoutLevel + if stdout < 0 || stdout > 7 || syslog < 0 || syslog > 7 { + return fmt.Errorf( + "invalid 'syslog', '%+v', valid log levels are 0-7", c.Syslog) + } + return nil +} + +// validateDebugAddr ensures the `debugAddr` received by `ObsConf` is +// properly formatted and a valid port. +func (c *ObsConf) validateDebugAddr() error { + _, p, err := net.SplitHostPort(c.DebugAddr) + if err != nil { + return fmt.Errorf( + "invalid 'debugaddr', %q, not expected format", c.DebugAddr) + } + port, _ := strconv.Atoi(p) + if port <= 0 || port > 65535 { + return fmt.Errorf( + "invalid 'debugaddr','%d' is not a valid port", port) + } + return nil +} + +func (c *ObsConf) makeMonitors() ([]*monitor, []error, error) { + var errs []error + var monitors []*monitor + for e, m := range c.MonConfs { + entry := strconv.Itoa(e + 1) + monitor, err := m.makeMonitor() + if err != nil { + // append validation error to errs + errs = append( + errs, fmt.Errorf( + "'monitors' entry #%s couldn't be validated: %v", entry, err)) + + // increment metrics + countMonitors.WithLabelValues(m.Kind, "false").Inc() + } else { + // append monitor to monitors + monitors = append(monitors, monitor) + + // increment metrics + countMonitors.WithLabelValues(m.Kind, "true").Inc() + } + } + if len(c.MonConfs) == len(errs) { + return nil, errs, errors.New("no valid monitors, cannot continue") + } + return monitors, errs, nil +} + +// MakeObserver constructs an `Observer` object from the contents of the +// bound `ObsConf`. If the `ObsConf` cannot be validated, an error +// appropriate for end-user consumption is returned instead. +func (c *ObsConf) MakeObserver() (*Observer, error) { + err := c.validateSyslog() + if err != nil { + return nil, err + } + + err = c.validateDebugAddr() + if err != nil { + return nil, err + } + + if len(c.MonConfs) == 0 { + return nil, errors.New("no monitors provided") + } + + if len(c.Buckets) == 0 { + return nil, errors.New("no histogram buckets provided") + } + + // Start monitoring and logging. + metrics, logger := cmd.StatsAndLogging(c.Syslog, c.DebugAddr) + histObservations = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "obs_observations", + Help: "details of each probe attempt", + Buckets: c.Buckets, + }, []string{"name", "kind", "success"}) + metrics.MustRegister(countMonitors) + metrics.MustRegister(histObservations) + defer logger.AuditPanic() + logger.Info(cmd.VersionString()) + logger.Infof("Initializing boulder-observer daemon") + logger.Debugf("Using config: %+v", c) + + monitors, errs, err := c.makeMonitors() + if len(errs) != 0 { + logger.Errf("%d of %d monitors failed validation", len(errs), len(c.MonConfs)) + for _, err := range errs { + logger.Errf("%s", err) + } + } else { + logger.Info("all monitors passed validation") + } + if err != nil { + return nil, err + } + return &Observer{logger, monitors}, nil +} diff --git a/observer/obs_conf_test.go b/observer/obs_conf_test.go new file mode 100644 index 00000000000..01d0b021142 --- /dev/null +++ b/observer/obs_conf_test.go @@ -0,0 +1,133 @@ +package observer + +import ( + "errors" + "testing" + "time" + + "github.com/letsencrypt/boulder/cmd" + "github.com/letsencrypt/boulder/observer/probers" + _ "github.com/letsencrypt/boulder/observer/probers/mock" +) + +const ( + debugAddr = ":8040" + errDBZMsg = "over 9000" + mockConf = "MockConf" +) + +func TestObsConf_makeMonitors(t *testing.T) { + var errDBZ = errors.New(errDBZMsg) + var cfgSyslog = cmd.SyslogConfig{StdoutLevel: 6, SyslogLevel: 6} + var cfgDur = cmd.ConfigDuration{Duration: time.Second * 5} + var cfgBuckets = []float64{.001} + var validMonConf = &MonConf{ + cfgDur, mockConf, probers.Settings{"valid": true, "pname": "foo", "pkind": "bar"}} + var invalidMonConf = &MonConf{ + cfgDur, mockConf, probers.Settings{"valid": false, "errmsg": errDBZMsg, "pname": "foo", "pkind": "bar"}} + type fields struct { + Syslog cmd.SyslogConfig + Buckets []float64 + DebugAddr string + MonConfs []*MonConf + } + tests := []struct { + name string + fields fields + errs []error + wantErr bool + }{ + // valid + {"1 valid", fields{cfgSyslog, cfgBuckets, debugAddr, []*MonConf{validMonConf}}, nil, false}, + {"2 valid", fields{ + cfgSyslog, cfgBuckets, debugAddr, []*MonConf{validMonConf, validMonConf}}, nil, false}, + {"1 valid, 1 invalid", fields{ + cfgSyslog, cfgBuckets, debugAddr, []*MonConf{validMonConf, invalidMonConf}}, []error{errDBZ}, false}, + {"1 valid, 2 invalid", fields{ + cfgSyslog, cfgBuckets, debugAddr, []*MonConf{invalidMonConf, validMonConf, invalidMonConf}}, []error{errDBZ, errDBZ}, false}, + // invalid + {"1 invalid", fields{cfgSyslog, cfgBuckets, debugAddr, []*MonConf{invalidMonConf}}, []error{errDBZ}, true}, + {"0", fields{cfgSyslog, cfgBuckets, debugAddr, []*MonConf{}}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ObsConf{ + Syslog: tt.fields.Syslog, + Buckets: tt.fields.Buckets, + DebugAddr: tt.fields.DebugAddr, + MonConfs: tt.fields.MonConfs, + } + _, errs, err := c.makeMonitors() + if len(errs) != len(tt.errs) { + t.Errorf("ObsConf.validateMonConfs() errs = %d, want %d", len(errs), len(tt.errs)) + t.Logf("%v", errs) + } + if (err != nil) != tt.wantErr { + t.Errorf("ObsConf.validateMonConfs() err = %v, want %v", err, tt.wantErr) + } + }) + } +} + +func TestObsConf_ValidateDebugAddr(t *testing.T) { + type fields struct { + DebugAddr string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // valid + {"max len and range", fields{":65535"}, false}, + {"min len and range", fields{":1"}, false}, + {"2 digits", fields{":80"}, false}, + // invalid + {"out of range high", fields{":65536"}, true}, + {"out of range low", fields{":0"}, true}, + {"not even a port", fields{":foo"}, true}, + {"missing :", fields{"foo"}, true}, + {"missing port", fields{"foo:"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ObsConf{ + DebugAddr: tt.fields.DebugAddr, + } + if err := c.validateDebugAddr(); (err != nil) != tt.wantErr { + t.Errorf("ObsConf.ValidateDebugAddr() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestObsConf_validateSyslog(t *testing.T) { + type fields struct { + Syslog cmd.SyslogConfig + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // valid + {"valid", fields{cmd.SyslogConfig{StdoutLevel: 6, SyslogLevel: 6}}, false}, + // invalid + {"both too high", fields{cmd.SyslogConfig{StdoutLevel: 9, SyslogLevel: 9}}, true}, + {"stdout too high", fields{cmd.SyslogConfig{StdoutLevel: 9, SyslogLevel: 6}}, true}, + {"syslog too high", fields{cmd.SyslogConfig{StdoutLevel: 6, SyslogLevel: 9}}, true}, + {"both too low", fields{cmd.SyslogConfig{StdoutLevel: -1, SyslogLevel: -1}}, true}, + {"stdout too low", fields{cmd.SyslogConfig{StdoutLevel: -1, SyslogLevel: 6}}, true}, + {"syslog too low", fields{cmd.SyslogConfig{StdoutLevel: 6, SyslogLevel: -1}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &ObsConf{ + Syslog: tt.fields.Syslog, + } + if err := c.validateSyslog(); (err != nil) != tt.wantErr { + t.Errorf("ObsConf.validateSyslog() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/observer/observer.go b/observer/observer.go new file mode 100644 index 00000000000..6ef7075a1f4 --- /dev/null +++ b/observer/observer.go @@ -0,0 +1,21 @@ +package observer + +import ( + blog "github.com/letsencrypt/boulder/log" + _ "github.com/letsencrypt/boulder/observer/probers/dns" + _ "github.com/letsencrypt/boulder/observer/probers/http" +) + +// Observer is the steward of goroutines started for each `monitor`. +type Observer struct { + logger blog.Logger + monitors []*monitor +} + +// Start spins off a goroutine for each monitor and then runs forever. +func (o Observer) Start() { + for _, mon := range o.monitors { + go mon.start(o.logger) + } + select {} +} diff --git a/observer/probers/dns/dns.go b/observer/probers/dns/dns.go new file mode 100644 index 00000000000..5cb7676df5c --- /dev/null +++ b/observer/probers/dns/dns.go @@ -0,0 +1,55 @@ +package probers + +import ( + "fmt" + "time" + + "github.com/miekg/dns" +) + +// DNSProbe is the exported 'Prober' object for monitors configured to +// perform DNS requests. +type DNSProbe struct { + proto string + server string + recurse bool + qname string + qtype uint16 +} + +// Name returns a string that uniquely identifies the monitor. +func (p DNSProbe) Name() string { + recursion := func() string { + if p.recurse { + return "recurse" + } + return "no-recurse" + }() + return fmt.Sprintf( + "%s-%s-%s-%s-%s", p.server, p.proto, recursion, dns.TypeToString[p.qtype], p.qname) +} + +// Kind returns a name that uniquely identifies the `Kind` of `Prober`. +func (p DNSProbe) Kind() string { + return "DNS" +} + +// Probe performs the configured DNS query. +func (p DNSProbe) Probe(timeout time.Duration) (bool, time.Duration) { + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(p.qname), p.qtype) + m.RecursionDesired = p.recurse + c := dns.Client{Timeout: timeout, Net: p.proto} + start := time.Now() + r, _, err := c.Exchange(m, p.server) + if err != nil { + return false, time.Since(start) + } + if r == nil { + return false, time.Since(start) + } + if r.Rcode != dns.RcodeSuccess { + return false, time.Since(start) + } + return true, time.Since(start) +} diff --git a/observer/probers/dns/dns_conf.go b/observer/probers/dns/dns_conf.go new file mode 100644 index 00000000000..76a7d543a5d --- /dev/null +++ b/observer/probers/dns/dns_conf.go @@ -0,0 +1,133 @@ +package probers + +import ( + "fmt" + "net" + "strconv" + "strings" + + "github.com/letsencrypt/boulder/observer/probers" + "github.com/miekg/dns" + "gopkg.in/yaml.v2" +) + +var ( + validQTypes = map[string]uint16{"A": 1, "TXT": 16, "AAAA": 28, "CAA": 257} +) + +// DNSConf is exported to receive YAML configuration +type DNSConf struct { + Proto string `yaml:"protocol"` + Server string `yaml:"server"` + Recurse bool `yaml:"recurse"` + QName string `yaml:"query_name"` + QType string `yaml:"query_type"` +} + +// UnmarshalSettings constructs a DNSConf object from YAML as bytes. +func (c DNSConf) UnmarshalSettings(settings []byte) (probers.Configurer, error) { + var conf DNSConf + err := yaml.Unmarshal(settings, &conf) + if err != nil { + return nil, err + } + return conf, nil +} + +func (c DNSConf) validateServer() error { + server := strings.Trim(strings.ToLower(c.Server), " ") + // Ensure `server` contains a port. + host, port, err := net.SplitHostPort(server) + if err != nil || port == "" { + return fmt.Errorf( + "invalid `server`, %q, could not be split: %s", c.Server, err) + } + // Ensure `server` port is valid. + portNum, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf( + "invalid `server`, %q, port must be a number", c.Server) + } + if portNum <= 0 || portNum > 65535 { + return fmt.Errorf( + "invalid `server`, %q, port number must be one in [1-65535]", c.Server) + } + // Ensure `server` is a valid FQDN or IPv4 / IPv6 address. + IPv6 := net.ParseIP(host).To16() + IPv4 := net.ParseIP(host).To4() + FQDN := dns.IsFqdn(dns.Fqdn(host)) + if IPv6 == nil && IPv4 == nil && !FQDN { + return fmt.Errorf( + "invalid `server`, %q, is not an FQDN or IPv4 / IPv6 address", c.Server) + } + return nil +} + +func (c DNSConf) validateProto() error { + validProtos := []string{"udp", "tcp"} + proto := strings.Trim(strings.ToLower(c.Proto), " ") + for _, i := range validProtos { + if proto == i { + return nil + } + } + return fmt.Errorf( + "invalid `protocol`, got: %q, expected one in: %s", c.Proto, validProtos) +} + +func (c DNSConf) validateQType() error { + validQTypes = map[string]uint16{"A": 1, "TXT": 16, "AAAA": 28, "CAA": 257} + qtype := strings.Trim(strings.ToUpper(c.QType), " ") + q := make([]string, 0, len(validQTypes)) + for i := range validQTypes { + q = append(q, i) + if qtype == i { + return nil + } + } + return fmt.Errorf( + "invalid `query_type`, got: %q, expected one in %s", c.QType, q) +} + +// MakeProber constructs a `DNSProbe` object from the contents of the +// bound `DNSConf` object. If the `DNSConf` cannot be validated, an +// error appropriate for end-user consumption is returned instead. +func (c DNSConf) MakeProber() (probers.Prober, error) { + // validate `query_name` + if !dns.IsFqdn(dns.Fqdn(c.QName)) { + return nil, fmt.Errorf( + "invalid `query_name`, %q is not an fqdn", c.QName) + } + + // validate `server` + err := c.validateServer() + if err != nil { + return nil, err + } + + // validate `protocol` + err = c.validateProto() + if err != nil { + return nil, err + } + + // validate `query_type` + err = c.validateQType() + if err != nil { + return nil, err + } + + return DNSProbe{ + proto: strings.Trim(strings.ToLower(c.Proto), " "), + recurse: c.Recurse, + qname: c.QName, + server: c.Server, + qtype: validQTypes[strings.Trim(strings.ToUpper(c.QType), " ")], + }, nil +} + +// init is called at runtime and registers `DNSConf`, a `Prober` +// `Configurer` type, as "DNS". +func init() { + probers.Register("DNS", DNSConf{}) +} diff --git a/observer/probers/dns/dns_conf_test.go b/observer/probers/dns/dns_conf_test.go new file mode 100644 index 00000000000..18c6329ba48 --- /dev/null +++ b/observer/probers/dns/dns_conf_test.go @@ -0,0 +1,195 @@ +package probers + +import ( + "reflect" + "testing" + + "github.com/letsencrypt/boulder/observer/probers" + "gopkg.in/yaml.v2" +) + +func TestDNSConf_validateServer(t *testing.T) { + type fields struct { + Server string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // ipv4 cases + {"ipv4 with port", fields{"1.1.1.1:53"}, false}, + {"ipv4 without port", fields{"1.1.1.1"}, true}, + {"ipv4 port num missing", fields{"1.1.1.1:"}, true}, + {"ipv4 string for port", fields{"1.1.1.1:foo"}, true}, + {"ipv4 port out of range high", fields{"1.1.1.1:65536"}, true}, + {"ipv4 port out of range low", fields{"1.1.1.1:0"}, true}, + + // ipv6 cases + {"ipv6 with port", fields{"[2606:4700:4700::1111]:53"}, false}, + {"ipv6 without port", fields{"[2606:4700:4700::1111]"}, true}, + {"ipv6 port num missing", fields{"[2606:4700:4700::1111]:"}, true}, + {"ipv6 string for port", fields{"[2606:4700:4700::1111]:foo"}, true}, + {"ipv6 port out of range high", fields{"[2606:4700:4700::1111]:65536"}, true}, + {"ipv6 port out of range low", fields{"[2606:4700:4700::1111]:0"}, true}, + + // hostname cases + {"hostname with port", fields{"foo:53"}, false}, + {"hostname without port", fields{"foo"}, true}, + {"hostname port num missing", fields{"foo:"}, true}, + {"hostname string for port", fields{"foo:bar"}, true}, + {"hostname port out of range high", fields{"foo:65536"}, true}, + {"hostname port out of range low", fields{"foo:0"}, true}, + + // fqdn cases + {"fqdn with port", fields{"bar.foo.baz:53"}, false}, + {"fqdn without port", fields{"bar.foo.baz"}, true}, + {"fqdn port num missing", fields{"bar.foo.baz:"}, true}, + {"fqdn string for port", fields{"bar.foo.baz:bar"}, true}, + {"fqdn port out of range high", fields{"bar.foo.baz:65536"}, true}, + {"fqdn port out of range low", fields{"bar.foo.baz:0"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := DNSConf{ + Server: tt.fields.Server, + } + if err := c.validateServer(); (err != nil) != tt.wantErr { + t.Errorf("DNSConf.validateServer() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestDNSConf_validateQType(t *testing.T) { + type fields struct { + QType string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // valid + {"A", fields{"A"}, false}, + {"AAAA", fields{"AAAA"}, false}, + {"TXT", fields{"TXT"}, false}, + // invalid + {"AAA", fields{"AAA"}, true}, + {"TXTT", fields{"TXTT"}, true}, + {"D", fields{"D"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := DNSConf{ + QType: tt.fields.QType, + } + if err := c.validateQType(); (err != nil) != tt.wantErr { + t.Errorf("DNSConf.validateQType() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestDNSConf_validateProto(t *testing.T) { + type fields struct { + Proto string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // valid + {"tcp", fields{"tcp"}, false}, + {"udp", fields{"udp"}, false}, + // invalid + {"foo", fields{"foo"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := DNSConf{ + Proto: tt.fields.Proto, + } + if err := c.validateProto(); (err != nil) != tt.wantErr { + t.Errorf("DNSConf.validateProto() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestDNSConf_MakeProber(t *testing.T) { + type fields struct { + Proto string + Server string + Recurse bool + QName string + QType string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // valid + {"valid", fields{"udp", "1.1.1.1:53", true, "google.com", "A"}, false}, + // invalid + {"bad proto", fields{"can with string", "1.1.1.1:53", true, "google.com", "A"}, true}, + {"bad server", fields{"udp", "1.1.1.1:9000000", true, "google.com", "A"}, true}, + {"bad qtype", fields{"udp", "1.1.1.1:9000000", true, "google.com", "BAZ"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := DNSConf{ + Proto: tt.fields.Proto, + Server: tt.fields.Server, + Recurse: tt.fields.Recurse, + QName: tt.fields.QName, + QType: tt.fields.QType, + } + if _, err := c.MakeProber(); (err != nil) != tt.wantErr { + t.Errorf("HTTPConf.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestDNSConf_UnmarshalSettings(t *testing.T) { + type fields struct { + protocol interface{} + server interface{} + recurse interface{} + query_name interface{} + query_type interface{} + } + tests := []struct { + name string + fields fields + want probers.Configurer + wantErr bool + }{ + {"valid", fields{"udp", "1.1.1.1:53", true, "google.com", "A"}, DNSConf{"udp", "1.1.1.1:53", true, "google.com", "A"}, false}, + {"invalid", fields{42, 42, 42, 42, 42}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + settings := probers.Settings{ + "protocol": tt.fields.protocol, + "server": tt.fields.server, + "recurse": tt.fields.recurse, + "query_name": tt.fields.query_name, + "query_type": tt.fields.query_type, + } + settingsBytes, _ := yaml.Marshal(settings) + c := DNSConf{} + got, err := c.UnmarshalSettings(settingsBytes) + if (err != nil) != tt.wantErr { + t.Errorf("DNSConf.UnmarshalSettings() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DNSConf.UnmarshalSettings() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/observer/probers/http/http.go b/observer/probers/http/http.go new file mode 100644 index 00000000000..92d6b534026 --- /dev/null +++ b/observer/probers/http/http.go @@ -0,0 +1,47 @@ +package probers + +import ( + "fmt" + "net/http" + "time" +) + +// HTTPProbe is the exported 'Prober' object for monitors configured to +// perform HTTP requests. +type HTTPProbe struct { + url string + rcodes []int +} + +// Name returns a string that uniquely identifies the monitor. +func (p HTTPProbe) Name() string { + return fmt.Sprintf("%s-%d", p.url, p.rcodes) +} + +// Kind returns a name that uniquely identifies the `Kind` of `Prober`. +func (p HTTPProbe) Kind() string { + return "HTTP" +} + +// isExpected ensures that the received HTTP response code matches one +// that's expected. +func (p HTTPProbe) isExpected(received int) bool { + for _, c := range p.rcodes { + if received == c { + return true + } + } + return false +} + +// Probe performs the configured HTTP request. +func (p HTTPProbe) Probe(timeout time.Duration) (bool, time.Duration) { + client := http.Client{Timeout: timeout} + start := time.Now() + // TODO(@beautifulentropy): add support for more than HTTP GET + resp, err := client.Get(p.url) + if err != nil { + return false, time.Since(start) + } + return p.isExpected(resp.StatusCode), time.Since(start) +} diff --git a/observer/probers/http/http_conf.go b/observer/probers/http/http_conf.go new file mode 100644 index 00000000000..ce4ad92fad1 --- /dev/null +++ b/observer/probers/http/http_conf.go @@ -0,0 +1,78 @@ +package probers + +import ( + "fmt" + "net/url" + + "github.com/letsencrypt/boulder/observer/probers" + "gopkg.in/yaml.v2" +) + +// HTTPConf is exported to receive YAML configuration. +type HTTPConf struct { + URL string `yaml:"url"` + RCodes []int `yaml:"rcodes"` +} + +// UnmarshalSettings takes YAML as bytes and unmarshals it to the to an +// HTTPConf object. +func (c HTTPConf) UnmarshalSettings(settings []byte) (probers.Configurer, error) { + var conf HTTPConf + err := yaml.Unmarshal(settings, &conf) + if err != nil { + return nil, err + } + return conf, nil +} + +func (c HTTPConf) validateURL() error { + url, err := url.Parse(c.URL) + if err != nil { + return fmt.Errorf( + "invalid 'url', got: %q, expected a valid url", c.URL) + } + if url.Scheme == "" { + return fmt.Errorf( + "invalid 'url', got: %q, missing scheme", c.URL) + } + return nil +} + +func (c HTTPConf) validateRCodes() error { + if len(c.RCodes) == 0 { + return fmt.Errorf( + "invalid 'rcodes', got: %q, please specify at least one", c.RCodes) + } + for _, c := range c.RCodes { + // ensure rcode entry is in range 100-599 + if c < 100 || c > 599 { + return fmt.Errorf( + "'rcodes' contains an invalid HTTP response code, '%d'", c) + } + } + return nil +} + +// MakeProber constructs a `HTTPProbe` object from the contents of the +// bound `HTTPConf` object. If the `HTTPConf` cannot be validated, an +// error appropriate for end-user consumption is returned instead. +func (c HTTPConf) MakeProber() (probers.Prober, error) { + // validate `url` + err := c.validateURL() + if err != nil { + return nil, err + } + + // validate `rcodes` + err = c.validateRCodes() + if err != nil { + return nil, err + } + return HTTPProbe{c.URL, c.RCodes}, nil +} + +// init is called at runtime and registers `HTTPConf`, a `Prober` +// `Configurer` type, as "HTTP". +func init() { + probers.Register("HTTP", HTTPConf{}) +} diff --git a/observer/probers/http/http_conf_test.go b/observer/probers/http/http_conf_test.go new file mode 100644 index 00000000000..c4789e7b36d --- /dev/null +++ b/observer/probers/http/http_conf_test.go @@ -0,0 +1,76 @@ +package probers + +import ( + "reflect" + "testing" + + "github.com/letsencrypt/boulder/observer/probers" + "gopkg.in/yaml.v2" +) + +func TestHTTPConf_MakeProber(t *testing.T) { + type fields struct { + URL string + RCodes []int + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // valid + {"valid fqdn valid rcode", fields{"http://example.com", []int{200}}, false}, + {"valid hostname valid rcode", fields{"example", []int{200}}, true}, + // invalid + {"valid fqdn no rcode", fields{"http://example.com", nil}, true}, + {"valid fqdn invalid rcode", fields{"http://example.com", []int{1000}}, true}, + {"valid fqdn 1 invalid rcode", fields{"http://example.com", []int{200, 1000}}, true}, + {"bad fqdn good rcode", fields{":::::", []int{200}}, true}, + {"missing scheme", fields{"example.com", []int{200}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := HTTPConf{ + URL: tt.fields.URL, + RCodes: tt.fields.RCodes, + } + if _, err := c.MakeProber(); (err != nil) != tt.wantErr { + t.Errorf("HTTPConf.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestHTTPConf_UnmarshalSettings(t *testing.T) { + type fields struct { + url interface{} + rcodes interface{} + } + tests := []struct { + name string + fields fields + want probers.Configurer + wantErr bool + }{ + {"valid", fields{"google.com", []int{200}}, HTTPConf{"google.com", []int{200}}, false}, + {"invalid", fields{42, 42}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + settings := probers.Settings{ + "url": tt.fields.url, + "rcodes": tt.fields.rcodes, + } + settingsBytes, _ := yaml.Marshal(settings) + c := HTTPConf{} + got, err := c.UnmarshalSettings(settingsBytes) + if (err != nil) != tt.wantErr { + t.Errorf("DNSConf.UnmarshalSettings() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DNSConf.UnmarshalSettings() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/observer/probers/mock/mock_conf.go b/observer/probers/mock/mock_conf.go new file mode 100644 index 00000000000..7840a1949fb --- /dev/null +++ b/observer/probers/mock/mock_conf.go @@ -0,0 +1,38 @@ +package probers + +import ( + "errors" + + "github.com/letsencrypt/boulder/cmd" + "github.com/letsencrypt/boulder/observer/probers" + "gopkg.in/yaml.v2" +) + +type MockConfigurer struct { + Valid bool `yaml:"valid"` + ErrMsg string `yaml:"errmsg"` + PName string `yaml:"pname"` + PKind string `yaml:"pkind"` + PTook cmd.ConfigDuration `yaml:"ptook"` + PSuccess bool `yaml:"psuccess"` +} + +func (c MockConfigurer) UnmarshalSettings(settings []byte) (probers.Configurer, error) { + var conf MockConfigurer + err := yaml.Unmarshal(settings, &conf) + if err != nil { + return nil, err + } + return conf, nil +} + +func (c MockConfigurer) MakeProber() (probers.Prober, error) { + if !c.Valid { + return nil, errors.New("could not be validated") + } + return MockProber{c.PName, c.PKind, c.PTook, c.PSuccess}, nil +} + +func init() { + probers.Register("MockConf", MockConfigurer{}) +} diff --git a/observer/probers/mock/mock_prober.go b/observer/probers/mock/mock_prober.go new file mode 100644 index 00000000000..446dc3a527f --- /dev/null +++ b/observer/probers/mock/mock_prober.go @@ -0,0 +1,26 @@ +package probers + +import ( + "time" + + "github.com/letsencrypt/boulder/cmd" +) + +type MockProber struct { + name string + kind string + took cmd.ConfigDuration + success bool +} + +func (p MockProber) Name() string { + return p.name +} + +func (p MockProber) Kind() string { + return p.kind +} + +func (p MockProber) Probe(timeout time.Duration) (bool, time.Duration) { + return p.success, p.took.Duration +} diff --git a/observer/probers/prober.go b/observer/probers/prober.go new file mode 100644 index 00000000000..76c6aafb35b --- /dev/null +++ b/observer/probers/prober.go @@ -0,0 +1,77 @@ +package probers + +import ( + "fmt" + "strings" + "time" + + "github.com/letsencrypt/boulder/cmd" +) + +var ( + // Registry is the global mapping of all `Configurer` types. Types + // are added to this mapping on import by including a call to + // `Register` in their `init` function. + Registry = make(map[string]Configurer) +) + +// Prober is the interface for `Prober` types. +type Prober interface { + // Name returns a name that uniquely identifies the monitor that + // configured this `Prober`. + Name() string + + // Kind returns a name that uniquely identifies the `Kind` of + // `Prober`. + Kind() string + + // Probe attempts the configured request or query, Each `Prober` + // must treat the duration passed to it as a timeout. + Probe(time.Duration) (bool, time.Duration) +} + +// Configurer is the interface for `Configurer` types. +type Configurer interface { + // UnmarshalSettings unmarshals YAML as bytes to a `Configurer` + // object. + UnmarshalSettings([]byte) (Configurer, error) + + // MakeProber constructs a `Prober` object from the contents of the + // bound `Configurer` object. If the `Configurer` cannot be + // validated, an error appropriate for end-user consumption is + // returned instead. + MakeProber() (Prober, error) +} + +// Settings is exported as a temporary receiver for the `settings` field +// of `MonConf`. `Settings` is always marshaled back to bytes and then +// unmarshalled into the `Configurer` specified by the `Kind` field of +// the `MonConf`. +type Settings map[string]interface{} + +// GetConfigurer returns the probe configurer specified by name from +// `Registry`. +func GetConfigurer(kind string) (Configurer, error) { + // normalize + name := strings.Trim(strings.ToLower(kind), " ") + // check if exists + if _, ok := Registry[name]; ok { + return Registry[name], nil + } + return nil, fmt.Errorf("%s is not a registered Prober type", kind) +} + +// Register is called by the `init` function of every `Configurer` to +// add the caller to the global `Registry` map. If the caller attempts +// to add a `Configurer` to the registry using the same name as a prior +// `Configurer` Observer will exit after logging an error. +func Register(kind string, c Configurer) { + // normalize + name := strings.Trim(strings.ToLower(kind), " ") + // check for name collision + if _, exists := Registry[name]; exists { + cmd.Fail(fmt.Sprintf( + "problem registering configurer %s: name collision", kind)) + } + Registry[name] = c +} diff --git a/test/config-next/observer.yml b/test/config-next/observer.yml new file mode 100644 index 00000000000..70a8e9fac0d --- /dev/null +++ b/test/config-next/observer.yml @@ -0,0 +1,91 @@ +--- +debugaddr: :8040 +buckets: [.001, .002, .005, .01, .02, .05, .1, .2, .5, 1, 2, 5, 10] +syslog: + stdoutlevel: 6 + sysloglevel: 6 +monitors: + - + period: 5s + kind: DNS + settings: + protocol: udp + server: owen.ns.cloudflare.com:53 + recurse: false + query_name: letsencrypt.org + query_type: A + - + period: 5s + kind: DNS + settings: + protocol: udp + server: 1.1.1.1:53 + recurse: true + query_name: google.com + query_type: A + - + period: 10s + kind: DNS + settings: + protocol: tcp + server: 8.8.8.8:53 + recurse: true + query_name: google.com + query_type: A + - + period: 2s + kind: HTTP + settings: + url: https://letsencrypt.org + rcodes: [200] + - + period: 5s + kind: DNS + settings: + protocol: tcp + server: owen.ns.cloudflare.com:53 + recurse: false + query_name: letsencrypt.org + query_type: A + - + period: 5s + kind: DNS + settings: + protocol: tcp + server: 1.1.1.1:53 + recurse: true + query_name: google.com + query_type: A + - + period: 10s + kind: DNS + settings: + protocol: udp + server: 8.8.8.8:53 + recurse: true + query_name: google.com + query_type: A + - + period: 5s + kind: DNS + settings: + protocol: tcp + server: "[2606:4700:4700::1111]:53" + recurse: true + query_name: google.com + query_type: A + - + period: 5s + kind: DNS + settings: + protocol: udp + server: "[2606:4700:4700::1111]:53" + recurse: true + query_name: google.com + query_type: A + - + period: 2s + kind: HTTP + settings: + url: http://letsencrypt.org/foo + rcodes: [200, 404] diff --git a/test/prometheus/prometheus.yml b/test/prometheus/prometheus.yml index b4e81b243a9..488d0aa5bae 100644 --- a/test/prometheus/prometheus.yml +++ b/test/prometheus/prometheus.yml @@ -16,3 +16,4 @@ scrape_configs: - boulder:8008 - boulder:8009 - boulder:8010 + - boulder:8040