Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat (telemetry) Handling synchronization when registering prometheus metrics #113

Merged
merged 11 commits into from
Jul 16, 2024
Merged

feat (telemetry) Handling synchronization when registering prometheus metrics #113

Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d8d4817
Handling synchronization when registering prometheus metrics
Jul 15, 2024
3343b80
use sync.Map and lint
Jul 15, 2024
0711ac5
fix linting
Jul 15, 2024
146447a
use RWMap
Jul 15, 2024
81f5381
Using rwmap to cache metrics
Jul 15, 2024
06533b9
refactor
Jul 16, 2024
89f2006
Use lock for registration
Jul 16, 2024
cbda7a7
noop
Jul 16, 2024
18e693c
renaming
Jul 16, 2024
b5addbe
renaming
Jul 16, 2024
4b8a10a
renaming
Jul 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 54 additions & 31 deletions telemetry/prometheus/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
"unicode"

"github.com/prometheus/client_golang/prometheus"

"github.com/berachain/offchain-sdk/tools/rwstore"
)

const (
Expand All @@ -28,9 +30,9 @@
cfg *Config

gaugeVecs map[string]*prometheus.GaugeVec
counterVecs map[string]*prometheus.CounterVec
counterVecs *rwstore.RWMap[string, *prometheus.CounterVec]
gordonbear marked this conversation as resolved.
Show resolved Hide resolved
histogramVecs map[string]*prometheus.HistogramVec
summaryVecs map[string]*prometheus.SummaryVec
summaryVecs *rwstore.RWMap[string, *prometheus.SummaryVec]
}

// NewMetrics initializes a new instance of Prometheus metrics.
Expand All @@ -46,9 +48,9 @@
}

p.gaugeVecs = make(map[string]*prometheus.GaugeVec, initialVecCapacity)
p.counterVecs = make(map[string]*prometheus.CounterVec, initialVecCapacity)
p.counterVecs = rwstore.NewRWMap[string, *prometheus.CounterVec]()
p.histogramVecs = make(map[string]*prometheus.HistogramVec, initialVecCapacity)
p.summaryVecs = make(map[string]*prometheus.SummaryVec, initialVecCapacity)
p.summaryVecs = rwstore.NewRWMap[string, *prometheus.SummaryVec]()
return p, nil
}

Expand Down Expand Up @@ -130,7 +132,7 @@

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
counterVec, exists := p.counterVecs[name]
counterVec, exists := p.counterVecs.Get(name)
if !exists {
counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: name,
Expand All @@ -139,7 +141,7 @@
Help: name + " counter",
}, labels)
prometheus.MustRegister(counterVec)
p.counterVecs[name] = counterVec
p.counterVecs.Set(name, counterVec)
}
counterVec.WithLabelValues(labelValues...).Add(float64(value))
}
Expand All @@ -152,16 +154,26 @@

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
counterVec, exists := p.counterVecs[name]
if !exists {
counterVec = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " counter",
}, labels)
prometheus.MustRegister(counterVec)
p.counterVecs[name] = counterVec
if counterVec, exists := p.counterVecs.Get(name); exists {
counterVec.WithLabelValues(labelValues...).Inc()
}

counterVec := prometheus.NewCounterVec(prometheus.CounterOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " counter",
}, labels)
if err := prometheus.Register(counterVec); err != nil {
gordonbear marked this conversation as resolved.
Show resolved Hide resolved
// In case of concurrent registration, get the one that has already registered
if existingCollector, ok := err.(prometheus.AlreadyRegisteredError); ok {

Check failure on line 169 in telemetry/prometheus/metrics.go

View workflow job for this annotation

GitHub Actions / ci (lint, ubuntu-latest, 1.21.0) 

type assertion on error will fail on wrapped errors. Use errors.As to check for specific errors (errorlint)
counterVec = existingCollector.ExistingCollector.(*prometheus.CounterVec)

Check failure on line 170 in telemetry/prometheus/metrics.go

View workflow job for this annotation

GitHub Actions / ci (lint, ubuntu-latest, 1.21.0) 

Error return value is not checked (errcheck)
} else {
// Otherwise we should panic to fail fast
panic(err)
}
} else {
p.counterVecs.Set(name, counterVec)
}
counterVec.WithLabelValues(labelValues...).Inc()
}
Expand Down Expand Up @@ -206,23 +218,34 @@

name = forceValidName(name)
labels, labelValues := parseTagsToLabelPairs(tags)
summaryVec, exists := p.summaryVecs[name]
if !exists {
summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " timing summary",
Objectives: map[float64]float64{
quantile50: errorMargin50,
quantile90: errorMargin90,
quantile99: errorMargin99,
},
}, labels)
prometheus.MustRegister(summaryVec)
p.summaryVecs[name] = summaryVec
if summaryVec, exists := p.summaryVecs.Get(name); exists {
// Convert time.Duration to seconds since Prometheus prefers base units
// see https://prometheus.io/docs/practices/naming/#base-units
summaryVec.WithLabelValues(labelValues...).Observe(value.Seconds())
}

summaryVec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: name,
Namespace: p.cfg.Namespace,
Subsystem: p.cfg.Subsystem,
Help: name + " timing summary",
Objectives: map[float64]float64{
quantile50: errorMargin50,
quantile90: errorMargin90,
quantile99: errorMargin99,
},
}, labels)
if err := prometheus.Register(summaryVec); err != nil {
// In case of concurrent registration, get the one that has already registered
if existingCollector, ok := err.(prometheus.AlreadyRegisteredError); ok {

Check failure on line 240 in telemetry/prometheus/metrics.go

View workflow job for this annotation

GitHub Actions / ci (lint, ubuntu-latest, 1.21.0) 

type assertion on error will fail on wrapped errors. Use errors.As to check for specific errors (errorlint)
summaryVec = existingCollector.ExistingCollector.(*prometheus.SummaryVec)

Check failure on line 241 in telemetry/prometheus/metrics.go

View workflow job for this annotation

GitHub Actions / ci (lint, ubuntu-latest, 1.21.0) 

Error return value is not checked (errcheck)
} else {
// Otherwise we should panic to fail fast
panic(err)
}
} else {
p.summaryVecs.Set(name, summaryVec)
}
// Convert time.Duration to seconds since Prometheus prefers base units
// see https://prometheus.io/docs/practices/naming/#base-units
summaryVec.WithLabelValues(labelValues...).Observe(value.Seconds())
Expand Down
Loading