Skip to content
This repository has been archived by the owner on Feb 6, 2025. It is now read-only.

fix: missing metrics issue #219

Merged
merged 7 commits into from
Jan 16, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -553,8 +553,8 @@ zstd = "0.13"
metrics = "0.24.0"
metrics-derive = "0.1"
metrics-exporter-prometheus = { version = "0.16.0", default-features = false }
metrics-process = { version = "2.3.1"}
metrics-util = { default-features = false, version = "0.18.0" }
metrics-process = "2.1.0"
unclezoro marked this conversation as resolved.
Show resolved Hide resolved
metrics-util = { default-features = false, version = "0.19.0" }

# proc-macros
proc-macro2 = "1.0"
Expand Down
6 changes: 5 additions & 1 deletion crates/bsc/evm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@ reth-errors.workspace = true
reth-evm.workspace = true
reth-primitives.workspace = true
reth-prune-types.workspace = true
reth-revm.workspace = true
reth-provider.workspace = true
reth-revm = { workspace = true, features = ["std"] }
reth-trie-db.workspace = true

# revm with required ethereum, bsc features
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] }

# bsc
reth-bsc-consensus.workspace = true
Expand Down
4 changes: 3 additions & 1 deletion crates/bsc/node/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ reth-primitives.workspace = true
reth-config.workspace = true
reth-rpc.workspace = true
reth-node-api.workspace = true
reth-trie-db.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }

# bsc-reth
reth-bsc-chainspec.workspace = true
Expand All @@ -36,6 +37,7 @@ reth-bsc-payload-builder.workspace = true
reth-bsc-engine.workspace = true

# misc
clap = { workspace = true, features = ["derive", "env"] }
eyre.workspace = true

[dev-dependencies]
Expand Down
3 changes: 3 additions & 0 deletions crates/engine/tree/src/tree/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2347,6 +2347,9 @@ where
debug!(target: "engine::tree", pending = ?executed.block().num_hash() ,"updating pending block");
// if the parent is the canonical head, we can insert the block as the pending block
self.canonical_in_memory_state.set_pending_block(executed.clone());
// update metrics
debug!(target: "engine::tree", block_number, "updating canonical chain height metric");
self.metrics.tree.canonical_chain_height.set(block_number as f64);
}

self.state.tree_state.insert_executed(executed);
Expand Down
15 changes: 9 additions & 6 deletions crates/node/builder/src/launch/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

use std::{sync::Arc, thread::available_parallelism};

use crate::{
components::{NodeComponents, NodeComponentsBuilder},
hooks::OnComponentInitializedHook,
BuilderContext, NodeAdapter,
};
use alloy_primitives::{BlockNumber, B256};
use eyre::{Context, OptionExt};
use rayon::ThreadPoolBuilder;
Expand Down Expand Up @@ -34,6 +39,7 @@ use reth_node_core::{
use reth_node_metrics::{
chain::ChainSpecInfo,
hooks::Hooks,
recorder::install_prometheus_recorder,
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
Expand All @@ -57,12 +63,6 @@ use tokio::sync::{
oneshot, watch,
};

use crate::{
components::{NodeComponents, NodeComponentsBuilder},
hooks::OnComponentInitializedHook,
BuilderContext, NodeAdapter,
};

/// Allows to set a tree viewer for a configured blockchain provider.
// TODO: remove this helper trait once the engine revamp is done, the new
// blockchain provider won't require a TreeViewer.
Expand Down Expand Up @@ -511,6 +511,9 @@ where

/// Starts the prometheus endpoint.
pub async fn start_prometheus_endpoint(&self) -> eyre::Result<()> {
// ensure recorder runs upkeep periodically
install_prometheus_recorder().spawn_upkeep();

let listen_addr = self.node_config().metrics;
if let Some(addr) = listen_addr {
info!(target: "reth::cli", "Starting metrics endpoint at {}", addr);
Expand Down
69 changes: 61 additions & 8 deletions crates/node/metrics/src/recorder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,78 @@
use eyre::WrapErr;
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
use metrics_util::layers::{PrefixLayer, Stack};
use std::sync::LazyLock;
use std::sync::{atomic::AtomicBool, LazyLock};

/// Installs the Prometheus recorder as the global recorder.
pub fn install_prometheus_recorder() -> &'static PrometheusHandle {
///
/// Note: This must be installed before any metrics are `described`.
///
/// Caution: This only configures the global recorder and does not spawn the exporter.
/// Callers must run [`PrometheusRecorder::spawn_upkeep`] manually.
pub fn install_prometheus_recorder() -> &'static PrometheusRecorder {
&PROMETHEUS_RECORDER_HANDLE
}

/// The default Prometheus recorder handle. We use a global static to ensure that it is only
/// installed once.
static PROMETHEUS_RECORDER_HANDLE: LazyLock<PrometheusHandle> =
static PROMETHEUS_RECORDER_HANDLE: LazyLock<PrometheusRecorder> =
LazyLock::new(|| PrometheusRecorder::install().unwrap());

/// Prometheus recorder installer
/// A handle to the Prometheus recorder.
///
/// This is intended to be used as the global recorder.
/// Callers must ensure that [`PrometheusRecorder::spawn_upkeep`] is called once.
#[derive(Debug)]
pub struct PrometheusRecorder;
pub struct PrometheusRecorder {
handle: PrometheusHandle,
upkeep: AtomicBool,
}

impl PrometheusRecorder {
const fn new(handle: PrometheusHandle) -> Self {
Self { handle, upkeep: AtomicBool::new(false) }
}

/// Returns a reference to the [`PrometheusHandle`].
pub const fn handle(&self) -> &PrometheusHandle {
&self.handle
}

/// Spawns the upkeep task if there hasn't been one spawned already.
///
/// ## Panics
///
/// This method must be called from within an existing Tokio runtime or it will panic.
///
/// See also [`PrometheusHandle::run_upkeep`]
pub fn spawn_upkeep(&self) {
unclezoro marked this conversation as resolved.
Show resolved Hide resolved
if self
.upkeep
.compare_exchange(
false,
true,
std::sync::atomic::Ordering::SeqCst,
std::sync::atomic::Ordering::Acquire,
)
.is_err()
{
return;
}

let handle = self.handle.clone();
tokio::spawn(async move {
loop {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
handle.run_upkeep();
}
});
}

/// Installs Prometheus as the metrics recorder.
pub fn install() -> eyre::Result<PrometheusHandle> {
///
/// Caution: This only configures the global recorder and does not spawn the exporter.
/// Callers must run [`Self::spawn_upkeep`] manually.
pub fn install() -> eyre::Result<Self> {
let recorder = PrometheusBuilder::new().build_recorder();
let handle = recorder.handle();

Expand All @@ -31,7 +84,7 @@ impl PrometheusRecorder {
.install()
.wrap_err("Couldn't set metrics recorder.")?;

Ok(handle)
Ok(Self::new(handle))
}
}

Expand All @@ -52,7 +105,7 @@ mod tests {
process.describe();
process.collect();

let metrics = PROMETHEUS_RECORDER_HANDLE.render();
let metrics = PROMETHEUS_RECORDER_HANDLE.handle.render();
assert!(metrics.contains("process_cpu_seconds_total"), "{metrics:?}");
}
}
2 changes: 1 addition & 1 deletion crates/node/metrics/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ impl MetricServer {
let hook = hook.clone();
let service = tower::service_fn(move |_| {
(hook)();
let metrics = handle.render();
let metrics = handle.handle().render();
let mut response = Response::new(metrics);
response
.headers_mut()
Expand Down
Loading