From f9614983227920612859781e239cac15c87f1aac Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Fri, 24 Nov 2023 12:51:54 +0200 Subject: [PATCH 1/8] Attempt to initialize OS FD limits during the kaspad startup sequence. --- Cargo.lock | 1 + kaspad/Cargo.toml | 1 + kaspad/src/main.rs | 28 ++++++++++++++++++++++++++++ utils/src/fd_budget.rs | 11 +++++++++++ 4 files changed, 41 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index fbbfe662e8..32a68b68b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3097,6 +3097,7 @@ dependencies = [ "num_cpus", "rand 0.8.5", "rayon", + "rlimit", "tempfile", "thiserror", "tokio", diff --git a/kaspad/Cargo.toml b/kaspad/Cargo.toml index d15517eb9a..bef0ca299e 100644 --- a/kaspad/Cargo.toml +++ b/kaspad/Cargo.toml @@ -43,6 +43,7 @@ log.workspace = true num_cpus.workspace = true rand.workspace = true rayon.workspace = true +rlimit.workspace = true tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] } diff --git a/kaspad/src/main.rs b/kaspad/src/main.rs index e7cdc94c09..ffaf0478d7 100644 --- a/kaspad/src/main.rs +++ b/kaspad/src/main.rs @@ -12,11 +12,39 @@ use kaspad_lib::{args::parse_args, daemon::create_core}; #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; +/// FD limits that kaspad tries to configure on startup +const DEFAULT_SOFT_FD_LIMIT: u64 = 8 * 1024; +const DEFAULT_HARD_FD_LIMIT: u64 = u64::MAX; +/// The warning to the user will be issued only +/// if the soft FD limit is below this value. +/// (We desire 8k but deem 1k acceptable) +const ALLOWED_SOFT_FD_LIMIT: u64 = 1024; + pub fn main() { #[cfg(feature = "heap")] let _profiler = dhat::Profiler::builder().file_name("kaspad-heap.json").build(); + // let args = parse_args(); + + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + match fd_budget::ensure_os_limits(DEFAULT_SOFT_FD_LIMIT, DEFAULT_HARD_FD_LIMIT) { + Ok((soft, hard)) => { + if soft < ALLOWED_SOFT_FD_LIMIT { + println!("Current OS file descriptor (FD) limits are - soft: {soft}, hard: {hard}"); + println!("The kaspad node requires at least {DEFAULT_SOFT_FD_LIMIT} file descriptors to operate properly."); + println!("Please increase the limits using the following command:"); + println!("ulimit -n {DEFAULT_SOFT_FD_LIMIT}"); + } + } + Err(err) => { + println!("Unable to initialize the necessary OS file descriptor (FD) limits: {}", err); + println!("The kaspad node requires at least {DEFAULT_SOFT_FD_LIMIT} file descriptors to operate properly."); + } + } + } let args = parse_args(); + let fd_total_budget = fd_budget::limit() - args.rpc_max_clients as i32 - args.inbound_limit as i32 - args.outbound_target as i32; let (core, _) = create_core(args, fd_total_budget); diff --git a/utils/src/fd_budget.rs b/utils/src/fd_budget.rs index b853ffb281..55d025f865 100644 --- a/utils/src/fd_budget.rs +++ b/utils/src/fd_budget.rs @@ -50,6 +50,17 @@ pub fn acquire_guard(value: i32) -> Result { } } +#[cfg(not(any(target_arch = "wasm32", target_os = "windows")))] +pub fn ensure_os_limits(user_soft_limit: u64, user_hard_limit: u64) -> std::io::Result<(u64, u64)> { + let (soft, hard) = rlimit::Resource::NOFILE.get()?; + if soft < user_soft_limit || hard < user_hard_limit { + rlimit::Resource::NOFILE.set(user_soft_limit, user_hard_limit)?; + Ok(rlimit::Resource::NOFILE.get()?) + } else { + Ok((soft, hard)) + } +} + pub fn limit() -> i32 { cfg_if::cfg_if! { if #[cfg(test)] { From 6227676fdad66adb9c8e7348fbe5d1a8b3b1963c Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Fri, 24 Nov 2023 12:52:44 +0200 Subject: [PATCH 2/8] cleanup --- Cargo.lock | 1 - kaspad/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32a68b68b2..fbbfe662e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3097,7 +3097,6 @@ dependencies = [ "num_cpus", "rand 0.8.5", "rayon", - "rlimit", "tempfile", "thiserror", "tokio", diff --git a/kaspad/Cargo.toml b/kaspad/Cargo.toml index bef0ca299e..d15517eb9a 100644 --- a/kaspad/Cargo.toml +++ b/kaspad/Cargo.toml @@ -43,7 +43,6 @@ log.workspace = true num_cpus.workspace = true rand.workspace = true rayon.workspace = true -rlimit.workspace = true tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] } From cbd1f82d44d4caba73a3dd3aa0ccc3b1e8c37469 Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Fri, 24 Nov 2023 17:10:41 +0200 Subject: [PATCH 3/8] Refactor to use setmaxstdio on windows, remove hard limit and increase DEFAULT and DESIRED limit constants. --- kaspad/src/main.rs | 37 ++++++++++++++++--------------------- utils/src/fd_budget.rs | 18 +++++++++--------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/kaspad/src/main.rs b/kaspad/src/main.rs index ffaf0478d7..0695cbff98 100644 --- a/kaspad/src/main.rs +++ b/kaspad/src/main.rs @@ -13,37 +13,32 @@ use kaspad_lib::{args::parse_args, daemon::create_core}; static ALLOC: dhat::Alloc = dhat::Alloc; /// FD limits that kaspad tries to configure on startup -const DEFAULT_SOFT_FD_LIMIT: u64 = 8 * 1024; -const DEFAULT_HARD_FD_LIMIT: u64 = u64::MAX; +const DEFAULT_SOFT_FD_LIMIT: u64 = 16 * 1024; /// The warning to the user will be issued only /// if the soft FD limit is below this value. -/// (We desire 8k but deem 1k acceptable) -const ALLOWED_SOFT_FD_LIMIT: u64 = 1024; +/// (We desire 16k but deem 4k acceptable) +const DESIRED_SOFT_FD_LIMIT: u64 = 4096; pub fn main() { #[cfg(feature = "heap")] let _profiler = dhat::Profiler::builder().file_name("kaspad-heap.json").build(); - // let args = parse_args(); - - #[cfg(any(target_os = "macos", target_os = "linux"))] - { - match fd_budget::ensure_os_limits(DEFAULT_SOFT_FD_LIMIT, DEFAULT_HARD_FD_LIMIT) { - Ok((soft, hard)) => { - if soft < ALLOWED_SOFT_FD_LIMIT { - println!("Current OS file descriptor (FD) limits are - soft: {soft}, hard: {hard}"); - println!("The kaspad node requires at least {DEFAULT_SOFT_FD_LIMIT} file descriptors to operate properly."); - println!("Please increase the limits using the following command:"); - println!("ulimit -n {DEFAULT_SOFT_FD_LIMIT}"); - } - } - Err(err) => { - println!("Unable to initialize the necessary OS file descriptor (FD) limits: {}", err); - println!("The kaspad node requires at least {DEFAULT_SOFT_FD_LIMIT} file descriptors to operate properly."); + let args = parse_args(); + + match fd_budget::ensure_os_limits(DEFAULT_SOFT_FD_LIMIT) { + Ok(limit) => { + if limit < DESIRED_SOFT_FD_LIMIT { + println!("Current OS file descriptor limit (soft FD limit) is set to {limit}"); + println!("The kaspad node requires a setting of at least {DEFAULT_SOFT_FD_LIMIT} to operate properly."); + println!("Please increase the limits using the following command:"); + println!("ulimit -n {DEFAULT_SOFT_FD_LIMIT}"); } } + Err(err) => { + println!("Unable to initialize the necessary OS file descriptor limit (soft FD limit) to: {}", err); + println!("The kaspad node requires a setting of at least {DEFAULT_SOFT_FD_LIMIT} to operate properly."); + } } - let args = parse_args(); let fd_total_budget = fd_budget::limit() - args.rpc_max_clients as i32 - args.inbound_limit as i32 - args.outbound_target as i32; let (core, _) = create_core(args, fd_total_budget); diff --git a/utils/src/fd_budget.rs b/utils/src/fd_budget.rs index 55d025f865..38a436ce5c 100644 --- a/utils/src/fd_budget.rs +++ b/utils/src/fd_budget.rs @@ -50,14 +50,14 @@ pub fn acquire_guard(value: i32) -> Result { } } -#[cfg(not(any(target_arch = "wasm32", target_os = "windows")))] -pub fn ensure_os_limits(user_soft_limit: u64, user_hard_limit: u64) -> std::io::Result<(u64, u64)> { - let (soft, hard) = rlimit::Resource::NOFILE.get()?; - if soft < user_soft_limit || hard < user_hard_limit { - rlimit::Resource::NOFILE.set(user_soft_limit, user_hard_limit)?; - Ok(rlimit::Resource::NOFILE.get()?) - } else { - Ok((soft, hard)) +#[cfg(not(target_arch = "wasm32"))] +pub fn ensure_os_limits(limit: u64) -> std::io::Result { + cfg_if::cfg_if! { + if #[cfg(target_os = "windows")] { + Ok(rlimit::setmaxstdio(limit as u32)?.map(|v| v as u64)) + } else if #[cfg(unix)] { + rlimit::increase_nofile_limit(limit) + } } } @@ -69,7 +69,7 @@ pub fn limit() -> i32 { else if #[cfg(target_os = "windows")] { rlimit::getmaxstdio() as i32 } - else if #[cfg(any(target_os = "macos", target_os = "linux"))] { + else if #[cfg(unix)] { rlimit::getrlimit(rlimit::Resource::NOFILE).unwrap().0 as i32 } else { From c471bb9751b3b0e61dac75f150f9dc8416d4ffef Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Fri, 24 Nov 2023 17:25:04 +0200 Subject: [PATCH 4/8] relocate FD limit constants to daemon so that they are available as a part of the kaspad_lib crate (this way if these values are changed, they will propagate downstream) --- kaspad/src/daemon.rs | 9 +++++++++ kaspad/src/main.rs | 22 +++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/kaspad/src/daemon.rs b/kaspad/src/daemon.rs index 26d8442b43..dd6e9c24c4 100644 --- a/kaspad/src/daemon.rs +++ b/kaspad/src/daemon.rs @@ -30,6 +30,15 @@ use kaspa_perf_monitor::builder::Builder as PerfMonitorBuilder; use kaspa_utxoindex::{api::UtxoIndexProxy, UtxoIndex}; use kaspa_wrpc_server::service::{Options as WrpcServerOptions, ServerCounters as WrpcServerCounters, WrpcEncoding, WrpcService}; +/// Desired soft FD limit that needs to be configured +/// for the kaspad process. +pub const DESIRED_DAEMON_SOFT_FD_LIMIT: u64 = 16 * 1024; +/// Minimum acceptable soft FD limit for the kaspad +/// process. (Rusty Kaspa will operate with the minimal +/// acceptable limit of `1024`, but a setting below +/// this value may impact the database performance). +pub const MINIMUM_DAEMON_SOFT_FD_LIMIT: u64 = 4096; + use crate::args::Args; const DEFAULT_DATA_DIR: &str = "datadir"; diff --git a/kaspad/src/main.rs b/kaspad/src/main.rs index 0695cbff98..bd70cf679c 100644 --- a/kaspad/src/main.rs +++ b/kaspad/src/main.rs @@ -6,37 +6,33 @@ use std::sync::Arc; use kaspa_core::{info, signals::Signals}; use kaspa_utils::fd_budget; -use kaspad_lib::{args::parse_args, daemon::create_core}; +use kaspad_lib::{ + args::parse_args, + daemon::{create_core, DESIRED_DAEMON_SOFT_FD_LIMIT, MINIMUM_DAEMON_SOFT_FD_LIMIT}, +}; #[cfg(feature = "heap")] #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; -/// FD limits that kaspad tries to configure on startup -const DEFAULT_SOFT_FD_LIMIT: u64 = 16 * 1024; -/// The warning to the user will be issued only -/// if the soft FD limit is below this value. -/// (We desire 16k but deem 4k acceptable) -const DESIRED_SOFT_FD_LIMIT: u64 = 4096; - pub fn main() { #[cfg(feature = "heap")] let _profiler = dhat::Profiler::builder().file_name("kaspad-heap.json").build(); let args = parse_args(); - match fd_budget::ensure_os_limits(DEFAULT_SOFT_FD_LIMIT) { + match fd_budget::ensure_os_limits(DESIRED_DAEMON_SOFT_FD_LIMIT) { Ok(limit) => { - if limit < DESIRED_SOFT_FD_LIMIT { + if limit < MINIMUM_DAEMON_SOFT_FD_LIMIT { println!("Current OS file descriptor limit (soft FD limit) is set to {limit}"); - println!("The kaspad node requires a setting of at least {DEFAULT_SOFT_FD_LIMIT} to operate properly."); + println!("The kaspad node requires a setting of at least {DESIRED_DAEMON_SOFT_FD_LIMIT} to operate properly."); println!("Please increase the limits using the following command:"); - println!("ulimit -n {DEFAULT_SOFT_FD_LIMIT}"); + println!("ulimit -n {DESIRED_DAEMON_SOFT_FD_LIMIT}"); } } Err(err) => { println!("Unable to initialize the necessary OS file descriptor limit (soft FD limit) to: {}", err); - println!("The kaspad node requires a setting of at least {DEFAULT_SOFT_FD_LIMIT} to operate properly."); + println!("The kaspad node requires a setting of at least {DESIRED_DAEMON_SOFT_FD_LIMIT} to operate properly."); } } From 7b74a7e74fa01c548e17a009569ec77ba2572e08 Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Fri, 24 Nov 2023 17:30:22 +0200 Subject: [PATCH 5/8] rename fd_budget fn to try_set_fd_limit --- kaspad/src/main.rs | 2 +- utils/src/fd_budget.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kaspad/src/main.rs b/kaspad/src/main.rs index bd70cf679c..04b2d2b11a 100644 --- a/kaspad/src/main.rs +++ b/kaspad/src/main.rs @@ -21,7 +21,7 @@ pub fn main() { let args = parse_args(); - match fd_budget::ensure_os_limits(DESIRED_DAEMON_SOFT_FD_LIMIT) { + match fd_budget::try_set_fd_limit(DESIRED_DAEMON_SOFT_FD_LIMIT) { Ok(limit) => { if limit < MINIMUM_DAEMON_SOFT_FD_LIMIT { println!("Current OS file descriptor limit (soft FD limit) is set to {limit}"); diff --git a/utils/src/fd_budget.rs b/utils/src/fd_budget.rs index 38a436ce5c..b9b6e811da 100644 --- a/utils/src/fd_budget.rs +++ b/utils/src/fd_budget.rs @@ -51,7 +51,7 @@ pub fn acquire_guard(value: i32) -> Result { } #[cfg(not(target_arch = "wasm32"))] -pub fn ensure_os_limits(limit: u64) -> std::io::Result { +pub fn try_set_fd_limit(limit: u64) -> std::io::Result { cfg_if::cfg_if! { if #[cfg(target_os = "windows")] { Ok(rlimit::setmaxstdio(limit as u32)?.map(|v| v as u64)) From 383e1181fc6447225426fe6946b1658b7778f02b Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Fri, 24 Nov 2023 23:47:04 +0200 Subject: [PATCH 6/8] fix typo (1024->4096) --- kaspad/src/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kaspad/src/daemon.rs b/kaspad/src/daemon.rs index dd6e9c24c4..75330dfbdb 100644 --- a/kaspad/src/daemon.rs +++ b/kaspad/src/daemon.rs @@ -35,7 +35,7 @@ use kaspa_wrpc_server::service::{Options as WrpcServerOptions, ServerCounters as pub const DESIRED_DAEMON_SOFT_FD_LIMIT: u64 = 16 * 1024; /// Minimum acceptable soft FD limit for the kaspad /// process. (Rusty Kaspa will operate with the minimal -/// acceptable limit of `1024`, but a setting below +/// acceptable limit of `4096`, but a setting below /// this value may impact the database performance). pub const MINIMUM_DAEMON_SOFT_FD_LIMIT: u64 = 4096; From 26cd6c499cc1395ba74fa729fd3fef89bf6944b6 Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Fri, 24 Nov 2023 23:54:17 +0200 Subject: [PATCH 7/8] change 4096 to 4*1024 --- kaspad/src/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kaspad/src/daemon.rs b/kaspad/src/daemon.rs index 75330dfbdb..c15328b36e 100644 --- a/kaspad/src/daemon.rs +++ b/kaspad/src/daemon.rs @@ -37,7 +37,7 @@ pub const DESIRED_DAEMON_SOFT_FD_LIMIT: u64 = 16 * 1024; /// process. (Rusty Kaspa will operate with the minimal /// acceptable limit of `4096`, but a setting below /// this value may impact the database performance). -pub const MINIMUM_DAEMON_SOFT_FD_LIMIT: u64 = 4096; +pub const MINIMUM_DAEMON_SOFT_FD_LIMIT: u64 = 4*1024; use crate::args::Args; From 1e0f36bda4291eddf29ae069679666ca4927d6be Mon Sep 17 00:00:00 2001 From: Anton Yemelyanov Date: Mon, 27 Nov 2023 05:35:23 +0200 Subject: [PATCH 8/8] lints --- kaspad/src/daemon.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kaspad/src/daemon.rs b/kaspad/src/daemon.rs index c15328b36e..3dbaff978b 100644 --- a/kaspad/src/daemon.rs +++ b/kaspad/src/daemon.rs @@ -37,7 +37,7 @@ pub const DESIRED_DAEMON_SOFT_FD_LIMIT: u64 = 16 * 1024; /// process. (Rusty Kaspa will operate with the minimal /// acceptable limit of `4096`, but a setting below /// this value may impact the database performance). -pub const MINIMUM_DAEMON_SOFT_FD_LIMIT: u64 = 4*1024; +pub const MINIMUM_DAEMON_SOFT_FD_LIMIT: u64 = 4 * 1024; use crate::args::Args;