Modify fd_budget for better resource allocation

Adjusted fd_budget allocation within multiple files to enhance resource utilization across various components of the application. The changes include dividing fd_total_budget equally among active and staging consensuses and assigning fd_budget based on the given limit values in each module. These modifications were implemented to ensure more efficient and fair distribution of resources, and to prevent potential resource allocation issues for unsupported operating systems.

consensus/src/consensus/factory.rs

@@ -193,7 +193,7 @@ pub struct Factory {
     notification_root: Arc<ConsensusNotificationRoot>,
     counters: Arc<ProcessingCounters>,
     tx_script_cache_counters: Arc<TxScriptCacheCounters>,
-    fd_total_budget: i32,
+    fd_budget: i32,
 }
 
 impl Factory {
@@ -205,7 +205,7 @@ impl Factory {
         notification_root: Arc<ConsensusNotificationRoot>,
         counters: Arc<ProcessingCounters>,
         tx_script_cache_counters: Arc<TxScriptCacheCounters>,
-        fd_total_budget: i32,
+        fd_budget: i32,
     ) -> Self {
         let mut config = config.clone();
         #[cfg(feature = "devnet-prealloc")]
@@ -221,7 +221,7 @@ impl Factory {
             notification_root,
             counters,
             tx_script_cache_counters,
-            fd_total_budget,
+            fd_budget,
         };
         factory.clean_non_active_consensus_entries();
         factory
@@ -271,7 +271,7 @@ impl ConsensusFactory for Factory {
         let db = kaspa_database::prelude::ConnBuilder::default()
             .with_db_path(dir)
             .with_parallelism(self.db_parallelism)
-            .with_files_limit(200.max(self.fd_total_budget * 70 / 100))
+            .with_files_limit(self.fd_budget / 2) // active and staging consensuses should have equal budgets
             .build()
             .unwrap();
 
@@ -305,7 +305,7 @@ impl ConsensusFactory for Factory {
         let db = kaspa_database::prelude::ConnBuilder::default()
             .with_db_path(dir)
             .with_parallelism(self.db_parallelism)
-            .with_files_limit(10.max(self.fd_total_budget * 10 / 100))
+            .with_files_limit(self.fd_budget / 2) // active and staging consensuses should have equal budgets
             .build()
             .unwrap();
 

kaspad/src/daemon.rs

@@ -36,7 +36,6 @@ const DEFAULT_DATA_DIR: &str = "datadir";
 const CONSENSUS_DB: &str = "consensus";
 const UTXOINDEX_DB: &str = "utxoindex";
 const META_DB: &str = "meta";
-const UTXO_INDEX_DB_FILE_LIMIT: i32 = 100;
 const META_DB_FILE_LIMIT: i32 = 5;
 const DEFAULT_LOG_DIR: &str = "logs";
 
@@ -177,7 +176,14 @@ pub fn create_core(args: Args, fd_total_budget: i32) -> (Arc<Core>, Arc<RpcCoreS
 ///
 pub fn create_core_with_runtime(runtime: &Runtime, args: &Args, fd_total_budget: i32) -> (Arc<Core>, Arc<RpcCoreService>) {
     let network = args.network();
-
+    let mut fd_remaining = fd_total_budget;
+    let utxo_files_limit = if args.utxoindex {
+        let utxo_files_limit = fd_remaining * 10 / 100;
+        fd_remaining -= utxo_files_limit;
+        utxo_files_limit
+    } else {
+        0
+    };
     // Make sure args forms a valid set of properties
     if let Err(err) = validate_args(args) {
         println!("{}", err);
@@ -233,7 +239,7 @@ do you confirm? (answer y/n or pass --yes to the Kaspad command line to confirm
     // DB used for addresses store and for multi-consensus management
     let mut meta_db = kaspa_database::prelude::ConnBuilder::default()
         .with_db_path(meta_db_dir.clone())
-        .with_files_limit(META_DB_FILE_LIMIT.max(fd_total_budget * 5 / 100))
+        .with_files_limit(META_DB_FILE_LIMIT)
         .build()
         .unwrap();
 
@@ -258,7 +264,7 @@ do you confirm? (answer y/n or pass --yes to the Kaspad command line to confirm
         // Reopen the DB
         meta_db = kaspa_database::prelude::ConnBuilder::default()
             .with_db_path(meta_db_dir)
-            .with_files_limit(META_DB_FILE_LIMIT.max(fd_total_budget * 5 / 100))
+            .with_files_limit(META_DB_FILE_LIMIT)
             .build()
             .unwrap();
     }
@@ -299,7 +305,7 @@ do you confirm? (answer y/n or pass --yes to the Kaspad command line to confirm
         notification_root.clone(),
         processing_counters.clone(),
         tx_script_cache_counters.clone(),
-        fd_total_budget,
+        fd_remaining,
     ));
     let consensus_manager = Arc::new(ConsensusManager::new(consensus_factory));
     let consensus_monitor = Arc::new(ConsensusMonitor::new(processing_counters.clone(), tick_service.clone()));
@@ -323,7 +329,7 @@ do you confirm? (answer y/n or pass --yes to the Kaspad command line to confirm
         // Use only a single thread for none-consensus databases
         let utxoindex_db = kaspa_database::prelude::ConnBuilder::default()
             .with_db_path(utxoindex_db_dir)
-            .with_files_limit(UTXO_INDEX_DB_FILE_LIMIT.max(fd_total_budget * 15 / 100))
+            .with_files_limit(utxo_files_limit)
             .build()
             .unwrap();
         let utxoindex = UtxoIndexProxy::new(UtxoIndex::new(consensus_manager.clone(), utxoindex_db).unwrap());

kaspad/src/main.rs

@@ -17,7 +17,7 @@ pub fn main() {
     let _profiler = dhat::Profiler::builder().file_name("kaspad-heap.json").build();
 
     let args = parse_args();
-    let fd_total_budget = limit() - 32;
+    let fd_total_budget = limit() - args.rpc_max_clients - args.inbound_limit - args.outbound_target;
     let (core, _) = create_core(args, fd_total_budget);
 
     // Bind the keyboard signal to the core

simpa/src/main.rs

@@ -24,6 +24,7 @@ use kaspa_database::prelude::ConnBuilder;
 use kaspa_database::{create_temp_db, load_existing_db};
 use kaspa_hashes::Hash;
 use kaspa_perf_monitor::builder::Builder;
+use kaspa_utils::fd_budget::limit;
 use simulator::network::KaspaNetworkSimulator;
 use std::{collections::VecDeque, sync::Arc, time::Duration};
 
@@ -169,7 +170,8 @@ fn main() {
         builder = builder.set_archival();
     }
     let config = Arc::new(builder.build());
-    let mut conn_builder = ConnBuilder::default().with_parallelism(num_cpus::get()).with_files_limit(200);
+    let default_fd = limit() / 2;
+    let mut conn_builder = ConnBuilder::default().with_parallelism(num_cpus::get()).with_files_limit(default_fd);
     if let Some(rocksdb_files_limit) = args.rocksdb_files_limit {
         conn_builder = conn_builder.with_files_limit(rocksdb_files_limit);
     }
@@ -220,7 +222,7 @@ fn main() {
     }
 
     // Benchmark the DAG validation time
-    let (_lifetime2, db2) = create_temp_db!(ConnBuilder::default().with_parallelism(num_cpus::get()).with_files_limit(200));
+    let (_lifetime2, db2) = create_temp_db!(ConnBuilder::default().with_parallelism(num_cpus::get()).with_files_limit(default_fd));
     let (dummy_notification_sender, _) = unbounded();
     let notification_root = Arc::new(ConsensusNotificationRoot::new(dummy_notification_sender));
     let consensus2 = Arc::new(Consensus::new(

simpa/src/simulator/network.rs

@@ -12,6 +12,7 @@ use kaspa_consensus_core::block::Block;
 use kaspa_database::prelude::ConnBuilder;
 use kaspa_database::utils::DbLifetime;
 use kaspa_database::{create_permanent_db, create_temp_db};
+use kaspa_utils::fd_budget::limit;
 use kaspa_utils::sim::Simulation;
 
 type ConsensusWrapper = (Arc<Consensus>, Vec<JoinHandle<()>>, DbLifetime);
@@ -53,7 +54,7 @@ impl KaspaNetworkSimulator {
         let secp = secp256k1::Secp256k1::new();
         let mut rng = rand::thread_rng();
         for i in 0..num_miners {
-            let mut builder = ConnBuilder::default().with_files_limit(200);
+            let mut builder = ConnBuilder::default().with_files_limit(limit() / 2 / num_miners as i32);
             if let Some(rocksdb_files_limit) = rocksdb_files_limit {
                 builder = builder.with_files_limit(rocksdb_files_limit);
             }

testing/integration/src/mempool_benchmarks.rs

@@ -207,7 +207,7 @@ async fn bench_bbt_latency() {
     verify_tx_dag(&utxoset, &txs);
     info!("Generated overall {} txs", txs.len());
 
-    let fd_total_budget = limit() - 64;
+    let fd_total_budget = limit();
     let mut daemon = Daemon::new_random_with_args(args, fd_total_budget);
     let client = daemon.start().await;
     let bbt_client = daemon.new_client().await;