From 880aa177085eabf4e1a94b32b57cf712b1887521 Mon Sep 17 00:00:00 2001 From: ismaelsadeeq Date: Tue, 20 Feb 2024 13:38:36 +0000 Subject: [PATCH] [tx fees, policy]: create a mempool fee estimator Calculate the fee rate estimate for a given confirmation target using the mempool unconfirmed transactions. Co-authored-by: willcl-ark --- src/Makefile.am | 5 ++ src/init.cpp | 6 ++ src/node/context.cpp | 1 + src/node/context.h | 2 + src/policy/mempool_fees.cpp | 126 ++++++++++++++++++++++++++++++++++++ src/policy/mempool_fees.h | 110 +++++++++++++++++++++++++++++++ 6 files changed, 250 insertions(+) create mode 100644 src/policy/mempool_fees.cpp create mode 100644 src/policy/mempool_fees.h diff --git a/src/Makefile.am b/src/Makefile.am index 3e8870c8286684..21a929d7ea3a1b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -247,6 +247,7 @@ BITCOIN_CORE_H = \ policy/fees.h \ policy/fees_args.h \ policy/packages.h \ + policy/mempool_fees.h \ policy/policy.h \ policy/rbf.h \ policy/settings.h \ @@ -445,6 +446,7 @@ libbitcoin_node_a_SOURCES = \ policy/v3_policy.cpp \ policy/fees.cpp \ policy/fees_args.cpp \ + policy/mempool_fees.cpp \ policy/packages.cpp \ policy/rbf.cpp \ policy/settings.cpp \ @@ -706,6 +708,7 @@ libbitcoin_common_a_SOURCES = \ outputtype.cpp \ policy/v3_policy.cpp \ policy/feerate.cpp \ + policy/mempool_fees.cpp \ policy/policy.cpp \ protocol.cpp \ psbt.cpp \ @@ -965,6 +968,8 @@ libbitcoinkernel_la_SOURCES = \ node/utxo_snapshot.cpp \ policy/v3_policy.cpp \ policy/feerate.cpp \ + policy/fees.cpp \ + policy/mempool_fees.cpp policy/packages.cpp \ policy/policy.cpp \ policy/rbf.cpp \ diff --git a/src/init.cpp b/src/init.cpp index 988daefeec800c..f1089ecc13c1d1 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -57,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -375,6 +376,7 @@ void Shutdown(NodeContext& node) node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); + node.kernel.reset(); node.mempool.reset(); node.fee_estimator.reset(); node.chainman.reset(); @@ -1267,6 +1269,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) RegisterValidationInterface(fee_estimator); } + assert(!node.mempool_fee_estimator); + + node.mempool_fee_estimator = std::make_unique(); + // Check port numbers for (const std::string port_option : { "-port", diff --git a/src/node/context.cpp b/src/node/context.cpp index ca56fa0b86624c..9f6cd3c5b80837 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/src/node/context.h b/src/node/context.h index 4f3b640b2d2752..dfbb507bd93ac3 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -23,6 +23,7 @@ class CConnman; class CScheduler; class CTxMemPool; class ChainstateManager; +class MemPoolPolicyEstimator; class NetGroupManager; class PeerManager; namespace interfaces { @@ -57,6 +58,7 @@ struct NodeContext { std::unique_ptr mempool; std::unique_ptr netgroupman; std::unique_ptr fee_estimator; + std::unique_ptr mempool_fee_estimator; std::unique_ptr peerman; std::unique_ptr chainman; std::unique_ptr banman; diff --git a/src/policy/mempool_fees.cpp b/src/policy/mempool_fees.cpp new file mode 100644 index 00000000000000..9054d7fe02ba70 --- /dev/null +++ b/src/policy/mempool_fees.cpp @@ -0,0 +1,126 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include + +using node::GetCustomBlockFeeRateHistogram; + +MemPoolPolicyEstimator::MemPoolPolicyEstimator() {} + +CFeeRate MemPoolPolicyEstimator::EstimateFeeWithMemPool(Chainstate& chainstate, const CTxMemPool& mempool, unsigned int confTarget, const bool force, std::string& err_message) const +{ + std::optional cached_fee{std::nullopt}; + std::map fee_rates; + CFeeRate block_fee_rate{0}; + + if (confTarget > MAX_CONF_TARGET) { + err_message = strprintf("Confirmation target %s is above maximum limit of %s, mempool conditions might change and estimates above %s are unreliable.\n", confTarget, MAX_CONF_TARGET, MAX_CONF_TARGET); + return CFeeRate(0); + } + + if (!mempool.GetLoadTried()) { + err_message = "Mempool did not finish loading, can't get accurate fee rate estimate."; + return CFeeRate(0); + } + // Try the cache if not forced + if (!force) { + cached_fee = cache.get(confTarget); + } + + if (!cached_fee) { + // Run block builder and update cache + std::map mempool_fee_stats; + size_t target_weight = confTarget * DEFAULT_BLOCK_MAX_WEIGHT; + mempool_fee_stats = GetCustomBlockFeeRateHistogram(chainstate, &mempool, target_weight); + if (mempool_fee_stats.empty()) { + err_message = "No transactions available in the mempool yet."; + return CFeeRate(0); + } + fee_rates = EstimateBlockFeeRatesWithMempool(mempool_fee_stats, confTarget); + cache.update(fee_rates); + block_fee_rate = fee_rates[confTarget]; + } else { + // Use the cached value + block_fee_rate = *cached_fee; + } + + if (block_fee_rate == CFeeRate(0)) { + err_message = "Insufficient mempool transactions to perform an estimate."; + } + return block_fee_rate; +} + +std::map MemPoolPolicyEstimator::EstimateBlockFeeRatesWithMempool( + const std::map& mempool_fee_stats, unsigned int confTarget) const +{ + std::map fee_rates; + // Return empty if no stats + if (mempool_fee_stats.empty()) return fee_rates; + + auto start = mempool_fee_stats.begin(); + auto cur = mempool_fee_stats.begin(); + auto end = mempool_fee_stats.end(); + + size_t block_number{1}; + size_t block_weight{0}; + + while (block_number <= confTarget && cur != end) { + size_t transaction_weight = cur->second * WITNESS_SCALE_FACTOR; + block_weight += transaction_weight; + + auto next_cur = std::next(cur); + + // Check if we've accumulated enough weight for this block or if we're at the last bin + if (block_weight >= DEFAULT_BLOCK_MAX_WEIGHT || next_cur == end) { + // Include the current bin in this block's calculation + auto end_it = next_cur; + if (next_cur != end) ++end_it; // Ensure the range includes the current bin + + fee_rates[block_number] = CalculateMedianFeeRate(start, end_it); + + block_number++; + block_weight = 0; + start = end_it; + } + + cur = next_cur; + } + + return fee_rates; +} + +CFeeRate MemPoolPolicyEstimator::CalculateMedianFeeRate( + std::map::const_iterator& start_it, + std::map::const_iterator& end_it) const +{ + // Ensure we have enough txs in the target block template. + auto curr = start_it; + unsigned int block_weight{0}; + while (curr != end_it) { + block_weight += curr->second * WITNESS_SCALE_FACTOR; + ++curr; + } + if (block_weight < (DEFAULT_BLOCK_MAX_WEIGHT / 2)) { + return CFeeRate(0); + } + + std::size_t size = std::distance(start_it, end_it); + if (size % 2 == 0) { + // If the number of txs is even, average the two middle fee rates + auto first_mid_it = std::next(start_it, size / 2); + auto second_mid_it = std::next(start_it, (size / 2) + 1); + auto mid_fee = (first_mid_it->first.GetFeePerK() + second_mid_it->first.GetFeePerK()) / 2; + return CFeeRate(mid_fee); + } else { + // If the number of txs is odd, return the fee rate of the middle tx + auto mid_it = std::next(start_it, (size / 2) + 1); + return mid_it->first; + } +} + diff --git a/src/policy/mempool_fees.h b/src/policy/mempool_fees.h new file mode 100644 index 00000000000000..aa528345a416f8 --- /dev/null +++ b/src/policy/mempool_fees.h @@ -0,0 +1,110 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POLICY_MEMPOOL_FEES_H +#define BITCOIN_POLICY_MEMPOOL_FEES_H + +#include +#include +#include +#include + +#include +#include + +class Chainstate; +class CTxMemPool; + +// Fee rate estimates above this confirmation target are not reliable, +// mempool condition might likely change. +static const unsigned int MAX_CONF_TARGET{3}; + +// Cache mempool-based feerate estimates to avoid repeatedly running the +// expensive block-building algorithm. +struct CachedEstimates { +private: + static constexpr std::chrono::seconds cache_life{30}; + std::map estimates; + std::chrono::steady_clock::time_point last_updated; + + bool isStale() const + { + LogPrintf("MemPoolFeeEstimator: Checking if cache is stale...\n"); + return (last_updated + cache_life) < std::chrono::steady_clock::now(); + } + +public: + std::optional get(uint64_t number_of_blocks) const + { + if (isStale()) return std::nullopt; + LogPrintf("MemPoolFeeEstimator: Cache is not stale, using cached value\n"); + + auto it = estimates.find(number_of_blocks); + if (it != estimates.end()) { + return it->second; + } + return std::nullopt; + } + + void update(std::map& newEstimates) + { + // Overwrite the entire map with the new data to avoid old + // estimates remaining. + estimates = newEstimates; + last_updated = std::chrono::steady_clock::now(); + LogPrintf("MemPoolFeeEstimator: Updated cache\n"); + } +}; + +/** + * MemPoolPolicyEstimator estimates the fee rate that a tx should pay + * to be included in a confirmation target based on the mempool + * txs and their fee rates. + * + * The estimator works by generating template block up to a given confirmation target and then calculate the median + * fee rate of the txs in the confirmation target block as the approximate fee rate that a tx will pay to + * likely be included in the block. + */ +class MemPoolPolicyEstimator +{ +public: + + MemPoolPolicyEstimator(); + + ~MemPoolPolicyEstimator() = default; + + /** + * Estimate the fee rate from mempool txs data given a confirmation target. + * + * @param[in] chainstate The reference to the active chainstate. + * @param[in] mempool The reference to the mempool from which we will estimate the fee rate. + * @param[in] confTarget The confirmation target of transactions. + * @param[out] err_message optional error message. + * @return The estimated fee rate. + */ + CFeeRate EstimateFeeWithMemPool(Chainstate& chainstate, const CTxMemPool& mempool, unsigned int confTarget, const bool force, std::string& err_message) const; + +private: + mutable CachedEstimates cache; + /** + * Calculate the fee rate estimate for blocks of txs up to num_blocks. + * + * @param[in] mempool_fee_stats The mempool fee statistics (fee rate and size). + * @param[in] confTarget The next block we wish to a tx to be included in. + * @param[in] target_weight Calculated of the histogram. + * @return The fee rate estimate in satoshis per kilobyte. + */ +std::map EstimateBlockFeeRatesWithMempool(const std::map& mempool_fee_stats, unsigned int num_blocks) const; + + /** + * Calculate the median fee rate for a range of txs in the mempool. + * + * @param[in] start_it The iterator pointing to the beginning of the range. + * @param[in] end_it The iterator pointing to the end of the range. + * @return The median fee rate. + */ + CFeeRate CalculateMedianFeeRate(std::map::const_iterator& start_it, std::map::const_iterator& end_it) const; +}; +#endif // BITCOIN_POLICY_MEMPOOL_FEES_H