[tx fees, policy]: create a mempool fee estimator

Calculate the fee rate estimate for a given confirmation target using the mempool
unconfirmed transactions.

Co-authored-by: willcl-ark <[email protected]>

src/Makefile.am

@@ -247,6 +247,7 @@ BITCOIN_CORE_H = \
   policy/fees.h \
   policy/fees_args.h \
   policy/packages.h \
+  policy/mempool_fees.h \
   policy/policy.h \
   policy/rbf.h \
   policy/settings.h \
@@ -445,6 +446,7 @@ libbitcoin_node_a_SOURCES = \
   policy/v3_policy.cpp \
   policy/fees.cpp \
   policy/fees_args.cpp \
+  policy/mempool_fees.cpp \
   policy/packages.cpp \
   policy/rbf.cpp \
   policy/settings.cpp \
@@ -706,6 +708,7 @@ libbitcoin_common_a_SOURCES = \
   outputtype.cpp \
   policy/v3_policy.cpp \
   policy/feerate.cpp \
+  policy/mempool_fees.cpp \
   policy/policy.cpp \
   protocol.cpp \
   psbt.cpp \
@@ -965,6 +968,8 @@ libbitcoinkernel_la_SOURCES = \
   node/utxo_snapshot.cpp \
   policy/v3_policy.cpp \
   policy/feerate.cpp \
+  policy/fees.cpp \
+  policy/mempool_fees.cpp
   policy/packages.cpp \
   policy/policy.cpp \
   policy/rbf.cpp \

src/init.cpp

@@ -57,6 +57,7 @@
 #include <policy/feerate.h>
 #include <policy/fees.h>
 #include <policy/fees_args.h>
+#include <policy/mempool_fees.h>
 #include <policy/policy.h>
 #include <policy/settings.h>
 #include <protocol.h>
@@ -375,6 +376,7 @@ void Shutdown(NodeContext& node)
     node.chain_clients.clear();
     UnregisterAllValidationInterfaces();
     GetMainSignals().UnregisterBackgroundSignalScheduler();
+    node.kernel.reset();
     node.mempool.reset();
     node.fee_estimator.reset();
     node.chainman.reset();
@@ -1267,6 +1269,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         RegisterValidationInterface(fee_estimator);
     }
 
+    assert(!node.mempool_fee_estimator);
+
+    node.mempool_fee_estimator = std::make_unique<MemPoolPolicyEstimator>();
+
     // Check port numbers
     for (const std::string port_option : {
         "-port",

src/node/context.cpp

@@ -13,6 +13,7 @@
 #include <netgroup.h>
 #include <node/kernel_notifications.h>
 #include <policy/fees.h>
+#include <policy/mempool_fees.h>
 #include <scheduler.h>
 #include <txmempool.h>
 #include <validation.h>

src/node/context.h

@@ -23,6 +23,7 @@ class CConnman;
 class CScheduler;
 class CTxMemPool;
 class ChainstateManager;
+class MemPoolPolicyEstimator;
 class NetGroupManager;
 class PeerManager;
 namespace interfaces {
@@ -57,6 +58,7 @@ struct NodeContext {
     std::unique_ptr<CTxMemPool> mempool;
     std::unique_ptr<const NetGroupManager> netgroupman;
     std::unique_ptr<CBlockPolicyEstimator> fee_estimator;
+    std::unique_ptr<MemPoolPolicyEstimator> mempool_fee_estimator;
     std::unique_ptr<PeerManager> peerman;
     std::unique_ptr<ChainstateManager> chainman;
     std::unique_ptr<BanMan> banman;

src/policy/mempool_fees.cpp

@@ -0,0 +1,126 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <logging.h>
+
+#include <node/miner.h>
+#include <policy/mempool_fees.h>
+#include <policy/policy.h>
+
+using node::GetCustomBlockFeeRateHistogram;
+
+MemPoolPolicyEstimator::MemPoolPolicyEstimator() {}
+
+CFeeRate MemPoolPolicyEstimator::EstimateFeeWithMemPool(Chainstate& chainstate, const CTxMemPool& mempool, unsigned int confTarget, const bool force, std::string& err_message) const
+{
+    std::optional<CFeeRate> cached_fee{std::nullopt};
+    std::map<uint64_t, CFeeRate> fee_rates;
+    CFeeRate block_fee_rate{0};
+
+    if (confTarget > MAX_CONF_TARGET) {
+        err_message = strprintf("Confirmation target %s is above maximum limit of %s, mempool conditions might change and estimates above %s are unreliable.\n", confTarget, MAX_CONF_TARGET, MAX_CONF_TARGET);
+        return CFeeRate(0);
+    }
+
+    if (!mempool.GetLoadTried()) {
+        err_message = "Mempool did not finish loading, can't get accurate fee rate estimate.";
+        return CFeeRate(0);
+    }
+    // Try the cache if not forced
+    if (!force) {
+        cached_fee = cache.get(confTarget);
+    }
+
+    if (!cached_fee) {
+        // Run block builder and update cache
+        std::map<CFeeRate, uint64_t> mempool_fee_stats;
+        size_t target_weight = confTarget * DEFAULT_BLOCK_MAX_WEIGHT;
+        mempool_fee_stats = GetCustomBlockFeeRateHistogram(chainstate, &mempool, target_weight);
+        if (mempool_fee_stats.empty()) {
+            err_message = "No transactions available in the mempool yet.";
+            return CFeeRate(0);
+        }
+        fee_rates = EstimateBlockFeeRatesWithMempool(mempool_fee_stats, confTarget);
+        cache.update(fee_rates);
+        block_fee_rate = fee_rates[confTarget];
+    } else {
+        // Use the cached value
+        block_fee_rate = *cached_fee;
+    }
+
+    if (block_fee_rate == CFeeRate(0)) {
+        err_message = "Insufficient mempool transactions to perform an estimate.";
+    }
+    return block_fee_rate;
+}
+
+std::map<uint64_t, CFeeRate> MemPoolPolicyEstimator::EstimateBlockFeeRatesWithMempool(
+    const std::map<CFeeRate, uint64_t>& mempool_fee_stats, unsigned int confTarget) const
+{
+    std::map<uint64_t, CFeeRate> fee_rates;
+     // Return empty if no stats
+    if (mempool_fee_stats.empty()) return fee_rates;
+
+    auto start = mempool_fee_stats.begin();
+    auto cur = mempool_fee_stats.begin();
+    auto end = mempool_fee_stats.end();
+
+    size_t block_number{1};
+    size_t block_weight{0};
+
+    while (block_number <= confTarget && cur != end) {
+        size_t transaction_weight = cur->second * WITNESS_SCALE_FACTOR;
+        block_weight += transaction_weight;
+
+        auto next_cur = std::next(cur);
+
+        // Check if we've accumulated enough weight for this block or if we're at the last bin
+        if (block_weight >= DEFAULT_BLOCK_MAX_WEIGHT || next_cur == end) {
+            // Include the current bin in this block's calculation
+            auto end_it = next_cur;
+            if (next_cur != end) ++end_it; // Ensure the range includes the current bin
+
+            fee_rates[block_number] = CalculateMedianFeeRate(start, end_it);
+
+            block_number++;
+            block_weight = 0;
+            start = end_it;
+        }
+
+        cur = next_cur;
+    }
+
+    return fee_rates;
+}
+
+CFeeRate MemPoolPolicyEstimator::CalculateMedianFeeRate(
+    std::map<CFeeRate, uint64_t>::const_iterator& start_it,
+    std::map<CFeeRate, uint64_t>::const_iterator& end_it) const
+{
+    // Ensure we have enough txs in the target block template.
+    auto curr = start_it;
+    unsigned int block_weight{0};
+    while (curr != end_it) {
+        block_weight += curr->second * WITNESS_SCALE_FACTOR;
+        ++curr;
+    }
+    if (block_weight < (DEFAULT_BLOCK_MAX_WEIGHT / 2)) {
+        return CFeeRate(0);
+    }
+
+    std::size_t size = std::distance(start_it, end_it);
+    if (size % 2 == 0) {
+        // If the number of txs is even, average the two middle fee rates
+        auto first_mid_it = std::next(start_it, size / 2);
+        auto second_mid_it = std::next(start_it, (size / 2) + 1);
+        auto mid_fee = (first_mid_it->first.GetFeePerK() + second_mid_it->first.GetFeePerK()) / 2;
+        return CFeeRate(mid_fee);
+    } else {
+        // If the number of txs is odd, return the fee rate of the middle tx
+        auto mid_it = std::next(start_it, (size / 2) + 1);
+        return mid_it->first;
+    }
+}
+

src/policy/mempool_fees.h

@@ -0,0 +1,110 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_POLICY_MEMPOOL_FEES_H
+#define BITCOIN_POLICY_MEMPOOL_FEES_H
+
+#include <chrono>
+#include <map>
+#include <optional>
+#include <string>
+
+#include <logging.h>
+#include <policy/feerate.h>
+
+class Chainstate;
+class CTxMemPool;
+
+// Fee rate estimates above this confirmation target are not reliable,
+// mempool condition might likely change.
+static const unsigned int MAX_CONF_TARGET{3};
+
+// Cache mempool-based feerate estimates to avoid repeatedly running the
+// expensive block-building algorithm.
+struct CachedEstimates {
+private:
+    static constexpr std::chrono::seconds cache_life{30};
+    std::map<uint64_t, CFeeRate> estimates;
+    std::chrono::steady_clock::time_point last_updated;
+
+    bool isStale() const
+    {
+        LogPrintf("MemPoolFeeEstimator: Checking if cache is stale...\n");
+        return (last_updated + cache_life) < std::chrono::steady_clock::now();
+    }
+
+public:
+    std::optional<CFeeRate> get(uint64_t number_of_blocks) const
+    {
+        if (isStale()) return std::nullopt;
+        LogPrintf("MemPoolFeeEstimator: Cache is not stale, using cached value\n");
+
+        auto it = estimates.find(number_of_blocks);
+        if (it != estimates.end()) {
+            return it->second;
+        }
+        return std::nullopt;
+    }
+
+    void update(std::map<uint64_t, CFeeRate>& newEstimates)
+    {
+        // Overwrite the entire map with the new data to avoid old
+        // estimates remaining.
+        estimates = newEstimates;
+        last_updated = std::chrono::steady_clock::now();
+        LogPrintf("MemPoolFeeEstimator: Updated cache\n");
+    }
+};
+
+/**
+ * MemPoolPolicyEstimator estimates the fee rate that a tx should pay
+ * to be included in a confirmation target based on the mempool
+ * txs and their fee rates.
+ *
+ * The estimator works by generating template block up to a given confirmation target and then calculate the median
+ * fee rate of the txs in the confirmation target block as the approximate fee rate that a tx will pay to
+ * likely be included in the block.
+ */
+class MemPoolPolicyEstimator
+{
+public:
+
+    MemPoolPolicyEstimator();
+
+    ~MemPoolPolicyEstimator() = default;
+
+    /**
+     * Estimate the fee rate from mempool txs data given a confirmation target.
+     *
+     * @param[in] chainstate The reference to the active chainstate.
+     * @param[in] mempool The reference to the mempool from which we will estimate the fee rate.
+     * @param[in] confTarget The confirmation target of transactions.
+     * @param[out] err_message  optional error message.
+     * @return The estimated fee rate.
+     */
+    CFeeRate EstimateFeeWithMemPool(Chainstate& chainstate, const CTxMemPool& mempool, unsigned int confTarget, const bool force, std::string& err_message) const;
+
+private:
+    mutable CachedEstimates cache;
+    /**
+     * Calculate the fee rate estimate for blocks of txs up to num_blocks.
+     *
+     * @param[in] mempool_fee_stats The mempool fee statistics (fee rate and size).
+     * @param[in] confTarget The next block we wish to a tx to be included in.
+     * @param[in] target_weight Calculated of the histogram.
+     * @return The fee rate estimate in satoshis per kilobyte.
+     */
+std::map<uint64_t, CFeeRate> EstimateBlockFeeRatesWithMempool(const std::map<CFeeRate, uint64_t>& mempool_fee_stats, unsigned int num_blocks) const;
+
+    /**
+     * Calculate the median fee rate for a range of txs in the mempool.
+     *
+     * @param[in] start_it The iterator pointing to the beginning of the range.
+     * @param[in] end_it The iterator pointing to the end of the range.
+     * @return The median fee rate.
+     */
+    CFeeRate CalculateMedianFeeRate(std::map<CFeeRate, uint64_t>::const_iterator& start_it, std::map<CFeeRate, uint64_t>::const_iterator& end_it) const;
+};
+#endif // BITCOIN_POLICY_MEMPOOL_FEES_H