[tx fees, policy]: create a mempool fee estimator

Calculate the fee rate estimate for a given confirmation target using the mempool
unconfirmed transactions.

Co-authored-by: willcl-ark <[email protected]>

src/Makefile.am

@@ -247,6 +247,7 @@ BITCOIN_CORE_H = \
   policy/fees.h \
   policy/fees_args.h \
   policy/packages.h \
+  policy/mempool_fees.h \
   policy/policy.h \
   policy/rbf.h \
   policy/settings.h \
@@ -445,6 +446,7 @@ libbitcoin_node_a_SOURCES = \
   policy/v3_policy.cpp \
   policy/fees.cpp \
   policy/fees_args.cpp \
+  policy/mempool_fees.cpp \
   policy/packages.cpp \
   policy/rbf.cpp \
   policy/settings.cpp \
@@ -706,6 +708,7 @@ libbitcoin_common_a_SOURCES = \
   outputtype.cpp \
   policy/v3_policy.cpp \
   policy/feerate.cpp \
+  policy/mempool_fees.cpp \
   policy/policy.cpp \
   protocol.cpp \
   psbt.cpp \
@@ -965,6 +968,8 @@ libbitcoinkernel_la_SOURCES = \
   node/utxo_snapshot.cpp \
   policy/v3_policy.cpp \
   policy/feerate.cpp \
+  policy/fees.cpp \
+  policy/mempool_fees.cpp
   policy/packages.cpp \
   policy/policy.cpp \
   policy/rbf.cpp \

src/init.cpp

@@ -57,6 +57,7 @@
 #include <policy/feerate.h>
 #include <policy/fees.h>
 #include <policy/fees_args.h>
+#include <policy/mempool_fees.h>
 #include <policy/policy.h>
 #include <policy/settings.h>
 #include <protocol.h>
@@ -375,6 +376,7 @@ void Shutdown(NodeContext& node)
     node.chain_clients.clear();
     UnregisterAllValidationInterfaces();
     GetMainSignals().UnregisterBackgroundSignalScheduler();
+    node.kernel.reset();
     node.mempool.reset();
     node.fee_estimator.reset();
     node.chainman.reset();
@@ -1267,6 +1269,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         RegisterValidationInterface(fee_estimator);
     }
 
+    assert(!node.mempool_fee_estimator);
+
+    node.mempool_fee_estimator = std::make_unique<MemPoolPolicyEstimator>();
+
     // Check port numbers
     for (const std::string port_option : {
         "-port",

src/node/context.cpp

@@ -13,6 +13,7 @@
 #include <netgroup.h>
 #include <node/kernel_notifications.h>
 #include <policy/fees.h>
+#include <policy/mempool_fees.h>
 #include <scheduler.h>
 #include <txmempool.h>
 #include <validation.h>

src/node/context.h

@@ -23,6 +23,7 @@ class CConnman;
 class CScheduler;
 class CTxMemPool;
 class ChainstateManager;
+class MemPoolPolicyEstimator;
 class NetGroupManager;
 class PeerManager;
 namespace interfaces {
@@ -57,6 +58,7 @@ struct NodeContext {
     std::unique_ptr<CTxMemPool> mempool;
     std::unique_ptr<const NetGroupManager> netgroupman;
     std::unique_ptr<CBlockPolicyEstimator> fee_estimator;
+    std::unique_ptr<MemPoolPolicyEstimator> mempool_fee_estimator;
     std::unique_ptr<PeerManager> peerman;
     std::unique_ptr<ChainstateManager> chainman;
     std::unique_ptr<BanMan> banman;

src/policy/mempool_fees.cpp

@@ -0,0 +1,110 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <logging.h>
+
+#include <node/miner.h>
+#include <policy/mempool_fees.h>
+#include <policy/policy.h>
+
+using node::GetCustomBlockFeeRateHistogram;
+
+MemPoolPolicyEstimator::MemPoolPolicyEstimator() {}
+
+CFeeRate MemPoolPolicyEstimator::EstimateFeeWithMemPool(Chainstate& chainstate, const CTxMemPool& mempool, unsigned int confTarget, const bool force, std::string& err_message) const
+{
+    std::optional<CFeeRate> cached_fee{std::nullopt};
+    std::map<uint64_t, CFeeRate> fee_rates;
+    CFeeRate block_fee_rate{0};
+
+    if (confTarget > MAX_CONF_TARGET) {
+        err_message = strprintf("Confirmation target %s is above maximum limit of %s, mempool conditions might change and estimates above %s are unreliable.\n", confTarget, MAX_CONF_TARGET, MAX_CONF_TARGET);
+        return CFeeRate(0);
+    }
+
+    if (!mempool.GetLoadTried()) {
+        err_message = "Mempool not finished loading, can't get accurate fee rate estimate.";
+        return CFeeRate(0);
+    }
+    if (!force) {
+        cached_fee = cache.get(confTarget);
+    }
+
+    if (!cached_fee) {
+        std::map<CFeeRate, uint64_t> mempool_fee_stats;
+        // Always get stats for MAX_CONF_TARGET blocks (3) because current algo
+        // fast enough to run that far while we're locked and in here
+        mempool_fee_stats = GetCustomBlockFeeRateHistogram(chainstate, &mempool, DEFAULT_BLOCK_MAX_WEIGHT * MAX_CONF_TARGET);
+        if (mempool_fee_stats.empty()) {
+            err_message = "No transactions available in the mempool yet.";
+            return CFeeRate(0);
+        }
+        fee_rates = EstimateBlockFeeRatesWithMempool(mempool_fee_stats, MAX_CONF_TARGET);
+        cache.update(fee_rates);
+        block_fee_rate = fee_rates[confTarget];
+    } else {
+        block_fee_rate = *cached_fee;
+    }
+
+    if (block_fee_rate == CFeeRate(0)) {
+        err_message = "Insufficient mempool transactions to perform an estimate.";
+    }
+    return block_fee_rate;
+}
+
+std::map<uint64_t, CFeeRate> MemPoolPolicyEstimator::EstimateBlockFeeRatesWithMempool(
+    const std::map<CFeeRate, uint64_t>& mempool_fee_stats, unsigned int confTarget) const
+{
+    std::map<uint64_t, CFeeRate> fee_rates;
+    if (mempool_fee_stats.empty()) return fee_rates;
+
+    auto rstart = mempool_fee_stats.rbegin();
+    auto rcur = mempool_fee_stats.rbegin();
+    auto rend = mempool_fee_stats.rend();
+
+    size_t block_number{1};
+    size_t block_weight{0};
+
+    while (block_number <= confTarget && rcur != rend) {
+        size_t bin_weight = rcur->second * WITNESS_SCALE_FACTOR;
+        block_weight += bin_weight;
+        auto next_rcur = std::next(rcur);
+        if (block_weight >= DEFAULT_BLOCK_MAX_WEIGHT || next_rcur == rend) {
+            fee_rates[block_number] = CalculateMedianFeeRate(rstart, rcur);
+            block_number++;
+            block_weight = 0;
+            rstart = next_rcur;
+        }
+        rcur = next_rcur;
+    }
+    return fee_rates;
+}
+
+CFeeRate MemPoolPolicyEstimator::CalculateMedianFeeRate(
+    std::map<CFeeRate, uint64_t>::const_reverse_iterator rstart,
+    std::map<CFeeRate, uint64_t>::const_reverse_iterator rend) const
+{
+    unsigned int total_weight = 0;
+    std::vector<CFeeRate> feeRates;
+    for (auto rit = rstart; rit != rend; ++rit) {
+        total_weight += rit->second * WITNESS_SCALE_FACTOR;
+        feeRates.push_back(rit->first);
+    }
+
+    // Not enough info to provide a decent estimate
+    if (total_weight < (DEFAULT_BLOCK_MAX_WEIGHT / 2)) {
+        return CFeeRate(0);
+    }
+
+    // Calculate median from the collected fee rates
+    size_t size = feeRates.size();
+    if (size % 2 == 0) {
+        auto mid_fee1 = feeRates[size / 2 - 1].GetFeePerK();
+        auto mid_fee2 = feeRates[size / 2].GetFeePerK();
+        return CFeeRate((mid_fee1 + mid_fee2) / 2);
+    } else {
+        return feeRates[size / 2];
+    }
+}

src/policy/mempool_fees.h

@@ -0,0 +1,120 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_POLICY_MEMPOOL_FEES_H
+#define BITCOIN_POLICY_MEMPOOL_FEES_H
+
+#include <chrono>
+#include <map>
+#include <optional>
+#include <shared_mutex>
+#include <string>
+
+#include <logging.h>
+#include <policy/feerate.h>
+
+class Chainstate;
+class CTxMemPool;
+
+// Fee rate estimates above this confirmation target are not reliable,
+// mempool condition might likely change.
+static const unsigned int MAX_CONF_TARGET{3};
+
+/**
+ * CachedMempoolEstimates holds a cache of recent mempool-based fee estimates.
+ * Running the block-building algorithm multiple times is undesriable due to
+ * locking.
+ */
+struct CachedMempoolEstimates {
+private:
+    // shared_mutex allows for multiple concurrent reads, but only a single update
+    mutable std::shared_mutex cache_mutex;
+    static constexpr std::chrono::seconds cache_life{30};
+    std::map<uint64_t, CFeeRate> estimates;
+    std::chrono::steady_clock::time_point last_updated;
+
+    bool isStale() const
+    {
+        std::shared_lock<std::shared_mutex> lock(cache_mutex);
+        return (last_updated + cache_life) < std::chrono::steady_clock::now();
+    }
+
+public:
+    CachedMempoolEstimates() : last_updated(std::chrono::steady_clock::now() - cache_life - std::chrono::seconds(1)) {}
+    CachedMempoolEstimates(const CachedMempoolEstimates&) = delete;
+    CachedMempoolEstimates& operator=(const CachedMempoolEstimates&) = delete;
+
+    std::optional<CFeeRate> get(uint64_t number_of_blocks) const
+    {
+        std::shared_lock<std::shared_mutex> lock(cache_mutex);
+        if (isStale()) return std::nullopt;
+        LogPrint(BCLog::MEMPOOL, "CachedMempoolEstimates : cache is not stale, using cached value\n");
+
+        auto it = estimates.find(number_of_blocks);
+        if (it != estimates.end()) {
+            return it->second;
+        }
+        return std::nullopt;
+    }
+
+    void update(const std::map<uint64_t, CFeeRate>& newEstimates)
+    {
+        std::unique_lock<std::shared_mutex> lock(cache_mutex);
+        // Overwrite the entire map with the new data to avoid old
+        // estimates remaining.
+        estimates = newEstimates;
+        last_updated = std::chrono::steady_clock::now();
+        LogPrint(BCLog::MEMPOOL, "CachedMempoolEstimates: updated cache\n");
+    }
+};
+
+/**
+ * MemPoolPolicyEstimator estimates the fee rate that a tx should pay
+ * to be included in a confirmation target based on the mempool
+ * txs and their fee rates.
+ *
+ * The estimator works by generating template block up to a given confirmation target and then calculate the median
+ * fee rate of the txs in the confirmation target block as the approximate fee rate that a tx will pay to
+ * likely be included in the block.
+ */
+class MemPoolPolicyEstimator
+{
+public:
+    MemPoolPolicyEstimator();
+
+    ~MemPoolPolicyEstimator() = default;
+
+    /**
+     * Estimate the fee rate from mempool txs data given a confirmation target.
+     *
+     * @param[in] chainstate The reference to the active chainstate.
+     * @param[in] mempool The reference to the mempool from which we will estimate the fee rate.
+     * @param[in] confTarget The confirmation target of transactions.
+     * @param[out] err_message  optional error message.
+     * @return The estimated fee rate.
+     */
+    CFeeRate EstimateFeeWithMemPool(Chainstate& chainstate, const CTxMemPool& mempool, unsigned int confTarget, const bool force, std::string& err_message) const;
+
+private:
+    mutable CachedMempoolEstimates cache;
+    /**
+     * Calculate the fee rate estimate for blocks of txs up to num_blocks.
+     *
+     * @param[in] mempool_fee_stats The mempool fee statistics (fee rate and size).
+     * @param[in] num_blocks The numbers of blocks to calculate fees for.
+     * @return The fee rate estimate in satoshis per kilobyte.
+     */
+    std::map<uint64_t, CFeeRate> EstimateBlockFeeRatesWithMempool(const std::map<CFeeRate, uint64_t>& mempool_fee_stats, unsigned int num_blocks) const;
+
+    /**
+     * Calculate the median fee rate for a range of txs in the mempool.
+     *
+     * @param[in] start_it The iterator pointing to the beginning of the range.
+     * @param[in] end_it The iterator pointing to the end of the range.
+     * @return The median fee rate.
+     */
+    CFeeRate CalculateMedianFeeRate(std::map<CFeeRate, uint64_t>::const_reverse_iterator start_it, std::map<CFeeRate, uint64_t>::const_reverse_iterator end_it) const;
+};
+#endif // BITCOIN_POLICY_MEMPOOL_FEES_H