Merge pull request #5840 from knst/bp-v21-p17

backport: bitcoin#19124, #19800, #19922, #19963, #20126, #20159, #20168, #20276, #20385, #20688, #20876

contrib/testgen/gen_key_io_test_vectors.py

@@ -15,7 +15,6 @@
 from itertools import islice
 from base58 import b58encode_chk, b58decode_chk, b58chars
 import random
-from binascii import b2a_hex
 
 # key types
 PUBKEY_ADDRESS = 76
@@ -96,9 +95,7 @@ def gen_valid_vectors():
             rv, payload = valid_vector_generator(template)
             assert is_valid(rv)
             metadata = {x: y for x, y in zip(metadata_keys,template[3]) if y is not None}
-            hexrepr = b2a_hex(payload)
-            if isinstance(hexrepr, bytes):
-                hexrepr = hexrepr.decode('utf8')
+            hexrepr = payload.hex()
             yield (rv, hexrepr, metadata)
 
 def gen_invalid_base58_vector(template):

depends/README.md

@@ -106,6 +106,9 @@ The following can be set when running make: `make FOO=bar`
 - `NO_BDB`: Don't download/build/cache BerkeleyDB
 - `NO_SQLITE`: Don't download/build/cache SQLite
 - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP
+- `ALLOW_HOST_PACKAGES`: Packages that are missed in dependencies (due to `NO_*` option or
+  build script logic) are searched for among the host system packages using
+  `pkg-config`. It allows building with packages of other (newer) versions
 - `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP
 - `MULTIPROCESS`: build libmultiprocess (experimental, requires cmake)
 - `DEBUG`: Disable some optimizations and enable more runtime checking

test/functional/mempool_compatibility.py

@@ -13,15 +13,15 @@
 import os
 
 from test_framework.test_framework import BitcoinTestFramework
+from test_framework.wallet import MiniWallet
 
 
 class MempoolCompatibilityTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 2
-        self.wallet_names = [None, self.default_wallet_name]
+        self.wallet_names = [None]
 
     def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
         self.skip_if_no_previous_releases()
 
     def setup_network(self):
@@ -39,8 +39,15 @@ def setup_network(self):
     def run_test(self):
         self.log.info("Test that mempool.dat is compatible between versions")
 
-        old_node = self.nodes[0]
-        new_node = self.nodes[1]
+        old_node, new_node = self.nodes
+        new_wallet = MiniWallet(new_node)
+        new_wallet.generate(1)
+        new_node.generate(100)
+        # Sync the nodes to ensure old_node has the block that contains the coinbase that new_wallet will spend.
+        # Otherwise, because coinbases are only valid in a block and not as loose txns, if the nodes aren't synced
+        # unbroadcasted_tx won't pass old_node's `MemPoolAccept::PreChecks`.
+        self.connect_nodes(0, 1)
+        self.sync_blocks()
         recipient = old_node.getnewaddress()
         self.stop_node(1)
 
@@ -59,7 +66,7 @@ def run_test(self):
         assert old_tx_hash in new_node.getrawmempool()
 
         self.log.info("Add unbroadcasted tx to mempool on new node and shutdown")
-        unbroadcasted_tx_hash = new_node.sendtoaddress(recipient, 0.0001)
+        unbroadcasted_tx_hash = new_wallet.send_self_transfer(from_node=new_node)['txid']
         assert unbroadcasted_tx_hash in new_node.getrawmempool()
         mempool = new_node.getrawmempool(True)
         assert mempool[unbroadcasted_tx_hash]['unbroadcast']

test/functional/mempool_expiry.py

@@ -16,8 +16,8 @@
 from test_framework.util import (
     assert_equal,
     assert_raises_rpc_error,
-    find_vout_for_address,
 )
+from test_framework.wallet import MiniWallet
 
 DEFAULT_MEMPOOL_EXPIRY = 336  # hours
 CUSTOM_MEMPOOL_EXPIRY = 10  # hours
@@ -26,44 +26,50 @@
 class MempoolExpiryTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
-
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
+        self.setup_clean_chain = True
 
     def test_transaction_expiry(self, timeout):
         """Tests that a transaction expires after the expiry timeout and its
         children are removed as well."""
         node = self.nodes[0]
+        self.wallet = MiniWallet(node)
+
+        # Add enough mature utxos to the wallet so that all txs spend confirmed coins.
+        self.wallet.generate(4)
+        node.generate(100)
 
         # Send a parent transaction that will expire.
-        parent_address = node.getnewaddress()
-        parent_txid = node.sendtoaddress(parent_address, 1.0)
+        parent_txid = self.wallet.send_self_transfer(from_node=node)['txid']
+        parent_utxo = self.wallet.get_utxo(txid=parent_txid)
+        independent_utxo = self.wallet.get_utxo()
+
+        # Ensure the transactions we send to trigger the mempool check spend utxos that are independent of
+        # the transactions being tested for expiration.
+        trigger_utxo1 = self.wallet.get_utxo()
+        trigger_utxo2 = self.wallet.get_utxo()
 
         # Set the mocktime to the arrival time of the parent transaction.
         entry_time = node.getmempoolentry(parent_txid)['time']
         node.setmocktime(entry_time)
 
-        # Create child transaction spending the parent transaction
-        vout = find_vout_for_address(node, parent_txid, parent_address)
-        inputs = [{'txid': parent_txid, 'vout': vout}]
-        outputs = {node.getnewaddress(): 0.99}
-        child_raw = node.createrawtransaction(inputs, outputs)
-        child_signed = node.signrawtransactionwithwallet(child_raw)['hex']
-
-        # Let half of the timeout elapse and broadcast the child transaction.
+        # Let half of the timeout elapse and broadcast the child transaction spending the parent transaction.
         half_expiry_time = entry_time + int(60 * 60 * timeout/2)
         node.setmocktime(half_expiry_time)
-        child_txid = node.sendrawtransaction(child_signed)
+        child_txid = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=parent_utxo)['txid']
+        assert_equal(parent_txid, node.getmempoolentry(child_txid)['depends'][0])
         self.log.info('Broadcast child transaction after {} hours.'.format(
             timedelta(seconds=(half_expiry_time-entry_time))))
 
+        # Broadcast another (independent) transaction.
+        independent_txid = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=independent_utxo)['txid']
+
         # Let most of the timeout elapse and check that the parent tx is still
         # in the mempool.
         nearly_expiry_time = entry_time + 60 * 60 * timeout - 5
         node.setmocktime(nearly_expiry_time)
-        # Expiry of mempool transactions is only checked when a new transaction
-        # is added to the to the mempool.
-        node.sendtoaddress(node.getnewaddress(), 1.0)
+        # Broadcast a transaction as the expiry of transactions in the mempool is only checked
+        # when a new transaction is added to the mempool.
+        self.wallet.send_self_transfer(from_node=node, utxo_to_spend=trigger_utxo1)
         self.log.info('Test parent tx not expired after {} hours.'.format(
             timedelta(seconds=(nearly_expiry_time-entry_time))))
         assert_equal(entry_time, node.getmempoolentry(parent_txid)['time'])
@@ -72,9 +78,8 @@ def test_transaction_expiry(self, timeout):
         # has passed.
         expiry_time = entry_time + 60 * 60 * timeout + 5
         node.setmocktime(expiry_time)
-        # Expiry of mempool transactions is only checked when a new transaction
-        # is added to the to the mempool.
-        node.sendtoaddress(node.getnewaddress(), 1.0)
+        # Again, broadcast a transaction so the expiry of transactions in the mempool is checked.
+        self.wallet.send_self_transfer(from_node=node, utxo_to_spend=trigger_utxo2)
         self.log.info('Test parent tx expiry after {} hours.'.format(
             timedelta(seconds=(expiry_time-entry_time))))
         assert_raises_rpc_error(-5, 'Transaction not in mempool',
@@ -85,6 +90,11 @@ def test_transaction_expiry(self, timeout):
         assert_raises_rpc_error(-5, 'Transaction not in mempool',
                                 node.getmempoolentry, child_txid)
 
+        # Check that the independent tx is still in the mempool.
+        self.log.info('Test the independent tx not expired after {} hours.'.format(
+            timedelta(seconds=(expiry_time-half_expiry_time))))
+        assert_equal(half_expiry_time, node.getmempoolentry(independent_txid)['time'])
+
     def run_test(self):
         self.log.info('Test default mempool expiry timeout of %d hours.' %
                       DEFAULT_MEMPOOL_EXPIRY)

test/functional/mempool_spend_coinbase.py

@@ -12,47 +12,49 @@
 but less mature coinbase spends are NOT.
 """
 
-from decimal import Decimal
-
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.blocktools import create_raw_transaction
 from test_framework.util import assert_equal, assert_raises_rpc_error
+from test_framework.wallet import MiniWallet
 
 
 class MempoolSpendCoinbaseTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
-
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
+        self.setup_clean_chain = True
 
     def run_test(self):
+        wallet = MiniWallet(self.nodes[0])
+
+        wallet.generate(200)
         chain_height = self.nodes[0].getblockcount()
         assert_equal(chain_height, 200)
-        node0_address = self.nodes[0].getnewaddress()
 
         # Coinbase at height chain_height-100+1 ok in mempool, should
         # get mined. Coinbase at height chain_height-100+2 is
-        # is too immature to spend.
-        b = [ self.nodes[0].getblockhash(n) for n in range(101, 103) ]
-        coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ]
-        spends_raw = [ create_raw_transaction(self.nodes[0], txid, node0_address, amount=500 - Decimal('0.00001')) for txid in coinbase_txids ]
+        # too immature to spend.
+        b = [self.nodes[0].getblockhash(n) for n in range(101, 103)]
+        coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b]
+        utxo_101 = wallet.get_utxo(txid=coinbase_txids[0])
+        utxo_102 = wallet.get_utxo(txid=coinbase_txids[1])
 
-        spend_101_id = self.nodes[0].sendrawtransaction(spends_raw[0])
+        spend_101_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_101)["txid"]
 
         # coinbase at height 102 should be too immature to spend
-        assert_raises_rpc_error(-26,"bad-txns-premature-spend-of-coinbase", self.nodes[0].sendrawtransaction, spends_raw[1])
+        assert_raises_rpc_error(-26,
+                                "bad-txns-premature-spend-of-coinbase",
+                                lambda: wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102))
 
         # mempool should have just spend_101:
-        assert_equal(self.nodes[0].getrawmempool(), [ spend_101_id ])
+        assert_equal(self.nodes[0].getrawmempool(), [spend_101_id])
 
         # mine a block, spend_101 should get confirmed
         self.nodes[0].generate(1)
         assert_equal(set(self.nodes[0].getrawmempool()), set())
 
         # ... and now height 102 can be spent:
-        spend_102_id = self.nodes[0].sendrawtransaction(spends_raw[1])
-        assert_equal(self.nodes[0].getrawmempool(), [ spend_102_id ])
+        spend_102_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_102)["txid"]
+        assert_equal(self.nodes[0].getrawmempool(), [spend_102_id])
+
 
 if __name__ == '__main__':
     MempoolSpendCoinbaseTest().main()

test/functional/mining_getblocktemplate_longpoll.py

@@ -5,11 +5,13 @@
 """Test longpolling with getblocktemplate."""
 
 from decimal import Decimal
+import random
+import threading
 
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import get_rpc_proxy, random_transaction, wait_until
+from test_framework.util import get_rpc_proxy, wait_until
+from test_framework.wallet import MiniWallet
 
-import threading
 
 class LongpollThread(threading.Thread):
     def __init__(self, node):
@@ -29,45 +31,48 @@ def set_test_params(self):
         self.num_nodes = 2
         self.supports_cli = False
 
-    def skip_test_if_missing_module(self):
-        self.skip_if_no_wallet()
-
     def run_test(self):
         self.log.info("Warning: this test will take about 70 seconds in the best case. Be patient.")
+        self.log.info("Test that longpollid doesn't change between successive getblocktemplate() invocations if nothing else happens")
         self.nodes[0].generate(10)
         template = self.nodes[0].getblocktemplate()
         longpollid = template['longpollid']
-        # longpollid should not change between successive invocations if nothing else happens
         template2 = self.nodes[0].getblocktemplate()
         assert template2['longpollid'] == longpollid
 
-        # Test 1: test that the longpolling wait if we do nothing
+        self.log.info("Test that longpoll waits if we do nothing")
         thr = LongpollThread(self.nodes[0])
         thr.start()
         # check that thread still lives
         thr.join(5)  # wait 5 seconds or until thread exits
         assert thr.is_alive()
 
-        # Test 2: test that longpoll will terminate if another node generates a block
-        self.nodes[1].generate(1)  # generate a block on another node
+        miniwallets = [ MiniWallet(node) for node in self.nodes ]
+        self.log.info("Test that longpoll will terminate if another node generates a block")
+        miniwallets[1].generate(1)  # generate a block on another node
         # check that thread will exit now that new transaction entered mempool
         thr.join(5)  # wait 5 seconds or until thread exits
         assert not thr.is_alive()
 
-        # Test 3: test that longpoll will terminate if we generate a block ourselves
+        self.log.info("Test that longpoll will terminate if we generate a block ourselves")
         thr = LongpollThread(self.nodes[0])
         thr.start()
-        self.nodes[0].generate(1)  # generate a block on another node
+        miniwallets[0].generate(1)  # generate a block on own node
         thr.join(5)  # wait 5 seconds or until thread exits
         assert not thr.is_alive()
 
-        # Test 4: test that introducing a new transaction into the mempool will terminate the longpoll
+        # Add enough mature utxos to the wallets, so that all txs spend confirmed coins
+        self.nodes[0].generate(100)
+        self.sync_blocks()
+
+        self.log.info("Test that introducing a new transaction into the mempool will terminate the longpoll")
         thr = LongpollThread(self.nodes[0])
         thr.start()
         # generate a random transaction and submit it
         min_relay_fee = self.nodes[0].getnetworkinfo()["relayfee"]
-        # min_relay_fee is fee per 1000 bytes, which should be more than enough.
-        (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), min_relay_fee, Decimal("0.001"), 20)
+        fee_rate = min_relay_fee + Decimal('0.00000010') * random.randint(0,20)
+        miniwallets[0].send_self_transfer(from_node=random.choice(self.nodes),
+                                          fee_rate=fee_rate)
         # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned
 
         def check():

test/functional/p2p_blocksonly.py

@@ -57,29 +57,34 @@ def run_test(self):
             self.nodes[0].p2p.wait_for_tx(txid)
             assert_equal(self.nodes[0].getmempoolinfo()['size'], 1)
 
-        self.log.info('Check that txs from forcerelay peers are not rejected and relayed to others')
-        self.log.info("Restarting node 0 with forcerelay permission and blocksonly")
-        self.restart_node(0, ["-persistmempool=0", "-whitelist=127.0.0.1", "-whitelistforcerelay", "-blocksonly"])
+        self.log.info('Check that txs from peers with relay-permission are not rejected and relayed to others')
+        self.log.info("Restarting node 0 with relay permission and blocksonly")
+        self.restart_node(0, ["-persistmempool=0", "-whitelist=relay@127.0.0.1", "-blocksonly"])
         assert_equal(self.nodes[0].getrawmempool(), [])
         first_peer = self.nodes[0].add_p2p_connection(P2PInterface())
         second_peer = self.nodes[0].add_p2p_connection(P2PInterface())
         peer_1_info = self.nodes[0].getpeerinfo()[0]
-        assert_equal(peer_1_info['whitelisted'], True)
-        assert_equal(peer_1_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool', 'download'])
+        assert_equal(peer_1_info['permissions'], ['relay'])
         peer_2_info = self.nodes[0].getpeerinfo()[1]
-        assert_equal(peer_2_info['whitelisted'], True)
-        assert_equal(peer_2_info['permissions'], ['noban', 'forcerelay', 'relay', 'mempool', 'download'])
+        assert_equal(peer_2_info['permissions'], ['relay'])
         assert_equal(self.nodes[0].testmempoolaccept([sigtx])[0]['allowed'], True)
         txid = self.nodes[0].testmempoolaccept([sigtx])[0]['txid']
 
-        self.log.info('Check that the tx from forcerelay first_peer is relayed to others (ie.second_peer)')
+        self.log.info('Check that the tx from first_peer with relay-permission is relayed to others (ie.second_peer)')
         with self.nodes[0].assert_debug_log(["received getdata"]):
+            # Note that normally, first_peer would never send us transactions since we're a blocksonly node.
+            # By activating blocksonly, we explicitly tell our peers that they should not send us transactions,
+            # and Bitcoin Core respects that choice and will not send transactions.
+            # But if, for some reason, first_peer decides to relay transactions to us anyway, we should relay them to
+            # second_peer since we gave relay permission to first_peer.
+            # See https://github.com/bitcoin/bitcoin/issues/19943 for details.
             first_peer.send_message(msg_tx(FromHex(CTransaction(), sigtx)))
-            self.log.info('Check that the forcerelay peer is still connected after sending the transaction')
+            self.log.info('Check that the peer with relay-permission is still connected after sending the transaction')
             assert_equal(first_peer.is_connected, True)
+            self.bump_mocktime(60)
             second_peer.wait_for_tx(txid)
             assert_equal(self.nodes[0].getmempoolinfo()['size'], 1)
-        self.log.info("Forcerelay peer's transaction is accepted and relayed")
+        self.log.info("Relay-permission peer's transaction is accepted and relayed")
 
 
 if __name__ == '__main__':