Wallet can spend takes ages

tests/test_closing.py

@@ -18,6 +18,7 @@
 import re
 import subprocess
 import threading
+import time
 import unittest
 
 
@@ -4250,3 +4251,40 @@ def censoring_sendrawtx(r):
     bitcoind.generate_block(1)
     height = bitcoind.rpc.getblockchaininfo()['blocks']
     l1.daemon.wait_for_log(r"Low-priority anchorspend aiming for block {} \(feerate 7500\)".format(height + 12))
+
+
+@pytest.mark.slow_test
+def test_slow_startup_many_addresses(node_factory, bitcoind):
+    l1, l2 = node_factory.get_nodes(2)
+
+    # Use really high key, to make scanning work hard.
+    l1.stop()
+    l1.db_manip("INSERT INTO vars (name, intval) VALUES ('bip32_max_index', 10000);")
+    l1.start()
+
+    NUM_CHANS = 20
+    FUND = 100000
+
+    addr = l1.rpc.newaddr()['bech32']
+    bitcoind.rpc.sendtoaddress(addr, (FUND * NUM_CHANS + 1000000) / 10**8)
+
+    bitcoind.generate_block(1)
+    sync_blockheight(bitcoind, [l1])
+    l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+
+    channels = []
+    for _ in range(NUM_CHANS):
+        channels.append(l1.rpc.fundchannel(l2.info['id'], FUND)['channel_id'])
+        bitcoind.generate_block(1)
+        wait_for(lambda: all([c['state'] == 'CHANNELD_NORMAL' for c in l1.rpc.listpeerchannels()['channels']]))
+
+    for c in channels:
+        l1.rpc.close(c)
+
+    l1.stop()
+    start = time.time()
+    l1.start()
+    end = time.time()
+
+    # Normally less than a second: give it 10.
+    assert end - start < 10

wallet/test/run-db.c

@@ -299,6 +299,9 @@ const char *rune_is_ours(struct lightningd *ld UNNEEDED, const struct rune *rune
 /* Generated stub for rune_unique_id */
 u64 rune_unique_id(const struct rune *rune UNNEEDED)
 { fprintf(stderr, "rune_unique_id called!\n"); abort(); }
+/* Generated stub for scriptpubkey_hash */
+size_t scriptpubkey_hash(const u8 *out UNNEEDED)
+{ fprintf(stderr, "scriptpubkey_hash called!\n"); abort(); }
 /* Generated stub for sendpay_index_created */
 u64 sendpay_index_created(struct lightningd *ld UNNEEDED,
 			  const struct sha256 *payment_hash UNNEEDED,

wallet/test/run-wallet.c

@@ -956,6 +956,9 @@ const char *rune_is_ours(struct lightningd *ld UNNEEDED, const struct rune *rune
 /* Generated stub for rune_unique_id */
 u64 rune_unique_id(const struct rune *rune UNNEEDED)
 { fprintf(stderr, "rune_unique_id called!\n"); abort(); }
+/* Generated stub for scriptpubkey_hash */
+size_t scriptpubkey_hash(const u8 *out UNNEEDED)
+{ fprintf(stderr, "scriptpubkey_hash called!\n"); abort(); }
 /* Generated stub for serialize_onionpacket */
 u8 *serialize_onionpacket(
 	const tal_t *ctx UNNEEDED,

wallet/txfilter.c

@@ -7,7 +7,7 @@
 #include <wallet/txfilter.h>
 #include <wallet/wallet.h>
 
-static size_t scriptpubkey_hash(const u8 *out)
+size_t scriptpubkey_hash(const u8 *out)
 {
 	struct siphash24_ctx ctx;
 	siphash24_init(&ctx, siphash_seed());

wallet/txfilter.h

@@ -66,4 +66,8 @@ void outpointfilter_remove(struct outpointfilter *of,
 
 void memleak_scan_outpointfilter(struct htable *memtable,
 				 const struct outpointfilter *opf);
+
+/* Useful for other callers */
+size_t scriptpubkey_hash(const u8 *out);
+
 #endif /* LIGHTNING_WALLET_TXFILTER_H */

wallet/wallet.c

@@ -7,6 +7,7 @@
 #include <channeld/channeld_wiregen.h>
 #include <common/blockheight_states.h>
 #include <common/fee_states.h>
+#include <common/memleak.h>
 #include <common/onionreply.h>
 #include <common/trace.h>
 #include <db/bindings.h>
@@ -64,6 +65,111 @@ static enum state_change state_change_in_db(enum state_change s)
 	fatal("%s: %u is invalid", __func__, s);
 }
 
+/* We keep a hash of these, for fast lookup */
+struct wallet_address {
+	u32 index;
+	enum addrtype addrtype;
+	const u8 *scriptpubkey;
+};
+
+static const u8 *wallet_address_keyof(const struct wallet_address *waddr)
+{
+	return waddr->scriptpubkey;
+}
+
+static bool wallet_address_eq_scriptpubkey(const struct wallet_address *waddr,
+					   const u8 *scriptpubkey)
+{
+	return tal_arr_eq(waddr->scriptpubkey, scriptpubkey);
+}
+
+HTABLE_DEFINE_NODUPS_TYPE(struct wallet_address,
+			  wallet_address_keyof,
+			  scriptpubkey_hash,
+			  wallet_address_eq_scriptpubkey,
+			  wallet_address_htable);
+
+static void our_addresses_add(struct wallet_address_htable *our_addresses,
+			      u32 index,
+			      const u8 *scriptpubkey TAKES,
+			      enum addrtype addrtype)
+{
+	struct wallet_address *waddr = tal(our_addresses, struct wallet_address);
+
+	waddr->index = index;
+	waddr->addrtype = addrtype;
+	waddr->scriptpubkey = tal_dup_talarr(waddr, u8, scriptpubkey);
+	wallet_address_htable_add(our_addresses, waddr);
+}
+
+static void our_addresses_add_for_index(struct wallet *w, u32 i)
+{
+	struct ext_key ext;
+	enum addrtype addrtype;
+	const u8 *scriptpubkey;
+	const u32 flags = BIP32_FLAG_KEY_PUBLIC | BIP32_FLAG_SKIP_HASH;
+
+	if (bip32_key_from_parent(w->ld->bip32_base, i,
+				  flags, &ext) != WALLY_OK) {
+		abort();
+	}
+
+	/* If we don't know (prior to 24.11), just add all
+	 * possibilities. */
+	/* FIXME: We could deprecate P2SH once we don't see
+	 * any, since we stopped publishing them in 24.02 */
+	if (!wallet_get_addrtype(w, i, &addrtype)) {
+		scriptpubkey = scriptpubkey_p2wpkh_derkey(NULL, ext.pub_key);
+		our_addresses_add(w->our_addresses,
+				  i,
+				  take(scriptpubkey_p2sh(NULL, scriptpubkey)),
+				  ADDR_P2SH_SEGWIT);
+		our_addresses_add(w->our_addresses,
+				  i,
+				  take(scriptpubkey),
+				  ADDR_BECH32);
+		scriptpubkey = scriptpubkey_p2tr_derkey(NULL, ext.pub_key);
+		our_addresses_add(w->our_addresses,
+				  i,
+				  take(scriptpubkey),
+				  ADDR_P2TR);
+		return;
+	}
+
+	switch (addrtype) {
+		/* This doesn't happen */
+	case ADDR_P2SH_SEGWIT:
+		abort();
+	case ADDR_BECH32:
+	case ADDR_ALL:
+		scriptpubkey = scriptpubkey_p2wpkh_derkey(NULL, ext.pub_key);
+		our_addresses_add(w->our_addresses,
+				  i,
+				  take(scriptpubkey),
+				  ADDR_BECH32);
+		if (addrtype != ADDR_ALL)
+			return;
+	/* Fall thru */
+	case ADDR_P2TR:
+		scriptpubkey = scriptpubkey_p2tr_derkey(NULL, ext.pub_key);
+		our_addresses_add(w->our_addresses,
+				  i,
+				  take(scriptpubkey),
+				  ADDR_P2TR);
+		return;
+	}
+	abort();
+}
+
+static void our_addresses_init(struct wallet *w)
+{
+	w->our_addresses_maxindex = 0;
+	w->our_addresses = tal(w, struct wallet_address_htable);
+	wallet_address_htable_init(w->our_addresses);
+
+	our_addresses_add_for_index(w, w->our_addresses_maxindex);
+}
+
 static void outpointfilters_init(struct wallet *w)
 {
 	struct db_stmt *stmt;
@@ -114,6 +220,10 @@ struct wallet *wallet_new(struct lightningd *ld, struct timers *timers)
 	outpointfilters_init(wallet);
 	trace_span_end(wallet);
 
+	trace_span_start("our_addresses_init", wallet);
+	our_addresses_init(wallet);
+	trace_span_end(wallet);
+
 	db_commit_transaction(wallet->db);
 	return wallet;
 }
@@ -785,55 +895,20 @@ bool wallet_add_onchaind_utxo(struct wallet *w,
 bool wallet_can_spend(struct wallet *w, const u8 *script,
 		      u32 *index)
 {
-	struct ext_key ext;
 	u64 bip32_max_index;
-	size_t script_len = tal_bytelen(script);
-	u32 i;
-	bool output_is_p2sh;
-
-	/* If not one of these, can't be for us. */
-	if (is_p2sh(script, script_len, NULL))
-		output_is_p2sh = true;
-	else if (is_p2wpkh(script, script_len, NULL))
-		output_is_p2sh = false;
-	else if (is_p2tr(script, script_len, NULL))
-		output_is_p2sh = false;
-	else
-		return false;
+	const struct wallet_address *waddr;
 
+	/* Update hash table if we need to */
 	bip32_max_index = db_get_intvar(w->db, "bip32_max_index", 0);
-	for (i = 0; i <= bip32_max_index + w->keyscan_gap; i++) {
-		const u32 flags = BIP32_FLAG_KEY_PUBLIC | BIP32_FLAG_SKIP_HASH;
-		u8 *s;
+	while (w->our_addresses_maxindex < bip32_max_index + w->keyscan_gap)
+		our_addresses_add_for_index(w, ++w->our_addresses_maxindex);
 
-		if (bip32_key_from_parent(w->ld->bip32_base, i,
-					  flags, &ext) != WALLY_OK) {
-			abort();
-		}
-		s = scriptpubkey_p2wpkh_derkey(w, ext.pub_key);
-		if (output_is_p2sh) {
-			u8 *p2sh = scriptpubkey_p2sh(w, s);
-			tal_free(s);
-			s = p2sh;
-		}
-		if (!scripteq(s, script)) {
-			/* Try taproot output now */
-			tal_free(s);
-			s = scriptpubkey_p2tr_derkey(w, ext.pub_key);
-			if (!scripteq(s, script))
-				s = tal_free(s);
-		}
-		tal_free(s);
-		if (s) {
-			/* If we found a used key in the keyscan_gap we should
-			 * remember that. */
-			if (i > bip32_max_index)
-				db_set_intvar(w->db, "bip32_max_index", i);
-			*index = i;
-			return true;
-		}
-	}
-	return false;
+	waddr = wallet_address_htable_get(w->our_addresses, script);
+	if (!waddr)
+		return false;
+
+	*index = waddr->index;
+	return true;
 }
 
 s64 wallet_get_newindex(struct lightningd *ld, enum addrtype addrtype)
@@ -857,10 +932,10 @@ s64 wallet_get_newindex(struct lightningd *ld, enum addrtype addrtype)
 	return newidx;
 }
 
-enum addrtype wallet_get_addrtype(struct wallet *wallet, u64 idx)
+bool wallet_get_addrtype(struct wallet *wallet, u64 idx,
+			 enum addrtype *addrtype)
 {
 	struct db_stmt *stmt;
-	enum addrtype type;
 
 	stmt = db_prepare_v2(wallet->db,
 			     SQL("SELECT addrtype"
@@ -872,12 +947,12 @@ enum addrtype wallet_get_addrtype(struct wallet *wallet, u64 idx)
 	/* Unknown means prior to v24.11 */
 	if (!db_step(stmt)) {
 		tal_free(stmt);
-		return ADDR_P2TR|ADDR_BECH32;
+		return false;
 	}
 
-	type = wallet_addrtype_in_db(db_col_int(stmt, "addrtype"));
+	*addrtype = wallet_addrtype_in_db(db_col_int(stmt, "addrtype"));
 	tal_free(stmt);
-	return type;
+	return true;
 }
 
 static void wallet_shachain_init(struct wallet *wallet,
@@ -6420,6 +6495,7 @@ void wallet_memleak_scan(struct htable *memtable, const struct wallet *w)
 {
 	memleak_scan_outpointfilter(memtable, w->utxoset_outpoints);
 	memleak_scan_outpointfilter(memtable, w->owned_outpoints);
+	memleak_scan_htable(memtable, &w->our_addresses->raw);
 }
 
 struct issued_address_type *wallet_list_addresses(const tal_t *ctx, struct wallet *wallet,

wallet/wallet.h

@@ -48,6 +48,10 @@ struct wallet {
 	 * the blockchain. This is currently all P2WSH outputs */
 	struct outpointfilter *utxoset_outpoints;
 
+	/* Our issued wallet addresses.  We update on lookup. */
+	u32 our_addresses_maxindex;
+	struct wallet_address_htable *our_addresses;
+
 	/* How many keys should we look ahead at most? */
 	u64 keyscan_gap;
 };
@@ -274,6 +278,7 @@ static inline enum channel_state channel_state_in_db(enum channel_state s)
 /* /!\ This is a DB ENUM, please do not change the numbering of any
  * already defined elements (adding is ok) /!\ */
 enum addrtype {
+	ADDR_P2SH_SEGWIT = 1,
 	ADDR_BECH32 = 2,
 	ADDR_P2TR = 4,
 	ADDR_ALL = (ADDR_BECH32 + ADDR_P2TR)
@@ -291,6 +296,10 @@ static inline enum addrtype wallet_addrtype_in_db(enum addrtype t)
 	case ADDR_ALL:
 		BUILD_ASSERT(ADDR_ALL == 6);
 		return t;
+	/* This existed, but is NEVER placed into db */
+	case ADDR_P2SH_SEGWIT:
+		BUILD_ASSERT(ADDR_P2SH_SEGWIT == 1);
+		break;
 	}
 	fatal("%s: %u is invalid", __func__, t);
 }
@@ -601,8 +610,12 @@ s64 wallet_get_newindex(struct lightningd *ld, enum addrtype addrtype);
  * wallet_get_addrtype - get the address types for this key.
  * @wallet: (in) wallet
  * @keyidx: what address types we've published.
+ * @addrtype: filled in if true.
+ *
+ * If we don't know, returns false.
  */
-enum addrtype wallet_get_addrtype(struct wallet *w, u64 keyidx);
+bool wallet_get_addrtype(struct wallet *wallet, u64 idx,
+			 enum addrtype *addrtype);
 
 /**
  * wallet_shachain_add_hash -- wallet wrapper around shachain_add_hash

wallet/walletrpc.c

@@ -75,6 +75,7 @@ encode_pubkey_to_addr(const tal_t *ctx,
 		goto done;
 	}
 
+	case ADDR_P2SH_SEGWIT:
 	case ADDR_ALL:
 		abort();
 	}