From b22a4d0e53e1079088214e15cf3e0ec893c2821f Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Fri, 7 Jan 2022 13:41:35 -0500 Subject: [PATCH] node: parse config arg to compact tree on launch --- CHANGELOG.md | 5 + lib/blockchain/chain.js | 18 +++- lib/node/fullnode.js | 3 +- lib/node/rpc.js | 8 +- test/chain-tree-compaction-test.js | 159 ++++++++++++++++++++++++++--- 5 files changed, 175 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fc238cbed..f7e07dc0d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ usage by deleting historical data. It will keep up to the last 288 blocks worth of tree data on disk (7-8 tree intervals) exposing the node to a similar deep reorganization vulnerability as a chain-pruning node. +- `FullNode` parses new configuration option `--compact-tree` which will compact +the Urkel Tree only when the node first opens. This is the preferred method +because it will run before the node connects to the network and the processing +time will not affect peers. + ## v3.0.0 **When upgrading to this version of hsd you must pass diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index ca743077f2..4828e24c31 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -100,8 +100,16 @@ class Chain extends AsyncEmitter { this.setDeploymentState(state); - if (!this.options.spv) - await this.syncTree(); + if (this.options.compactTree) { + if (this.options.spv) + throw new Error('Cannot compact tree in SPV mode.'); + + // Will call syncTree() after compaction. + await this.compactTree(); + } else { + if (!this.options.spv) + await this.syncTree(); + } this.logger.memory(); @@ -3695,6 +3703,7 @@ class ChainOptions { this.maxOrphans = 20; this.checkpoints = true; this.chainMigrate = -1; + this.compactTree = false; if (options) this.fromOptions(options); @@ -3807,6 +3816,11 @@ class ChainOptions { this.chainMigrate = options.chainMigrate; } + if (options.compactTree != null) { + assert(typeof options.compactTree === 'boolean'); + this.compactTree = options.compactTree; + } + if (this.spv || this.memory) this.treePrefix = null; diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index e69b9a40c0..c68a1b9f32 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -66,7 +66,8 @@ class FullNode extends Node { entryCache: this.config.uint('entry-cache'), chainMigrate: this.config.uint('chain-migrate'), indexTX: this.config.bool('index-tx'), - indexAddress: this.config.bool('index-address') + indexAddress: this.config.bool('index-address'), + compactTree: this.config.bool('compact-tree') }); // Fee estimation. diff --git a/lib/node/rpc.js b/lib/node/rpc.js index f524b654d8..1935187e1f 100644 --- a/lib/node/rpc.js +++ b/lib/node/rpc.js @@ -1112,8 +1112,12 @@ class RPC extends RPCBase { if (this.chain.options.spv) throw new RPCError(errs.MISC_ERROR, 'Cannot compact tree in SPV mode.'); - if (this.chain.height < this.network.block.pruneAfterHeight) - throw new RPCError(errs.MISC_ERROR, 'Chain is too short for compacting.'); + if (this.chain.height < this.network.block.pruneAfterHeight) { + throw new RPCError( + errs.MISC_ERROR, + 'Chain is too short to compact tree.' + ); + } try { await this.chain.compactTree(); diff --git a/test/chain-tree-compaction-test.js b/test/chain-tree-compaction-test.js index 7bd8a25463..dd418e454b 100644 --- a/test/chain-tree-compaction-test.js +++ b/test/chain-tree-compaction-test.js @@ -11,6 +11,9 @@ const blockstore = require('../lib/blockstore'); const MemWallet = require('./util/memwallet'); const rules = require('../lib/covenants/rules'); const NameState = require('../lib/covenants/namestate'); +const Address = require('../lib/primitives/address'); +const FullNode = require('../lib/node/fullnode'); +const SPVNode = require('../lib/node/spvnode'); const network = Network.get('regtest'); const { @@ -20,6 +23,22 @@ const { } = network.names; describe('Tree Compacting', function() { + const oldKeepBlocks = network.block.keepBlocks; + const oldpruneAfterHeight = network.block.pruneAfterHeight; + + before(async () => { + // Copy the 1:8 ratio from mainnet + network.block.keepBlocks = treeInterval * 8; + + // Ensure old blocks are pruned right away + network.block.pruneAfterHeight = 1; + }); + + after(async () => { + network.block.keepBlocks = oldKeepBlocks; + network.block.pruneAfterHeight = oldpruneAfterHeight; + }); + for (const prune of [true, false]) { describe(`Chain: ${prune ? 'Pruning' : 'Archival'}`, function() { const prefix = path.join( @@ -74,20 +93,10 @@ describe('Tree Compacting', function() { wallet.addBlock(entry, block.txs); } } - - const oldKeepBlocks = network.block.keepBlocks; - const oldpruneAfterHeight = network.block.pruneAfterHeight; - let name, nameHash; const treeRoots = []; before(async () => { - // Copy the 1:8 ratio from mainnet - network.block.keepBlocks = treeInterval * 8; - - // Ensure old blocks are pruned right away - network.block.pruneAfterHeight = 1; - await blocks.ensure(); await blocks.open(); await chain.open(); @@ -99,9 +108,6 @@ describe('Tree Compacting', function() { await chain.close(); await blocks.close(); await fs.rimraf(prefix); - - network.block.keepBlocks = oldKeepBlocks; - network.block.pruneAfterHeight = oldpruneAfterHeight; }); it('should throw if chain is too short to compact', async () => { @@ -376,4 +382,131 @@ describe('Tree Compacting', function() { }); }); } + + describe('SPV', function() { + it('should refuse to compact tree via RPC', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + + const node = new SPVNode({ + prefix, + network: 'regtest', + memory: false + }); + + await node.ensure(); + await node.open(); + + await assert.rejects( + node.rpc.compactTree([]), + {message: 'Cannot compact tree in SPV mode.'} + ); + + await node.close(); + }); + }); + + describe('Full Node', function() { + it('should throw if chain is too short to compact on launch', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + + const node = new FullNode({ + prefix, + network: 'regtest', + memory: false, + compactTree: true + }); + + await node.ensure(); + + await assert.rejects( + node.open(), + {message: 'Chain is too short to compact tree.'} + ); + }); + + it('should throw if chain is too short to compact via RPC', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + + const node = new FullNode({ + prefix, + network: 'regtest', + memory: false + }); + + await node.ensure(); + await node.open(); + + await assert.rejects( + node.rpc.compactTree([]), + {message: 'Chain is too short to compact tree.'} + ); + + await node.close(); + }); + + it('should compact tree on launch', async () => { + const prefix = path.join( + os.tmpdir(), + `hsd-tree-compacting-test-${Date.now()}` + ); + const treePath = path.join(prefix, 'regtest', 'tree', '0000000001'); + + // Fresh start + let node = new FullNode({ + prefix, + network: 'regtest', + memory: false + }); + await node.ensure(); + await node.open(); + const fresh = await fs.stat(treePath); + + // Grow + const waiter = new Promise((resolve) => { + node.on('connect', (entry) => { + if (entry.height >= 300) + resolve(); + }); + }); + await node.rpc.generateToAddress( + [300, new Address().toString('regtest')] + ); + await waiter; + + // Tree has grown + const grown = await fs.stat(treePath); + assert(fresh.size < grown.size); + + // Relaunch with compaction argument + await node.close(); + node = new FullNode({ + prefix, + network: 'regtest', + memory: false, + compactTree: true + }); + await node.open(); + + // Tree is compacted + const compacted = await fs.stat(treePath); + assert(compacted.size < grown.size); + + // Bonus: since there are no namestate updates in this test, + // all the nodes committed to the tree during "growth" are identically + // empty. When we compact, only the original empty node will remain. + assert.strictEqual(fresh.size, compacted.size); + + // done + await node.close(); + }); + }); });