merge coreth-0.8.5-rc.2

consensus/dummy/dynamic_fees.go

@@ -107,6 +107,18 @@ func CalcBaseFee(config *params.ChainConfig, parent *types.Header, timestamp uin
 	return newRollupWindow, baseFee, nil
 }
 
+// EstiamteNextBaseFee attempts to estimate the next base fee based on a block with [parent] being built at
+// [timestamp].
+// If [timestamp] is less than the timestamp of [parent], then it uses the same timestamp as parent.
+// Warning: This function should only be used in estimation and should not be used when calculating the canonical
+// base fee for a subsequent block.
+func EstimateNextBaseFee(config *params.ChainConfig, parent *types.Header, timestamp uint64) ([]byte, *big.Int, error) {
+	if timestamp < parent.Time {
+		timestamp = parent.Time
+	}
+	return CalcBaseFee(config, parent, timestamp)
+}
+
 // selectBigWithinBounds returns [value] if it is within the bounds:
 // lowerBound <= value <= upperBound or the bound at either end if [value]
 // is outside of the defined boundaries.

consensus/dummy/test_consensus.go

@@ -0,0 +1,53 @@
+// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package dummy
+
+import (
+	"github.com/ava-labs/subnet-evm/consensus"
+	"github.com/ava-labs/subnet-evm/core/state"
+	"github.com/ava-labs/subnet-evm/core/types"
+)
+
+type (
+	OnFinalizeAndAssembleCallbackType = func(header *types.Header, state *state.StateDB, txs []*types.Transaction) (err error)
+	OnExtraStateChangeType            = func(block *types.Block, statedb *state.StateDB) (err error)
+
+	ConsensusCallbacks struct {
+		OnFinalizeAndAssemble OnFinalizeAndAssembleCallbackType
+		OnExtraStateChange    OnExtraStateChangeType
+	}
+
+	DummyEngineCB struct {
+		cb *ConsensusCallbacks
+		DummyEngine
+	}
+)
+
+func NewTestConsensusCB(cb *ConsensusCallbacks) *DummyEngineCB {
+	return &DummyEngineCB{
+		cb: cb,
+	}
+}
+
+func (self *DummyEngineCB) Finalize(chain consensus.ChainHeaderReader, block *types.Block, parent *types.Header, state *state.StateDB, receipts []*types.Receipt) error {
+	if self.cb.OnExtraStateChange != nil {
+		err := self.cb.OnExtraStateChange(block, state)
+		if err != nil {
+			return err
+		}
+	}
+
+	return self.DummyEngine.Finalize(chain, block, parent, state, receipts)
+}
+
+func (self *DummyEngineCB) FinalizeAndAssemble(chain consensus.ChainHeaderReader, header *types.Header, parent *types.Header, state *state.StateDB, txs []*types.Transaction,
+	uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) {
+	if self.cb.OnFinalizeAndAssemble != nil {
+		err := self.cb.OnFinalizeAndAssemble(header, state, txs)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return self.DummyEngine.FinalizeAndAssemble(chain, header, parent, state, txs, uncles, receipts)
+}

core/blockchain.go

@@ -1080,15 +1080,15 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 	// Ensure the user sees large reorgs
 	if len(oldChain) > 0 && len(newChain) > 0 {
 		logFn := log.Info
-		msg := "Chain reorg detected"
+		msg := "Resetting chain preference"
 		if len(oldChain) > 63 {
-			msg = "Large chain reorg detected"
+			msg = "Large chain preference change detected"
 			logFn = log.Warn
 		}
 		logFn(msg, "number", commonBlock.Number(), "hash", commonBlock.Hash(),
 			"drop", len(oldChain), "dropfrom", oldChain[0].Hash(), "add", len(newChain), "addfrom", newChain[0].Hash())
 	} else {
-		log.Warn("Unlikely reorg (rewind to ancestor) occurred", "oldnum", oldHead.Number(), "oldhash", oldHead.Hash(), "newnum", newHead.Number(), "newhash", newHead.Hash())
+		log.Warn("Unlikely preference change (rewind to ancestor) occurred", "oldnum", oldHead.Number(), "oldhash", oldHead.Hash(), "newnum", newHead.Number(), "newhash", newHead.Hash())
 	}
 	// Insert the new chain(except the head block(reverse order)),
 	// taking care of the proper incremental order.
@@ -1293,3 +1293,74 @@ func (bc *BlockChain) reprocessState(current *types.Block, reexec uint64) error
 	}
 	return nil
 }
+
+// CleanBlockRootsAboveLastAccepted gathers the blocks that may have previously been in processing above the
+// last accepted block and wipes their block roots from disk to mark their tries as inaccessible.
+// This is used prior to pruning to ensure that all of the tries that may still be in processing are marked
+// as inaccessible and mirrors the handling of middle roots in the geth offline pruning implementation.
+// This is not strictly necessary, but maintains a soft assumption.
+func (bc *BlockChain) CleanBlockRootsAboveLastAccepted() error {
+	targetRoot := bc.LastAcceptedBlock().Root()
+
+	// Clean up any block roots above the last accepted block before we start pruning.
+	// Note: this takes the place of middleRoots in the geth implementation since we do not
+	// track processing block roots via snapshot journals in the same way.
+	processingRoots := bc.gatherBlockRootsAboveLastAccepted()
+	// If there is a block above the last accepted block with an identical state root, we
+	// explicitly remove it from the set to ensure we do not corrupt the last accepted trie.
+	delete(processingRoots, targetRoot)
+	for processingRoot := range processingRoots {
+		// Delete the processing root from disk to mark the trie as inaccessible (no need to handle this in a batch).
+		if err := bc.db.Delete(processingRoot[:]); err != nil {
+			return fmt.Errorf("failed to remove processing root (%s) preparing for offline pruning: %w", processingRoot, err)
+		}
+	}
+
+	return nil
+}
+
+// gatherBlockRootsAboveLastAccepted iterates forward from the last accepted block and returns a list of all block roots
+// for any blocks that were inserted above the last accepted block.
+// Given that we never insert a block into the chain unless all of its ancestors have been inserted, this should gather
+// all of the block roots for blocks inserted above the last accepted block that may have been in processing at some point
+// in the past and are therefore potentially still acceptable.
+// Note: there is an edge case where the node dies while the consensus engine is rejecting a branch of blocks since the
+// consensus engine will reject the lowest ancestor first. In this case, these blocks will not be considered acceptable in
+// the future.
+// Ex.
+//    A
+//  /   \
+// B     C
+// |
+// D
+// |
+// E
+// |
+// F
+//
+// The consensus engine accepts block C and proceeds to reject the other branch in order (B, D, E, F).
+// If the consensus engine dies after rejecting block D, block D will be deleted, such that the forward iteration
+// may not find any blocks at this height and will not reach the previously processing blocks E and F.
+func (bc *BlockChain) gatherBlockRootsAboveLastAccepted() map[common.Hash]struct{} {
+	blockRoots := make(map[common.Hash]struct{})
+	for height := bc.lastAccepted.NumberU64() + 1; ; height++ {
+		blockHashes := rawdb.ReadAllHashes(bc.db, height)
+		// If there are no block hashes at [height], then there should be no further acceptable blocks
+		// past this point.
+		if len(blockHashes) == 0 {
+			break
+		}
+
+		// Fetch the blocks and append their roots.
+		for _, blockHash := range blockHashes {
+			block := bc.GetBlockByHash(blockHash)
+			if block == nil {
+				continue
+			}
+
+			blockRoots[block.Root()] = struct{}{}
+		}
+	}
+
+	return blockRoots
+}

core/blockchain_test.go

@@ -4,10 +4,14 @@
 package core
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/ava-labs/subnet-evm/consensus/dummy"
 	"github.com/ava-labs/subnet-evm/core/rawdb"
+	"github.com/ava-labs/subnet-evm/core/state"
+	"github.com/ava-labs/subnet-evm/core/state/pruner"
+	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/core/vm"
 	"github.com/ava-labs/subnet-evm/ethdb"
 	"github.com/ava-labs/subnet-evm/params"
@@ -248,3 +252,84 @@ func TestCorruptSnapshots(t *testing.T) {
 		})
 	}
 }
+
+func TestBlockChainOfflinePruningUngracefulShutdown(t *testing.T) {
+	create := func(db ethdb.Database, chainConfig *params.ChainConfig, lastAcceptedHash common.Hash) (*BlockChain, error) {
+		// Import the chain. This runs all block validation rules.
+		blockchain, err := NewBlockChain(
+			db,
+			&CacheConfig{
+				TrieCleanLimit: 256,
+				TrieDirtyLimit: 256,
+				Pruning:        true, // Enable pruning
+				SnapshotLimit:  256,
+			},
+			chainConfig,
+			dummy.NewTestConsensusCB(&dummy.ConsensusCallbacks{
+				OnExtraStateChange: func(block *types.Block, sdb *state.StateDB) error {
+					sdb.SetState(common.HexToAddress("0xdeadbeef"), common.HexToHash("0xdeadbeef"), common.HexToHash("0xdeadbeef"))
+					return nil
+				},
+				OnFinalizeAndAssemble: func(header *types.Header, sdb *state.StateDB, txs []*types.Transaction) error {
+					sdb.SetState(common.HexToAddress("0xdeadbeef"), common.HexToHash("0xdeadbeef"), common.HexToHash("0xdeadbeef"))
+					return nil
+				},
+			}),
+			vm.Config{},
+			lastAcceptedHash,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		// Overwrite state manager, so that Shutdown is not called.
+		// This tests to ensure that the state manager handles an ungraceful shutdown correctly.
+		blockchain.stateManager = &wrappedStateManager{TrieWriter: blockchain.stateManager}
+
+		if lastAcceptedHash == (common.Hash{}) {
+			return blockchain, nil
+		}
+
+		tempDir := t.TempDir()
+		if err := blockchain.CleanBlockRootsAboveLastAccepted(); err != nil {
+			return nil, err
+		}
+		pruner, err := pruner.NewPruner(db, tempDir, 256)
+		if err != nil {
+			return nil, fmt.Errorf("offline pruning failed (%s, %d): %w", tempDir, 256, err)
+		}
+
+		targetRoot := blockchain.LastAcceptedBlock().Root()
+		if err := pruner.Prune(targetRoot); err != nil {
+			return nil, fmt.Errorf("failed to prune blockchain with target root: %s due to: %w", targetRoot, err)
+		}
+		// Re-initialize the blockchain after pruning
+		return NewBlockChain(
+			db,
+			&CacheConfig{
+				TrieCleanLimit: 256,
+				TrieDirtyLimit: 256,
+				Pruning:        true, // Enable pruning
+				SnapshotLimit:  256,
+			},
+			chainConfig,
+			dummy.NewTestConsensusCB(&dummy.ConsensusCallbacks{
+				OnExtraStateChange: func(block *types.Block, sdb *state.StateDB) error {
+					sdb.SetState(common.HexToAddress("0xdeadbeef"), common.HexToHash("0xdeadbeef"), common.HexToHash("0xdeadbeef"))
+					return nil
+				},
+				OnFinalizeAndAssemble: func(header *types.Header, sdb *state.StateDB, txs []*types.Transaction) error {
+					sdb.SetState(common.HexToAddress("0xdeadbeef"), common.HexToHash("0xdeadbeef"), common.HexToHash("0xdeadbeef"))
+					return nil
+				},
+			}),
+			vm.Config{},
+			lastAcceptedHash,
+		)
+	}
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			tt.testFunc(t, create)
+		})
+	}
+}

core/genesis.go

@@ -224,12 +224,14 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig
 
 	// Check config compatibility and write the config. Compatibility errors
 	// are returned to the caller unless we're already at block zero.
-	height := rawdb.ReadHeaderNumber(db, rawdb.ReadHeadHeaderHash(db))
-	if height == nil {
-		return newcfg, fmt.Errorf("missing block number for head header hash")
+	headBlock := rawdb.ReadHeadBlock(db)
+	if headBlock == nil {
+		return newcfg, fmt.Errorf("missing head block")
 	}
-	compatErr := storedcfg.CheckCompatible(newcfg, *height)
-	if compatErr != nil && *height != 0 && compatErr.RewindTo != 0 {
+	height := headBlock.NumberU64()
+	timestamp := headBlock.Time()
+	compatErr := storedcfg.CheckCompatible(newcfg, height, timestamp)
+	if compatErr != nil && height != 0 && compatErr.RewindTo != 0 {
 		return newcfg, compatErr
 	}
 	rawdb.WriteChainConfig(db, stored, newcfg)