Add automated prestate tracer tests

system_tests/arbos_upgrade_test.go

@@ -184,6 +184,8 @@ func TestArbos11To32UpgradeWithMcopy(t *testing.T) {
 	if blockSeq.Hash() != blockReplica.Hash() {
 		t.Errorf("expected sequencer and replica to have same block hash, got %v and %v", blockSeq.Hash(), blockReplica.Hash())
 	}
+
+	AutomatedPrestateTracerTest(t, builder.L2)
 }
 
 func TestArbos11To32UpgradeWithCalldata(t *testing.T) {

system_tests/debugapi_test.go

@@ -1,21 +1,26 @@
 package arbtest
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"math/big"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/gasestimator"
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/google/go-cmp/cmp"
 
 	"github.com/offchainlabs/nitro/arbos/l2pricing"
 	"github.com/offchainlabs/nitro/arbos/retryables"
@@ -69,7 +74,7 @@
 type account struct {
 	Balance *hexutil.Big                `json:"balance,omitempty"`
 	Code    []byte                      `json:"code,omitempty"`
-	Nonce   uint64                      `json:"nonce,omitempty"`
+	Nonce   *uint64                     `json:"nonce,omitempty"`
 	Storage map[common.Hash]common.Hash `json:"storage,omitempty"`
 }
 type prestateTrace struct {
@@ -130,10 +135,10 @@
 	if !arbmath.BigEquals(result.Post[receiver].Balance.ToInt(), value) {
 		Fatal(t, "Unexpected final balance of receiver")
 	}
-	if result.Post[sender].Nonce != result.Pre[sender].Nonce+1 {
+	if *result.Post[sender].Nonce != *result.Pre[sender].Nonce+1 {
 		Fatal(t, "sender nonce increment wasn't registered")
 	}
-	if result.Post[receiver].Nonce != result.Pre[receiver].Nonce {
+	if *result.Post[receiver].Nonce != *result.Pre[receiver].Nonce {
 		Fatal(t, "receiver nonce shouldn't change")
 	}
 }
@@ -183,8 +188,8 @@
 	if _, ok := result.Pre[faucetAddr]; !ok {
 		Fatal(t, "Faucet account not found in the result of prestate tracer")
 	}
-	// Nonce shouldn't exist (in this case defaults to 0) in the Post map of the trace in DiffMode
-	if l2Tx.SkipAccountChecks() && result.Post[faucetAddr].Nonce != 0 {
+	// Nonce shouldn't exist in the Post map of the trace in DiffMode
+	if l2Tx.SkipAccountChecks() && result.Post[faucetAddr].Nonce != nil {
 		Fatal(t, "Faucet account's nonce should remain unchanged ")
 	}
 	if !arbmath.BigEquals(result.Pre[faucetAddr].Balance.ToInt(), oldBalance) {
@@ -299,4 +304,168 @@
 	if !arbmath.BigEquals(result.Post[user2Address].Balance.ToInt(), callValue) {
 		Fatal(t, "Unexpected final balance of User2")
 	}
+
+	AutomatedPrestateTracerTest(t, builder.L2)
+}
+
+type accountDump struct {
+	Balance       *big.Int
+	Nonce         uint64
+	Code          []byte
+	HashedStorage map[common.Hash]common.Hash
+}
+
+type stateDump struct {
+	HashedAccounts map[common.Hash]*accountDump
+}
+
+// This uses the trie iterator to dump the state at a given block number.
+// Ideally we'd use debug_dumpBlock, but it has a limit of 256 accounts,
+// and we don't support configuring preimage storage which is necessary for it.
+func dumpState(t *testing.T, client *TestClient, blockNumber uint64) *stateDump {
+	bc := client.ExecNode.Backend.BlockChain()
+	block := bc.GetBlockByNumber(blockNumber)
+	sdb, err := bc.StateAt(block.Root())
+	Require(t, err)
+	trieIt, err := sdb.GetTrie().NodeIterator(nil)
+	Require(t, err)
+	it := trie.NewIterator(trieIt)
+	dump := &stateDump{
+		HashedAccounts: make(map[common.Hash]*accountDump),
+	}
+	for it.Next() {
+		var data types.StateAccount
+		err = rlp.DecodeBytes(it.Value, &data)
+		Require(t, err)
+		account := &accountDump{
+			Balance:       data.Balance.ToBig(),
+			Nonce:         data.Nonce,
+			HashedStorage: make(map[common.Hash]common.Hash),
+		}
+		addrHash := common.BytesToHash(it.Key)
+		dump.HashedAccounts[addrHash] = account
+		if len(data.CodeHash) > 0 {
+			codeHash := common.BytesToHash(data.CodeHash)
+			if codeHash != types.EmptyCodeHash {
+				account.Code, err = sdb.Database().ContractCode(common.Address{}, codeHash)
+				Require(t, err)
+			}
+		}
+		if data.Root != types.EmptyRootHash {
+			storageTrie, err := trie.NewStateTrie(trie.StorageTrieID(block.Root(), addrHash, data.Root), sdb.Database().TrieDB())
+			Require(t, err)
+			storageIt, err := storageTrie.NodeIterator(nil)
+			Require(t, err)
+			storageIterator := trie.NewIterator(storageIt)
+			for storageIterator.Next() {
+				key := common.BytesToHash(storageIterator.Key)
+				_, value, _, err := rlp.Split(storageIterator.Value)
+				Require(t, err)
+				account.HashedStorage[key] = common.BytesToHash(value)
+			}
+		}
+	}
+	return dump
+}
+
+func AutomatedPrestateTracerTest(t *testing.T, client *TestClient) {
+	blockHeight, err := client.Client.BlockNumber(client.ctx)
+	Require(t, err)
+	runningState := dumpState(t, client, 1)
+	for block := uint64(2); block <= blockHeight; block++ {
+		var trace []prestateTrace
+		traceConfig := map[string]interface{}{
+			"tracer": "prestateTracer",
+			"tracerConfig": map[string]interface{}{
+				"diffMode": true,
+			},
+		}
+		err = client.Client.Client().CallContext(client.ctx, &trace, "debug_traceBlockByNumber", hexutil.Uint64(block), traceConfig)
+		Require(t, err)
+		for _, trace := range trace {
+			for addr, contents := range trace.Pre {
+				hashedAddr := crypto.Keccak256Hash(addr.Bytes())
+				runningAccount := runningState.HashedAccounts[hashedAddr]
+				if runningAccount == nil {
+					Fatal(t, "Account ", addr, " not found in previous state for prestate tracer test")
+				}
+				if contents.Balance == nil {
+					Fatal(t, "Balance of account ", addr, " was nil in prestate tracer")
+				}
+				if !arbmath.BigEquals(contents.Balance.ToInt(), runningAccount.Balance) {
+					Fatal(t, "Balance of account ", addr, " was ", runningAccount.Balance, " but tracer shows ", contents.Balance.ToInt())
+				}
+				if contents.Nonce == nil {
+					Fatal(t, "Nonce of account ", addr, " was nil in prestate tracer")
+				}
+				if *contents.Nonce != runningAccount.Nonce {
+					Fatal(t, "Nonce of account ", addr, " was ", runningAccount.Nonce, " but tracer shows ", contents.Nonce)
+				}
+				if (len(contents.Code) != 0) != (len(runningAccount.Code) != 0) {
+					Fatal(t, "Code presence of account ", addr, " was ", len(runningAccount.Code) != 0, " but tracer shows ", len(contents.Code) != 0)
+				}
+				if !bytes.Equal(contents.Code, runningAccount.Code) {
+					Fatal(t, "Code of account ", addr, " was incorrect in prestate tracer")
+				}
+				accountPostTrace, accountInPost := trace.Post[addr]
+				for key, val := range contents.Storage {
+					hashedKey := crypto.Keccak256Hash(key.Bytes())
+					previousVal := runningAccount.HashedStorage[hashedKey]
+					if val != previousVal {
+						Fatal(t, "Account ", addr, " storage key ", key, " was ", previousVal, " but tracer shows ", val)
+					}
+					if accountInPost {
+						_, storageInPost := accountPostTrace.Storage[key]
+						if !storageInPost {
+							// This slot was deleted
+							delete(runningAccount.HashedStorage, hashedKey)
+						}
+					}
+				}
+				if !accountInPost {
+					// This account was deleted
+					delete(runningState.HashedAccounts, hashedAddr)
+				}
+			}
+			for addr, contents := range trace.Post {
+				hashedAddr := crypto.Keccak256Hash(addr.Bytes())
+				runningAccount, hadAccount := runningState.HashedAccounts[hashedAddr]
+				preTrace, inPre := trace.Pre[addr]
+				if !hadAccount {
+					runningAccount = &accountDump{
+						HashedStorage: make(map[common.Hash]common.Hash),
+					}
+					runningState.HashedAccounts[hashedAddr] = runningAccount
+				} else if !inPre {
+					Fatal(t, "Account ", addr, " was not in tracer prestate but was in state")
+				}
+				if contents.Balance != nil {
+					runningAccount.Balance = contents.Balance.ToInt()
+				}
+				if contents.Nonce != nil {
+					runningAccount.Nonce = *contents.Nonce
+				}
+				if len(contents.Code) != 0 {
+					runningAccount.Code = contents.Code
+				}
+				for key, val := range contents.Storage {
+					hashedKey := crypto.Keccak256Hash(key.Bytes())
+					if inPre {
+						_, hadPreStorage := preTrace.Storage[key]
+						if !hadPreStorage && runningAccount.HashedStorage[hashedKey] != (common.Hash{}) {
+							Fatal(t, "Account ", addr, " storage key ", key, " was in state but not in prestate tracer")
+						}
+					}
+					runningAccount.HashedStorage[hashedKey] = val
+				}
+			}
+		}
+		expectedState := dumpState(t, client, block)
+		diff := cmp.Diff(expectedState, runningState, cmp.Comparer(func(x, y *big.Int) bool {
+			return x.Cmp(y) == 0
+		}))
+		if diff != "" {
+			Fatal(t, "State mismatch at block ", block, ":\n", diff)
+		}
+	}
 }

system_tests/log_subscription_test.go

@@ -56,4 +56,6 @@ func TestLogSubscription(t *testing.T) {
 	}
 	_, err = builder.L2.Client.BlockByHash(ctx, subscriptionLog.BlockHash)
 	Require(t, err)
+
+	AutomatedPrestateTracerTest(t, builder.L2)
 }

system_tests/program_norace_test.go

@@ -103,6 +103,8 @@ func validateBlockRange(
 	if !success {
 		Fatal(t)
 	}
+
+	AutomatedPrestateTracerTest(t, builder.L2)
 }
 
 func TestProgramEvmData(t *testing.T) {

system_tests/retryable_test.go

@@ -240,6 +240,8 @@ func TestSubmitRetryableImmediateSuccess(t *testing.T) {
 	if !arbmath.BigEquals(l2balance, callValue) {
 		Fatal(t, "Unexpected balance:", l2balance)
 	}
+
+	AutomatedPrestateTracerTest(t, builder.L2)
 }
 
 func testSubmitRetryableEmptyEscrow(t *testing.T, arbosVersion uint64) {