Flatdb/storage

src/Nethermind/Chains/sepolia.json

@@ -80,7 +80,12 @@
     "enode://7c9740e4d64674801fe62b76798d46778a038c49caebb15843d8c0f2b2f80d7ceba2585b4be366e6161988f81ddcfcd6fca98b5da52ae9a6f22c1b2a84b24a04@18.130.169.73:30303",
     "enode://0d5f5a5119236fa5f9e1c138e9658775e791cf3a30c461e0316d01dc910405cc375e0a6d18042b34d3c409238ecffb7fdcfa1513b0921b89b702f474a1d7425c@34.135.219.85:30303",
     "enode://9246d00bc8fd1742e5ad2428b80fc4dc45d786283e05ef6edbd9002cbc335d40998444732fbe921cb88e1d2c73d1b1de53bae6a2237996e9bfe14f871baf7066@18.168.182.86:30303",
-    "enode://ec66ddcf1a974950bd4c782789a7e04f8aa7110a72569b6e65fcd51e937e74eed303b1ea734e4d19cfaec9fbff9b6ee65bf31dcb50ba79acce9dd63a6aca61c7@52.14.151.177:30303"
+    "enode://ec66ddcf1a974950bd4c782789a7e04f8aa7110a72569b6e65fcd51e937e74eed303b1ea734e4d19cfaec9fbff9b6ee65bf31dcb50ba79acce9dd63a6aca61c7@52.14.151.177:30303",
+    	"enode://4e5e92199ee224a01932a377160aa432f31d0b351f84ab413a8e0a42f4f36476f8fb1cbe914af0d9aef0d51665c214cf653c651c4bbd9d5550a934f241f1682b@138.197.51.181:30303",
+	"enode://143e11fb766781d22d92a2e33f8f104cddae4411a122295ed1fdb6638de96a6ce65f5b7c964ba3763bba27961738fef7d3ecc739268f3e5e771fb4c87b6234ba@146.190.1.103:30303",
+	"enode://8b61dc2d06c3f96fddcbebb0efb29d60d3598650275dc469c22229d3e5620369b0d3dedafd929835fe7f489618f19f456fe7c0df572bf2d914a9f4e006f783a9@170.64.250.88:30303",
+	"enode://10d62eff032205fcef19497f35ca8477bea0eadfff6d769a147e895d8b2b8f8ae6341630c645c30f5df6e67547c03494ced3d9c5764e8622a26587b083b028e8@139.59.49.206:30303",
+	"enode://9e9492e2e8836114cc75f5b929784f4f46c324ad01daf87d956f98b3b6c5fcba95524d6e5cf9861dc96a2c8a171ea7105bb554a197455058de185fa870970c7c@138.68.123.152:30303"
   ],
   "accounts": {
     "0xa2A6d93439144FFE4D27c9E088dCD8b783946263": {"balance": "0xD3C21BCECCEDA1000000"},

src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.TestAccountAbstractionRpcBlockchain.cs

@@ -103,7 +103,6 @@ protected override IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolT
                     DbProvider,
                     BlockTree,
                     ReadOnlyTrieStore,
-                    ReadOnlyStorageTrieStore,
                     SpecProvider,
                     BlockValidator,
                     NoBlockRewards.Instance,
@@ -215,7 +214,7 @@ protected override BlockProcessor CreateBlockProcessor()
                     UserOperationSimulator[entryPoint] = new(
                         UserOperationTxBuilder[entryPoint],
                         ReadOnlyState,
-                        new ReadOnlyTxProcessingEnvFactory(DbProvider, ReadOnlyTrieStore, ReadOnlyStorageTrieStore, BlockTree, SpecProvider, LogManager),
+                        new ReadOnlyTxProcessingEnvFactory(DbProvider, ReadOnlyTrieStore, BlockTree, SpecProvider, LogManager),
                         EntryPointContractAbi,
                         entryPoint!,
                         WhitelistedPayamsters,

src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionPlugin.cs

@@ -124,7 +124,6 @@ private UserOperationSimulator UserOperationSimulator(Address entryPoint)
             ReadOnlyTxProcessingEnvFactory readOnlyTxProcessingEnvFactory = new(
                 getFromApi.DbProvider,
                 getFromApi.ReadOnlyTrieStore,
-                getFromApi.ReadOnlyStorageTrieStore,
                 getFromApi.BlockTree,
                 getFromApi.SpecProvider,
                 getFromApi.LogManager);

src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs

@@ -44,7 +44,6 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory
         IHeaderValidator? HeaderValidator { get; set; }
         IManualBlockProductionTrigger ManualBlockProductionTrigger { get; }
         IReadOnlyTrieStore? ReadOnlyTrieStore { get; set; }
-        IReadOnlyTrieStore? ReadOnlyStorageTrieStore { get; set; }
         IRewardCalculatorSource? RewardCalculatorSource { get; set; }
         /// <summary>
         /// PoS switcher for The Merge

src/Nethermind/Nethermind.Api/IInitConfig.cs

@@ -86,6 +86,8 @@ public enum DiagnosticMode
         [ConfigItem(Description = "Just scan and sum supply on all accounts")]
         VerifySupply,
         [ConfigItem(Description = "Verifies if full state is stored")]
-        VerifyTrie
+        VerifyTrie,
+        [ConfigItem(Description = "Verifies if full state is stored and stop processing")]
+        VerifyTrieBlocking
     }
 }

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -78,7 +78,6 @@ public IBlockchainBridge CreateBlockchainBridge()
             ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv = new(
                 _readOnlyDbProvider,
                 ReadOnlyTrieStore,
-                ReadOnlyStorageTrieStore,
                 readOnlyTree,
                 SpecProvider,
                 LogManager);
@@ -197,7 +196,6 @@ public ISealEngine SealEngine
         public ITransactionProcessor? TransactionProcessor { get; set; }
         public ITrieStore? TrieStore { get; set; }
         public IReadOnlyTrieStore? ReadOnlyTrieStore { get; set; }
-        public IReadOnlyTrieStore? ReadOnlyStorageTrieStore { get; set; }
         public ITxSender? TxSender { get; set; }
         public INonceManager? NonceManager { get; set; }
         public ITxPool? TxPool { get; set; }

src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs

@@ -85,7 +85,6 @@ protected override BlockProcessor CreateBlockProcessor()
                     new ReadOnlyTxProcessingEnv(
                         DbProvider,
                         new TrieStoreByPath(DbProvider.StateDb, LimboLogs.Instance).AsReadOnly(),
-                        new TrieStore(DbProvider.StateDb, LimboLogs.Instance).AsReadOnly(),
                         BlockTree, SpecProvider, LimboLogs.Instance));
 
                 GasLimitOverrideCache = new AuRaContractGasLimitOverride.Cache();

src/Nethermind/Nethermind.AuRa.Test/Contract/TxPriorityContractTests.cs

@@ -263,7 +263,7 @@ protected override TxPoolTxSource CreateTxPoolTxSource()
                 TxPoolTxSource txPoolTxSource = base.CreateTxPoolTxSource();
 
                 TxPriorityContract = new TxPriorityContract(AbiEncoder.Instance, TestItem.AddressA,
-                    new ReadOnlyTxProcessingEnv(DbProvider, TrieStore.AsReadOnly(), ReadOnlyStorageTrieStore, BlockTree, SpecProvider, LimboLogs.Instance));
+                    new ReadOnlyTxProcessingEnv(DbProvider, TrieStore.AsReadOnly(), BlockTree, SpecProvider, LimboLogs.Instance));
 
                 Priorities = new DictionaryContractDataStore<TxPriorityContract.Destination>(
                     new TxPriorityContract.DestinationSortedListContractDataStoreCollection(),

src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs

@@ -139,7 +139,6 @@ protected override BlockProcessor CreateBlockProcessor()
                 ReadOnlyTransactionProcessorSource = new ReadOnlyTxProcessingEnv(
                     DbProvider,
                     new TrieStoreByPath(DbProvider.StateDb, LimboLogs.Instance).AsReadOnly(),
-                    new TrieStore(DbProvider.StateDb, LimboLogs.Instance).AsReadOnly(),
                 BlockTree, SpecProvider,
                     LimboLogs.Instance);
                 RegisterContract = new RegisterContract(abiEncoder, ChainSpec.Parameters.Registrar, ReadOnlyTransactionProcessorSource);

src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs

@@ -281,13 +281,12 @@ protected override BlockProcessor CreateBlockProcessor()
                 IReadOnlyTxProcessorSource txProcessorSource = new ReadOnlyTxProcessingEnv(
                     DbProvider,
                     trieStore,
-                    storageTrieStore,
                     BlockTree,
                     SpecProvider,
                     LimboLogs.Instance);
 
                 VersionedTransactionPermissionContract transactionPermissionContract = new(AbiEncoder.Instance, _contractAddress, 1,
-                    new ReadOnlyTxProcessingEnv(DbProvider, trieStore, storageTrieStore, BlockTree, SpecProvider, LimboLogs.Instance), TransactionPermissionContractVersions, LimboLogs.Instance, SpecProvider);
+                    new ReadOnlyTxProcessingEnv(DbProvider, trieStore, BlockTree, SpecProvider, LimboLogs.Instance), TransactionPermissionContractVersions, LimboLogs.Instance, SpecProvider);
 
                 TxPermissionFilterCache = new PermissionBasedTxFilter.Cache();
                 PermissionBasedTxFilter = new PermissionBasedTxFilter(transactionPermissionContract, TxPermissionFilterCache, LimboLogs.Instance);

src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs

@@ -40,7 +40,7 @@ public void Prepared_block_contains_author_field()
         {
             IDb stateDb = new MemDb();
             IDb codeDb = new MemDb();
-            TrieStore trieStore = new(stateDb, LimboLogs.Instance);
+            TrieStoreByPath trieStore = new(stateDb, LimboLogs.Instance);
             IStateProvider stateProvider = new StateProvider(trieStore, codeDb, LimboLogs.Instance);
             ITransactionProcessor transactionProcessor = Substitute.For<ITransactionProcessor>();
             BlockProcessor processor = new(
@@ -70,7 +70,7 @@ public void Can_store_a_witness()
         {
             IDb stateDb = new MemDb();
             IDb codeDb = new MemDb();
-            var trieStore = new TrieStore(stateDb, LimboLogs.Instance);
+            TrieStoreByPath trieStore = new TrieStoreByPath(stateDb, LimboLogs.Instance);
 
             IStateProvider stateProvider = new StateProvider(trieStore, codeDb, LimboLogs.Instance);
             ITransactionProcessor transactionProcessor = Substitute.For<ITransactionProcessor>();
@@ -102,7 +102,7 @@ public void Recovers_state_on_cancel()
         {
             IDb stateDb = new MemDb();
             IDb codeDb = new MemDb();
-            TrieStore trieStore = new(stateDb, LimboLogs.Instance);
+            TrieStoreByPath trieStore = new(stateDb, LimboLogs.Instance);
             IStateProvider stateProvider = new StateProvider(trieStore, codeDb, LimboLogs.Instance);
             ITransactionProcessor transactionProcessor = Substitute.For<ITransactionProcessor>();
             BlockProcessor processor = new(

src/Nethermind/Nethermind.Blockchain.Test/BlockchainProcessorTests.cs

@@ -12,6 +12,7 @@
 using Nethermind.Core;
 using Nethermind.Core.Attributes;
 using Nethermind.Core.Crypto;
+using Nethermind.Core.Test;
 using Nethermind.Core.Test.Blockchain;
 using Nethermind.Specs;
 using Nethermind.Core.Test.Builders;
@@ -32,9 +33,10 @@ namespace Nethermind.Blockchain.Test
     [Parallelizable(ParallelScope.Self)]
     public class BlockchainProcessorTests
     {
+        private static readonly ILogManager _logManager = new NUnitLogManager(LogLevel.Trace);
         private class ProcessingTestContext
         {
-            private ILogManager _logManager = LimboLogs.Instance;
+            private ILogManager _logManager = BlockchainProcessorTests._logManager;
 
             private class BlockProcessorMock : IBlockProcessor
             {
@@ -191,7 +193,7 @@ public void RecoverData(Block block)
             private ILogger _logger;
             private Keccak _headBefore;
             private int _processingQueueEmptyFired;
-            public const int ProcessingWait = 2000;
+            public const int ProcessingWait = 3000;
 
             public ProcessingTestContext(bool startProcessor)
             {
@@ -202,10 +204,10 @@ public ProcessingTestContext(bool startProcessor)
                 MemDb stateDb = new();
                 IStateReader stateReader = Substitute.For<IStateReader>();
 
-                _blockTree = new BlockTree(blockDb, headersDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), MainnetSpecProvider.Instance, NullBloomStorage.Instance, LimboLogs.Instance);
+                _blockTree = new BlockTree(blockDb, headersDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), MainnetSpecProvider.Instance, NullBloomStorage.Instance, _logManager);
                 _blockProcessor = new BlockProcessorMock(_logManager, stateReader);
                 _recoveryStep = new RecoveryStepMock(_logManager);
-                _processor = new BlockchainProcessor(_blockTree, _blockProcessor, _recoveryStep, new StateReader(new TrieStore(stateDb, LimboLogs.Instance), new TrieStore(stateDb, LimboLogs.Instance), new MemDb(), LimboLogs.Instance), LimboLogs.Instance, BlockchainProcessor.Options.Default);
+                _processor = new BlockchainProcessor(_blockTree, _blockProcessor, _recoveryStep, stateReader, LimboLogs.Instance, BlockchainProcessor.Options.Default);
                 _resetEvent = new AutoResetEvent(false);
                 _queueEmptyResetEvent = new AutoResetEvent(false);
 
@@ -517,19 +519,29 @@ public void Can_process_fast_sync_transition()
                 .FullyProcessed(_block4D8).BecomesNewHead();
         }
 
-        [Test, Timeout(Timeout.MaxTestTime)]
+        [Test]
         public async Task Can_process_fast_sync()
         {
+            var logger = _logManager.GetClassLogger();
             BasicTestBlockchain testBlockchain = await BasicTestBlockchain.Create();
             await testBlockchain.BuildSomeBlocks(5);
 
-            When.ProcessingBlocks
-                .FullyProcessed(testBlockchain.BlockTree.FindBlock(0)).BecomesGenesis()
-                .SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(1))
-                .SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(2))
-                .SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(3))
-                .SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(4))
-                .FullyProcessed(testBlockchain.BlockTree.FindBlock(5)).BecomesNewHead();
+            logger.Info("start test a");
+            ProcessingTestContext a = When.ProcessingBlocks;
+            logger.Info("start test b");
+            ProcessingTestContext b = a.FullyProcessed(testBlockchain.BlockTree.FindBlock(0)).BecomesGenesis();
+            logger.Info("start test b");
+            ProcessingTestContext c = b.SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(1));
+            logger.Info("start test c");
+            ProcessingTestContext d = c.SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(2));
+            logger.Info("start test d");
+            ProcessingTestContext e = d.SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(3));
+            logger.Info("start test e");
+            ProcessingTestContext f = e.SuggestedWithoutProcessingAndMoveToMain(testBlockchain.BlockTree.FindBlock(4));
+            logger.Info("start test f");
+            ProcessingTestContext.AfterBlock g = f.FullyProcessed(testBlockchain.BlockTree.FindBlock(5));
+            logger.Info("start test g");
+            g.BecomesNewHead();
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/CopyTreeVisitorTests.cs

@@ -51,7 +51,7 @@ private static IPruningContext CopyDb(IPruningContext pruningContext, MemDb trie
         {
             LimboLogs logManager = LimboLogs.Instance;
             PatriciaTree trie = Build.A.Trie(trieDb).WithAccountsByIndex(0, 100).TestObject;
-            IStateReader stateReader = new StateReader(new TrieStore(trieDb, logManager), new TrieStore(trieDb, logManager), new MemDb(), logManager);
+            IStateReader stateReader = new StateReader(new TrieStoreByPath(trieDb, logManager), new MemDb(), logManager);
 
             using CopyTreeVisitor copyTreeVisitor = new(pruningContext, logManager);
             stateReader.RunTreeVisitor(copyTreeVisitor, trie.RootHash);

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPrunerTests.cs

@@ -145,7 +145,7 @@ public TestContext(bool successfulPruning, bool clearPrunedDb = false)
 
                 PatriciaTree trie = Build.A.Trie(TrieDb).WithAccountsByIndex(0, 100).TestObject;
                 _stateRoot = trie.RootHash;
-                StateReader = new StateReader(new TrieStore(TrieDb, LimboLogs.Instance), new TrieStore(TrieDb, LimboLogs.Instance), new MemDb(), LimboLogs.Instance);
+                StateReader = new StateReader(new TrieStoreByPath(TrieDb, LimboLogs.Instance), new MemDb(), LimboLogs.Instance);
                 FullPruningDb = new TestFullPruningDb(new RocksDbSettings("test", "test"), rocksDbFactory, successfulPruning, clearPrunedDb);
 
                 Pruner = new(FullPruningDb, PruningTrigger, new PruningConfig(), BlockTree, StateReader, LimboLogs.Instance);

src/Nethermind/Nethermind.Blockchain.Test/GenesisLoaderTests.cs

@@ -59,7 +59,7 @@ private Block GetGenesisBlock(string chainspecPath)
         ChainSpec chainSpec = LoadChainSpec(path);
         IDb stateDb = new MemDb();
         IDb codeDb = new MemDb();
-        TrieStore trieStore = new(stateDb, LimboLogs.Instance);
+        TrieStoreByPath trieStore = new(stateDb, LimboLogs.Instance);
         IStateProvider stateProvider = new StateProvider(trieStore, codeDb, LimboLogs.Instance);
         ISpecProvider specProvider = Substitute.For<ISpecProvider>();
         specProvider.GetSpec(Arg.Any<BlockHeader>()).Returns(Berlin.Instance);

src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs

@@ -48,7 +48,7 @@ public void Test()
                 specProvider,
                 NullBloomStorage.Instance,
                 LimboLogs.Instance);
-            TrieStore trieStore = new(
+            TrieStoreByPath trieStore = new(
                 dbProvider.RegisteredDbs[DbNames.State],
                 NoPruning.Instance,
                 Archive.Instance,
@@ -57,7 +57,7 @@ public void Test()
                 trieStore,
                 dbProvider.RegisteredDbs[DbNames.Code],
                 LimboLogs.Instance);
-            StateReader stateReader = new(trieStore, trieStore, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            StateReader stateReader = new(trieStore, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             StorageProvider storageProvider = new(trieStore, stateProvider, LimboLogs.Instance);
             BlockhashProvider blockhashProvider = new(blockTree, LimboLogs.Instance);
             VirtualMachine virtualMachine = new(

src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs

@@ -43,9 +43,9 @@ public class ReorgTests
         public void Setup()
         {
             IDbProvider memDbProvider = TestMemDbProvider.Init();
-            TrieStore trieStore = new(new MemDb(), LimboLogs.Instance);
+            TrieStoreByPath trieStore = new(new MemDb(), LimboLogs.Instance);
             StateProvider stateProvider = new(trieStore, memDbProvider.CodeDb, LimboLogs.Instance);
-            StateReader stateReader = new(trieStore, trieStore, memDbProvider.CodeDb, LimboLogs.Instance);
+            StateReader stateReader = new(trieStore, memDbProvider.CodeDb, LimboLogs.Instance);
             StorageProvider storageProvider = new(trieStore, stateProvider, LimboLogs.Instance);
             ChainLevelInfoRepository chainLevelInfoRepository = new(memDbProvider);
             ISpecProvider specProvider = MainnetSpecProvider.Instance;

src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs

@@ -154,10 +154,8 @@ public void Proper_transactions_selected(ProperTransactionsSelectedTestCase test
         {
             MemDb stateDb = new();
             MemDb codeDb = new();
-            TrieStore trieStore = new(stateDb, LimboLogs.Instance);
+            TrieStoreByPath trieStore = new(stateDb, LimboLogs.Instance);
             StateProvider stateProvider = new(trieStore, codeDb, LimboLogs.Instance);
-            StateReader stateReader =
-                new(new TrieStore(stateDb, LimboLogs.Instance), new TrieStore(stateDb, LimboLogs.Instance), codeDb, LimboLogs.Instance);
             ISpecProvider specProvider = Substitute.For<ISpecProvider>();
 
             void SetAccountStates(IEnumerable<Address> missingAddresses)

src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs

@@ -264,7 +264,7 @@ public void Proper_transactions_selected(TransactionSelectorTests.ProperTransact
         {
             MemDb stateDb = new();
             MemDb codeDb = new();
-            TrieStore trieStore = new(stateDb, LimboLogs.Instance);
+            TrieStoreByPath trieStore = new(stateDb, LimboLogs.Instance);
             StateProvider stateProvider = new(trieStore, codeDb, LimboLogs.Instance);
             IStorageProvider storageProvider = Substitute.For<IStorageProvider>();
             ISpecProvider specProvider = Substitute.For<ISpecProvider>();