From b0e8c2c406566448ef25125f0a503350d62addc5 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 16 Oct 2024 23:35:32 +0800 Subject: [PATCH 01/13] Perf/parallelize storage commit (#7605) Co-authored-by: Lukasz Rozmej --- .../Ethereum.Trie.Test/TrieTests.cs | 2 +- .../Store/PatriciaTreeBenchmarks.cs | 56 ++-- .../Builders/TestItem.Tree.cs | 12 +- .../Builders/TrieBuilder.cs | 2 +- .../Eip1186/ProofConverterTests.cs | 8 +- .../PatriciaTreeTests.cs | 14 +- .../Proofs/AccountProofCollectorTests.cs | 58 ++-- .../RecreateStateFromAccountRangesTests.cs | 2 +- .../Nethermind.State.Test/StateTreeTests.cs | 56 ++-- .../PersistentStorageProvider.cs | 24 +- .../Nethermind.State/StateProvider.cs | 4 +- src/Nethermind/Nethermind.State/WorldState.cs | 7 +- .../FastSync/StateSyncFeedHealingTests.cs | 4 +- .../FastSync/StateSyncFeedTests.cs | 26 +- .../FastSync/StateSyncFeedTestsBase.cs | 2 +- .../RangeQueryVisitorTests.cs | 8 +- .../RecreateStateFromAccountRangesTests.cs | 4 +- .../SnapSync/SnapServerTest.cs | 6 +- .../SyncServerTests.cs | 9 +- .../TrieScenarios.cs | 60 ++-- .../SnapSync/SnapProviderHelper.cs | 4 +- .../IFullTrieStoreExtensions.cs | 37 ++ .../Pruning/TreeStoreTests.cs | 261 +++++++------- .../Nethermind.Trie.Test/TrieNodeTests.cs | 12 +- .../Nethermind.Trie.Test/TrieTests.cs | 97 +++--- .../Nethermind.Trie.Test/VisitingTests.cs | 9 +- .../Nethermind.Trie/CachedTrieStore.cs | 4 +- .../Nethermind.Trie/PatriciaTree.cs | 19 +- .../Nethermind.Trie/PreCachedTrieStore.cs | 10 +- .../Pruning/BlockCommitPackage.cs | 20 ++ .../Pruning/IScopedTrieStore.cs | 5 +- .../Nethermind.Trie/Pruning/ITrieStore.cs | 41 ++- .../Nethermind.Trie/Pruning/NullCommitter.cs | 21 ++ .../Nethermind.Trie/Pruning/NullTrieStore.cs | 10 +- .../Pruning/ReadOnlyTrieStore.cs | 11 +- .../Pruning/ScopedTrieStore.cs | 4 +- .../Nethermind.Trie/Pruning/TrieStore.cs | 317 +++++++++--------- .../Nethermind.Trie/TrieStoreWithReadFlags.cs | 4 +- 38 files changed, 680 insertions(+), 570 deletions(-) create mode 100644 src/Nethermind/Nethermind.Trie.Test/IFullTrieStoreExtensions.cs create mode 100644 src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs diff --git a/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs b/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs index ab7f94d6220..aedee4459aa 100644 --- a/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs +++ b/src/Nethermind/Ethereum.Trie.Test/TrieTests.cs @@ -307,7 +307,7 @@ public void Delete_on_empty() { PatriciaTree patriciaTree = new PatriciaTree(_db, Keccak.EmptyTreeHash, false, true, NullLogManager.Instance); patriciaTree.Set(Keccak.Compute("1").Bytes, new byte[0]); - patriciaTree.Commit(0); + patriciaTree.Commit(); Assert.That(patriciaTree.RootHash, Is.EqualTo(PatriciaTree.EmptyTreeHash)); } diff --git a/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs b/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs index 5d568377914..07d700e9352 100644 --- a/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs +++ b/src/Nethermind/Nethermind.Benchmark/Store/PatriciaTreeBenchmarks.cs @@ -56,14 +56,14 @@ public class PatriciaTreeBenchmarks tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(1); + tree.Commit(); }), ("set_3_via_hash", tree => { tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); - tree.Commit(1); + tree.Commit(); }), ("set_3_delete_1", tree => { @@ -71,7 +71,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(1); + tree.Commit(); }), ("set_3_delete_2", tree => { @@ -80,7 +80,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(1); + tree.Commit(); }), ("set_3_delete_all", tree => { @@ -90,7 +90,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); - tree.Commit(1); + tree.Commit(); }), ("extension_read_full_match", tree => { @@ -99,7 +99,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extension_read_missing", tree => { @@ -108,7 +108,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extension_new_branch", tree => { @@ -117,7 +117,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), _account2); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extension_delete_missing", tree => { @@ -126,7 +126,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("extenson_create_new_extension", tree => { @@ -136,7 +136,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeaaaaaaaaaaaaaaaab11111111"), _account3); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_new_value", tree => { @@ -144,7 +144,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account1); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_no_change", tree => { @@ -152,7 +152,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), _account0); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_delete", tree => { @@ -160,7 +160,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_delete_missing", tree => { @@ -168,7 +168,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("1111111111111111111111111111111ddddddddddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_update_extension", tree => { @@ -176,7 +176,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000000000000000000000000000"), _account1); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_read", tree => { @@ -184,7 +184,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("1111111111111111111111111111111111111111111111111111111111111111")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("leaf_update_missing", tree => { @@ -192,7 +192,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("111111111111111111111111111111111111111111111111111111111ddddddd")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("branch_update_missing", tree => { @@ -201,7 +201,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), _account2); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("branch_read_missing", tree => { @@ -210,7 +210,7 @@ public class PatriciaTreeBenchmarks Account account = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222")); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), ("branch_delete_missing", tree => { @@ -219,7 +219,7 @@ public class PatriciaTreeBenchmarks tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), null); tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; - tree.Commit(1); + tree.Commit(); }), }; @@ -247,7 +247,7 @@ public void Setup() { tempTree.Set(_entries[i].Item1, _entries[i].Item2); } - tempTree.Commit(0); + tempTree.Commit(); _rootHash = tempTree.RootHash; _fullTree = new StateTree(); @@ -255,7 +255,7 @@ public void Setup() { _fullTree.Set(_entries[i].Item1, _entries[i].Item2); } - _fullTree.Commit(0); + _fullTree.Commit(); _uncommittedFullTree = new StateTree(); for (int i = 0; i < _entryCount; i++) @@ -330,23 +330,23 @@ public void InsertAndCommit() { tempTree.Set(_entries[i].Item1, _entries[i].Item2); } - tempTree.Commit(0); + tempTree.Commit(); } [Benchmark] public void InsertAndCommitRepeatedlyTimes() { - StateTree tempTree = new StateTree( - new TrieStore(new MemDb(), + TrieStore trieStore = new TrieStore(new MemDb(), Prune.WhenCacheReaches(1.MiB()), - Persist.IfBlockOlderThan(2), - NullLogManager.Instance), NullLogManager.Instance); + Persist.IfBlockOlderThan(2), NullLogManager.Instance); + StateTree tempTree = new StateTree(trieStore, NullLogManager.Instance); for (int i = 0; i < _largerEntryCount; i++) { if (i % 2000 == 0) { - tempTree.Commit(i / 2000); + using IBlockCommitter _ = trieStore.BeginBlockCommit(i / 2000); + tempTree.Commit(); } (bool isWrite, Hash256 address, Account value) = _largerEntriesAccess[i]; diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs index aa440794481..64b9d22d833 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs @@ -66,7 +66,7 @@ public static void FillStateTreeWithTestAccounts(StateTree stateTree) stateTree.Set(AccountsWithPaths[3].Path, AccountsWithPaths[3].Account); stateTree.Set(AccountsWithPaths[4].Path, AccountsWithPaths[4].Account); stateTree.Set(AccountsWithPaths[5].Path, AccountsWithPaths[5].Account); - stateTree.Commit(0); + stateTree.Commit(); } public static void FillStateTreeMultipleAccount(StateTree stateTree, int accountNumber) @@ -76,7 +76,7 @@ public static void FillStateTreeMultipleAccount(StateTree stateTree, int account Account acc = Build.An.Account.WithBalance((UInt256)i).TestObject; stateTree.Set(Keccak.Compute(i.ToBigEndianByteArray()), acc); } - stateTree.Commit(0); + stateTree.Commit(); } public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr) GetTrees(ITrieStore? store) @@ -92,13 +92,13 @@ public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr storageTree.Set(SlotsWithPaths[4].Path, SlotsWithPaths[4].SlotRlpValue, false); storageTree.Set(SlotsWithPaths[5].Path, SlotsWithPaths[5].SlotRlpValue, false); - storageTree.Commit(0); + storageTree.Commit(); var account = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; var stateTree = new StateTree(store.GetTrieStore(null), LimboLogs.Instance); stateTree.Set(AccountAddress0, account); - stateTree.Commit(0); + stateTree.Commit(); return (stateTree, storageTree, AccountAddress0); } @@ -117,13 +117,13 @@ public static (StateTree stateTree, StorageTree storageTree, Hash256 accountAddr false); } - storageTree.Commit(0); + storageTree.Commit(); var account = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; var stateTree = new StateTree(store, LimboLogs.Instance); stateTree.Set(AccountAddress0, account); - stateTree.Commit(0); + stateTree.Commit(); return (stateTree, storageTree, AccountAddress0); } diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs index 691f2debad1..3f2bdcf9688 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/TrieBuilder.cs @@ -36,7 +36,7 @@ public TrieBuilder WithAccountsByIndex(int start, int count) TestObjectInternal.Set(key.Bytes, value); } - TestObjectInternal.Commit(0); + TestObjectInternal.Commit(); TestObjectInternal.UpdateRootHash(); return this; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs index cb0b0637829..986fd030146 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Eip1186/ProofConverterTests.cs @@ -38,14 +38,14 @@ public void Storage_proofs_have_values_set_complex_3_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new byte[][] { a, b, c, d, e }); tree.Accept(accountProofCollector, tree.RootHash); @@ -72,14 +72,14 @@ public void Does_not_fail_when_proofs_are_longer_than_number_of_proofs_regressio storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new byte[][] { a }); tree.Accept(accountProofCollector, tree.RootHash); diff --git a/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs b/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs index 9f1108a9f33..0dd057454aa 100644 --- a/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs +++ b/src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs @@ -24,11 +24,11 @@ public void Create_commit_change_balance_get() Account account = new(1); StateTree stateTree = new(); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Account accountRestored = stateTree.Get(TestItem.AddressA); Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2)); @@ -41,11 +41,11 @@ public void Create_create_commit_change_balance_get() StateTree stateTree = new(); stateTree.Set(TestItem.AddressA, account); stateTree.Set(TestItem.AddressB, account); - stateTree.Commit(0); + stateTree.Commit(); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Account accountRestored = stateTree.Get(TestItem.AddressA); Assert.That(accountRestored.Balance, Is.EqualTo((UInt256)2)); @@ -58,7 +58,7 @@ public void Create_commit_reset_change_balance_get() Account account = new(1); StateTree stateTree = new(new TrieStore(db, LimboLogs.Instance), LimboLogs.Instance); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Hash256 rootHash = stateTree.RootHash; stateTree.RootHash = null; @@ -67,7 +67,7 @@ public void Create_commit_reset_change_balance_get() stateTree.Get(TestItem.AddressA); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); Assert.That(db.Keys.Count, Is.EqualTo(2)); } @@ -84,7 +84,7 @@ public void Commit_with_skip_root_should_skip_root(bool skipRoot, bool hasRoot) stateTree.Set(TestItem.AddressA, account); stateTree.UpdateRootHash(); Hash256 stateRoot = stateTree.RootHash; - stateTree.Commit(0, skipRoot); + stateTree.Commit(skipRoot); if (hasRoot) { diff --git a/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs b/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs index 2e82c29714a..503368f463a 100644 --- a/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs +++ b/src/Nethermind/Nethermind.State.Test/Proofs/AccountProofCollectorTests.cs @@ -53,7 +53,7 @@ public void Non_existing_account_is_valid_on_non_empty_tree_with_branch_without_ Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressC, new UInt256[] { 1, 2, 3 }); tree.Accept(accountProofCollector, tree.RootHash); @@ -75,7 +75,7 @@ public void Non_existing_account_is_valid_even_when_leaf_is_the_last_part_of_the Account account1 = Build.An.Account.WithBalance(1).TestObject; tree.Set(TestItem.AddressA, account1); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressC, new UInt256[] { 1, 2, 3 }); tree.Accept(accountProofCollector, tree.RootHash); @@ -106,7 +106,7 @@ public void Non_existing_account_is_valid_even_when_extension_on_the_way_is_not_ Account account = Build.An.Account.WithBalance(1).TestObject; tree.Set(c.AsSpan(), Rlp.Encode(account.WithChangedBalance(3))); tree.Set(d.AsSpan(), Rlp.Encode(account.WithChangedBalance(4))); - tree.Commit(0); + tree.Commit(); // now wer are looking for a trying to trick the code to think that the extension of c and d is a good match // if everything is ok the proof length of 1 is enough since the extension from the root is not matched @@ -128,7 +128,7 @@ public void Addresses_are_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -150,7 +150,7 @@ public void Balance_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -173,7 +173,7 @@ public void Code_hash_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -196,7 +196,7 @@ public void Nonce_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -219,7 +219,7 @@ public void Storage_root_is_correct() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -242,7 +242,7 @@ public void Proof_path_is_filled() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA); tree.Accept(accountProofCollector, tree.RootHash); @@ -260,7 +260,7 @@ public void Storage_proofs_length_is_as_expected() Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new[] { Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000001") }); tree.Accept(accountProofCollector, tree.RootHash); @@ -277,14 +277,14 @@ public void Storage_proofs_have_values_set() StorageTree storageTree = new(trieStore.GetTrieStore(TestItem.AddressA.ToAccountPath), Keccak.EmptyTreeHash, LimboLogs.Instance); storageTree.Set(UInt256.Zero, Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000")); storageTree.Set(UInt256.One, Bytes.FromHexString("0xab34000000000000000000000000000000000000000000000000000000000000000000000000000000")); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new[] { Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000001") }); tree.Accept(accountProofCollector, tree.RootHash); @@ -302,14 +302,14 @@ public void Storage_proofs_have_keys_set() StorageTree storageTree = new(trieStore.GetTrieStore(TestItem.AddressA.ToAccountPath), Keccak.EmptyTreeHash, LimboLogs.Instance); storageTree.Set(UInt256.Zero, Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000")); storageTree.Set(UInt256.One, Bytes.FromHexString("0xab34000000000000000000000000000000000000000000000000000000000000000000000000000000")); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); AccountProofCollector accountProofCollector = new(TestItem.AddressA, new[] { Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000000"), Bytes.FromHexString("0x0000000000000000000000000000000000000000000000000000000000000001") }); tree.Accept(accountProofCollector, tree.RootHash); @@ -332,14 +332,14 @@ public void Storage_proofs_have_values_set_complex_setup() storageTree.Set(Keccak.Compute(a).Bytes, Rlp.Encode(Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(b).Bytes, Rlp.Encode(Bytes.FromHexString("0xab34000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -371,14 +371,14 @@ public void Storage_proofs_have_values_set_complex_2_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -412,14 +412,14 @@ public void Storage_proofs_have_values_set_complex_3_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -451,14 +451,14 @@ public void Storage_proofs_when_values_are_missing_setup() storageTree.Set(Keccak.Compute(a).Bytes, Rlp.Encode(Bytes.FromHexString("0xab12000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -486,7 +486,7 @@ public void Shows_empty_values_when_account_is_missing() byte[] code = new byte[] { 1, 2, 3 }; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -519,14 +519,14 @@ public void Storage_proofs_have_values_set_selective_setup() storageTree.Set(Keccak.Compute(c).Bytes, Rlp.Encode(Bytes.FromHexString("0xab56000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(d).Bytes, Rlp.Encode(Bytes.FromHexString("0xab78000000000000000000000000000000000000000000000000000000000000000000000000000000"))); storageTree.Set(Keccak.Compute(e).Bytes, Rlp.Encode(Bytes.FromHexString("0xab9a000000000000000000000000000000000000000000000000000000000000000000000000000000"))); - storageTree.Commit(0); + storageTree.Commit(); byte[] code = new byte[] { 1, 2, 3 }; Account account1 = Build.An.Account.WithBalance(1).WithStorageRoot(storageTree.RootHash).TestObject; Account account2 = Build.An.Account.WithBalance(2).TestObject; tree.Set(TestItem.AddressA, account1); tree.Set(TestItem.AddressB, account2); - tree.Commit(0); + tree.Commit(); TreeDumper dumper = new(); tree.Accept(dumper, tree.RootHash); @@ -629,14 +629,14 @@ public void _Test_storage_failed_case(string historicallyFailingCase) TestContext.Out.WriteLine($"Set {Keccak.Compute(rawKey).Bytes.ToHexString()}"); storageTree.Set(addressWithStorage.StorageCells[j].Index, new byte[] { 1 }); storageTree.UpdateRootHash(); - storageTree.Commit(0); + storageTree.Commit(); } Account account = Build.An.Account.WithBalance((UInt256)accountIndex).WithStorageRoot(storageTree.RootHash).TestObject; tree.Set(addressWithStorage.Address, account); tree.UpdateRootHash(); - tree.Commit(0); + tree.Commit(); TreeDumper treeDumper = new(); tree.Accept(treeDumper, tree.RootHash); @@ -705,14 +705,14 @@ public void Chaotic_test() } storageTree.UpdateRootHash(); - storageTree.Commit(0); + storageTree.Commit(); account = account.WithChangedStorageRoot(storageTree.RootHash); tree.Set(addressesWithStorage[i].Address, account); } tree.UpdateRootHash(); - tree.Commit(0); + tree.Commit(); for (int i = 0; i < accountsCount; i++) { diff --git a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs index ff3f2558ef5..0b0ce1d8b2b 100644 --- a/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs +++ b/src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs @@ -93,7 +93,7 @@ public void Test01() tree.Set(TestItem.Tree.AccountsWithPaths[4].Path, TestItem.Tree.AccountsWithPaths[4].Account); tree.Set(TestItem.Tree.AccountsWithPaths[5].Path, TestItem.Tree.AccountsWithPaths[5].Account); - tree.Commit(0); + tree.Commit(); Assert.That(tree.RootHash, Is.EqualTo(_inputTree.RootHash)); Assert.That(db.Keys.Count, Is.EqualTo(6)); // we don't persist proof nodes (boundary nodes) diff --git a/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs b/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs index 70e0244ef4d..c333a35b5fa 100644 --- a/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs +++ b/src/Nethermind/Nethermind.State.Test/StateTreeTests.cs @@ -36,7 +36,7 @@ public void No_reads_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.ReadsCount, Is.EqualTo(0), "reads"); } @@ -48,7 +48,7 @@ public void Minimal_writes_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(5), "writes"); // branch, branch, two leaves (one is stored as RLP) } @@ -60,7 +60,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_2() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(7), "writes"); // extension, branch, leaf, extension, branch, 2x same leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(7), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(7), "encodings"); @@ -75,7 +75,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_3() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(4), "writes"); // extension, branch, 2x leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(4), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(4), "encodings"); @@ -91,7 +91,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_4() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), _account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes"); // extension, branch, 2x leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(1), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(1), "encodings"); @@ -108,7 +108,7 @@ public void Minimal_writes_when_setting_on_empty_scenario_5() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(0), "writes"); // extension, branch, 2x leaf Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(0), "hashes"); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(0), "encodings"); @@ -127,7 +127,7 @@ public void Scenario_traverse_extension_read_full_match() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); } @@ -143,7 +143,7 @@ public void Scenario_traverse_extension_read_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); } @@ -158,7 +158,7 @@ public void Scenario_traverse_extension_new_branching() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x543c960143a2a06b685d6b92f0c37000273e616bc23888521e7edf15ad06da46")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x543c960143a2a06b685d6b92f0c37000273e616bc23888521e7edf15ad06da46")); } @@ -174,7 +174,7 @@ public void Scenario_traverse_extension_delete_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xf99f1d3234bad8d63d818db36ff63eefc8916263e654db8b800d3bd03f6339a5")); } @@ -191,7 +191,7 @@ public void Scenario_traverse_extension_create_new_extension() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x0918112fc898173562441709a2c1cbedb80d1aaecaeadf2f3e9492eeaa568c67")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x0918112fc898173562441709a2c1cbedb80d1aaecaeadf2f3e9492eeaa568c67")); } @@ -205,7 +205,7 @@ public void Scenario_traverse_leaf_update_new_value() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xaa5c248d4b4b8c27a654296a8e0cc51131eb9011d9166fa0fca56a966489e169")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xaa5c248d4b4b8c27a654296a8e0cc51131eb9011d9166fa0fca56a966489e169")); } @@ -219,7 +219,7 @@ public void Scenario_traverse_leaf_update_no_change() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -233,7 +233,7 @@ public void Scenario_traverse_leaf_read_matching_leaf() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")); } @@ -247,7 +247,7 @@ public void Scenario_traverse_leaf_delete_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -261,7 +261,7 @@ public void Scenario_traverse_leaf_update_with_extension() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x215a4bab4cf2d5ebbaa59c82ae94c9707fcf4cc0ca1fe7e18f918e46db428ef9")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x215a4bab4cf2d5ebbaa59c82ae94c9707fcf4cc0ca1fe7e18f918e46db428ef9")); } @@ -276,7 +276,7 @@ public void Scenario_traverse_leaf_delete_matching_leaf() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -291,7 +291,7 @@ public void Scenario_traverse_leaf_read_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x491fbb33aaff22c0a7ff68d5c81ec114dddf89d022ccdee838a0e9d6cd45cab4")); } @@ -306,7 +306,7 @@ public void Scenario_traverse_branch_update_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0xc063af0bd3dd88320bc852ff8452049c42fbc06d1a69661567bd427572824cbf")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0xc063af0bd3dd88320bc852ff8452049c42fbc06d1a69661567bd427572824cbf")); } @@ -322,7 +322,7 @@ public void Scenario_traverse_branch_read_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); } @@ -337,7 +337,7 @@ public void Scenario_traverse_branch_delete_missing() tree.UpdateRootHash(); Hash256 rootHash = tree.RootHash; Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); - tree.Commit(0); + tree.Commit(); Assert.That(rootHash.ToString(true), Is.EqualTo("0x94a193704e99c219d9a21428eb37d6d2d71b3d2cea80c77ff0e201c0df70a283")); } @@ -349,7 +349,7 @@ public void Minimal_hashes_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(Trie.Metrics.TreeNodeHashCalculations, Is.EqualTo(5), "hashes"); // branch, branch, three leaves } @@ -361,7 +361,7 @@ public void Minimal_encodings_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(Trie.Metrics.TreeNodeRlpEncodings, Is.EqualTo(5), "encodings"); // branch, branch, three leaves } @@ -373,7 +373,7 @@ public void Zero_decodings_when_setting_on_empty() tree.Set(TestItem.AddressA, _account0); tree.Set(TestItem.AddressB, _account0); tree.Set(TestItem.AddressC, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(Trie.Metrics.TreeNodeRlpDecodings, Is.EqualTo(0), "decodings"); } @@ -386,7 +386,7 @@ public void No_writes_on_continues_update() tree.Set(TestItem.AddressA, _account1); tree.Set(TestItem.AddressA, _account2); tree.Set(TestItem.AddressA, _account3); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes"); // extension, branch, two leaves } @@ -397,11 +397,11 @@ public void No_writes_on_reverted_update() MemDb db = new(); StateTree tree = new(new TrieStore(db, LimboLogs.Instance).GetTrieStore(null), LimboLogs.Instance); tree.Set(TestItem.AddressA, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes before"); // extension, branch, two leaves tree.Set(TestItem.AddressA, _account1); tree.Set(TestItem.AddressA, _account0); - tree.Commit(0); + tree.Commit(); Assert.That(db.WritesCount, Is.EqualTo(1), "writes after"); // extension, branch, two leaves } @@ -447,7 +447,7 @@ public void Can_ask_about_root_hash_without_when_emptied() tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); tree.UpdateRootHash(); Assert.That(tree.RootHash, Is.EqualTo(PatriciaTree.EmptyTreeHash)); - tree.Commit(0); + tree.Commit(); Assert.That(tree.RootHash, Is.EqualTo(PatriciaTree.EmptyTreeHash)); } diff --git a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs index af65595b304..27f814222f8 100644 --- a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -318,17 +319,36 @@ private void SaveToTree(HashSet toUpdateRoots, Change change) /// Commit persistent storage trees /// /// Current block number - public void CommitTrees(long blockNumber) + public void CommitTrees(IBlockCommitter blockCommitter) { + // Note: These all runs in about 0.4ms. So the little overhead like attempting to sort the tasks + // may make it worst. Always check on mainnet. + + using ArrayPoolList commitTask = new ArrayPoolList(_storages.Count); foreach (KeyValuePair storage in _storages) { if (!_toUpdateRoots.Contains(storage.Key)) { continue; } - storage.Value.Commit(blockNumber); + + if (blockCommitter.TryRequestConcurrencyQuota()) + { + commitTask.Add(Task.Factory.StartNew((ctx) => + { + StorageTree st = (StorageTree)ctx; + st.Commit(); + blockCommitter.ReturnConcurrencyQuota(); + }, storage.Value)); + } + else + { + storage.Value.Commit(); + } } + Task.WaitAll(commitTask.ToArray()); + _toUpdateRoots.Clear(); // only needed here as there is no control over cached storage size otherwise _storages.Clear(); diff --git a/src/Nethermind/Nethermind.State/StateProvider.cs b/src/Nethermind/Nethermind.State/StateProvider.cs index c4689da5d04..e8afceafeb8 100644 --- a/src/Nethermind/Nethermind.State/StateProvider.cs +++ b/src/Nethermind/Nethermind.State/StateProvider.cs @@ -850,14 +850,14 @@ public void Reset(bool resizeCollections = true) _needsStateRootUpdate = false; } - public void CommitTree(long blockNumber) + public void CommitTree() { if (_needsStateRootUpdate) { RecalculateStateRoot(); } - _tree.Commit(blockNumber); + _tree.Commit(); } public static void CommitBranch() diff --git a/src/Nethermind/Nethermind.State/WorldState.cs b/src/Nethermind/Nethermind.State/WorldState.cs index c18fb219381..8df066b9292 100644 --- a/src/Nethermind/Nethermind.State/WorldState.cs +++ b/src/Nethermind/Nethermind.State/WorldState.cs @@ -174,8 +174,11 @@ public void DecrementNonce(Address address, UInt256 delta) public void CommitTree(long blockNumber) { - _persistentStorageProvider.CommitTrees(blockNumber); - _stateProvider.CommitTree(blockNumber); + using (IBlockCommitter committer = _trieStore.BeginBlockCommit(blockNumber)) + { + _persistentStorageProvider.CommitTrees(committer); + _stateProvider.CommitTree(); + } _persistentStorageProvider.StateRoot = _stateProvider.StateRoot; } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs index 3c7a2ea1cb2..4c4c3f6a91f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedHealingTests.cs @@ -71,7 +71,7 @@ public async Task HealBigSqueezedRandomTree() accounts[path] = account; } - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); int startingHashIndex = 0; int endHashIndex; @@ -115,7 +115,7 @@ public async Task HealBigSqueezedRandomTree() } } - dbContext.RemoteStateTree.Commit(blockNumber); + dbContext.RemoteStateTree.Commit(); } endHashIndex = startingHashIndex + 1000; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs index 07505497bea..8450719c629 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs @@ -76,7 +76,7 @@ public async Task Big_test((string Name, Action Setu .WithChangedStorageRoot(SetStorage(dbContext.RemoteTrieStore, i, TestItem.Addresses[i]).RootHash)); dbContext.RemoteStateTree.UpdateRootHash(); - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); ctx.Feed.FallAsleep(); ctx.Pool.WakeUpAll(); @@ -94,7 +94,7 @@ public async Task Big_test((string Name, Action Setu .WithChangedStorageRoot(SetStorage(dbContext.RemoteTrieStore, (byte)(i % 7), TestItem.Addresses[i]).RootHash)); dbContext.RemoteStateTree.UpdateRootHash(); - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); ctx.Feed.FallAsleep(); @@ -193,7 +193,7 @@ public async Task When_saving_root_goes_asleep() { DbContext dbContext = new(_logger, _logManager); dbContext.RemoteStateTree.Set(TestItem.KeccakA, Build.An.Account.TestObject); - dbContext.RemoteStateTree.Commit(0); + dbContext.RemoteStateTree.Commit(); dbContext.CompareTrees("BEGIN"); @@ -234,7 +234,7 @@ public async Task Can_download_with_moving_target((string Name, Action accountWithStorage = new(); for (int i = 1000; i < 10000; i += 1000) @@ -353,12 +353,12 @@ public void TestWithHugeTree() { storageTree.Set(TestItem.GetRandomKeccak(), TestItem.GetRandomKeccak().Bytes.ToArray()); } - storageTree.Commit(1); + storageTree.Commit(); var account = TestItem.GenerateRandomAccount().WithChangedStorageRoot(storageTree.RootHash); stateTree.Set(address, account); accountWithStorage.Add(new PathWithAccount() { Path = Keccak.Compute(address.Bytes), Account = account }); } - stateTree.Commit(1); + stateTree.Commit(); SnapServer server = new(store.AsReadOnly(), codeDb, CreateConstantStateRootTracker(true), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs index 95cde2d1a4e..5d48d0a61dc 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs @@ -702,10 +702,13 @@ public void GetNodeData_returns_cached_trie_nodes() Hash256 nodeKey = TestItem.KeccakA; TrieNode node = new(NodeType.Leaf, nodeKey, TestItem.KeccakB.Bytes); IScopedTrieStore scopedTrieStore = trieStore.GetTrieStore(null); - using (ICommitter committer = scopedTrieStore.BeginCommit(TrieType.State, 1, node)) + using (IBlockCommitter _ = trieStore.BeginBlockCommit(1)) { - TreePath path = TreePath.Empty; - committer.CommitNode(ref path, new NodeCommitInfo(node)); + using (ICommitter committer = scopedTrieStore.BeginCommit(node)) + { + TreePath path = TreePath.Empty; + committer.CommitNode(ref path, new NodeCommitInfo(node)); + } } stateDb.KeyExists(nodeKey).Should().BeFalse(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs b/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs index 5d3c8cde41d..5f8d57a1820 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/TrieScenarios.cs @@ -69,7 +69,7 @@ private static (string Name, Action Action)[] InitSc ("empty", (tree, _, codeDb) => { codeDb[Keccak.Compute(Code0).Bytes] = Code0; - tree.Commit(0); + tree.Commit(); }), ("set_3_via_address", (tree, stateDb, codeDb) => { @@ -77,7 +77,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, TestItem.AddressA, Account0); SetStorage(tree, stateDb, TestItem.AddressB, Account0); SetStorage(tree, stateDb, TestItem.AddressC, Account0); - tree.Commit(0); + tree.Commit(); }), ("storage_hash_and_code_hash_same", (tree, stateDb, codeDb) => { @@ -86,11 +86,11 @@ private static (string Name, Action Action)[] InitSc Hash256 account = new Hash256("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); StorageTree remoteStorageTree = new(stateDb.GetTrieStore(account), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree.Commit(0); + remoteStorageTree.Commit(); remoteStorageTree.UpdateRootHash(); codeDb[codeHash.Bytes] = code; tree.Set(account, AccountJustState0.WithChangedStorageRoot(remoteStorageTree.RootHash).WithChangedCodeHash(codeHash)); - tree.Commit(0); + tree.Commit(); }), ("storage_hash_and_code_hash_same_with_additional_account_of_same_storage_root", (tree, stateDb, codeDb) => { @@ -100,19 +100,19 @@ private static (string Name, Action Action)[] InitSc Hash256 account1 = new Hash256("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); StorageTree remoteStorageTree1 = new(stateDb.GetTrieStore(account1), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree1.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree1.Commit(0); + remoteStorageTree1.Commit(); remoteStorageTree1.UpdateRootHash(); Hash256 account2 = new Hash256("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); StorageTree remoteStorageTree2 = new(stateDb.GetTrieStore(account2), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree2.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree2.Commit(0); + remoteStorageTree2.Commit(); remoteStorageTree2.UpdateRootHash(); codeDb[codeHash.Bytes] = code; tree.Set(account1, AccountJustState0.WithChangedStorageRoot(remoteStorageTree1.RootHash)); tree.Set(account2, AccountJustState0.WithChangedStorageRoot(remoteStorageTree2.RootHash).WithChangedCodeHash(codeHash)); - tree.Commit(0); + tree.Commit(); }), ("storage_hash_and_code_hash_same_with_additional_account_of_same_code", (tree, stateDb, codeDb) => { @@ -123,20 +123,20 @@ private static (string Name, Action Action)[] InitSc new Hash256("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); StorageTree remoteStorageTree = new(stateDb.GetTrieStore(accountWithStorage), Keccak.EmptyTreeHash, LimboLogs.Instance); remoteStorageTree.Set((UInt256) 1, new byte[] {1}); - remoteStorageTree.Commit(0); + remoteStorageTree.Commit(); remoteStorageTree.UpdateRootHash(); codeDb[codeHash.Bytes] = code; tree.Set(new Hash256("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), AccountJustState0.WithChangedCodeHash(codeHash)); tree.Set(accountWithStorage, AccountJustState0.WithChangedStorageRoot(remoteStorageTree.RootHash).WithChangedCodeHash(codeHash)); - tree.Commit(0); + tree.Commit(); }), ("branch_with_same_accounts_at_different_addresses", (tree, _, codeDb) => { codeDb[Keccak.Compute(Code0).Bytes] = Code0; tree.Set(new Hash256("1baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), AccountJustState0); tree.Set(new Hash256("2baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), AccountJustState0); - tree.Commit(0); + tree.Commit(); }), ("set_3_delete_1", (tree, stateDb, codeDb) => { @@ -145,7 +145,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), Account0); SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), Account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); }), ("set_3_delete_2", (tree, stateDb, codeDb) => { @@ -155,7 +155,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), Account0); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); - tree.Commit(0); + tree.Commit(); }), ("set_3_delete_all", (tree, _, _) => { @@ -166,7 +166,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb0"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb1eeeeeb1"), null); tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000"), null); - tree.Commit(0); + tree.Commit(); }), ("extension_read_full_match", (tree, stateDb, codeDb) => { @@ -177,7 +177,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb11111111"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_read_missing", (tree, stateDb, codeDb) => { @@ -188,7 +188,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_new_branch", (tree, stateDb, codeDb) => { @@ -200,7 +200,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), Account2); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("just_state", (tree, _, _) => { @@ -209,7 +209,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeedddddddddddddddddddddddd"), AccountJustState2); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_delete_missing", (tree, stateDb, codeDb) => { @@ -220,7 +220,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("extension_create_new_extension", (tree, stateDb, codeDb) => { @@ -234,7 +234,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeaaaaaaaaaaaaaaaab11111111"), Account3); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_new_value", (tree, stateDb, codeDb) => { @@ -243,7 +243,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), Account1); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_no_change", (tree, stateDb, codeDb) => { @@ -252,7 +252,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), Account0); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_delete", (tree, _, _) => { @@ -260,7 +260,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_delete_missing", (tree, stateDb, codeDb) => { @@ -269,7 +269,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("1111111111111111111111111111111ddddddddddddddddddddddddddddddddd"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_update_extension", (tree, stateDb, codeDb) => { @@ -279,7 +279,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb00000000000000000000000000000000"), Account1); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_read", (tree, stateDb, codeDb) => { @@ -288,7 +288,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("1111111111111111111111111111111111111111111111111111111111111111"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("leaf_update_missing", (tree, stateDb, codeDb) => { @@ -297,7 +297,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("111111111111111111111111111111111111111111111111111111111ddddddd"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("branch_update_missing", (tree, stateDb, codeDb) => { @@ -309,7 +309,7 @@ private static (string Name, Action Action)[] InitSc SetStorage(tree, stateDb, new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), Account2); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("branch_read_missing", (tree, stateDb, codeDb) => { @@ -320,7 +320,7 @@ private static (string Name, Action Action)[] InitSc Account _ = tree.Get(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"))!; tree.UpdateRootHash(); Hash256 __ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }), ("branch_delete_missing", (tree, stateDb, codeDb) => { @@ -331,7 +331,7 @@ private static (string Name, Action Action)[] InitSc tree.Set(new Hash256("eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeb22222"), null); tree.UpdateRootHash(); Hash256 _ = tree.RootHash; - tree.Commit(0); + tree.Commit(); }) }; } @@ -354,7 +354,7 @@ private static StorageTree SetStorage(ITrieStore trieStore, Hash256 account) remoteStorageTree.Set((UInt256)1007, new byte[] { 7 }); remoteStorageTree.Set((UInt256)1008, new byte[] { 8 }); - remoteStorageTree.Commit(0); + remoteStorageTree.Commit(); return remoteStorageTree; } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs index ea9dfc1f70a..504d66fb884 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs @@ -76,7 +76,7 @@ public static (AddRangeResult result, bool moreChildrenToRight, List reorgBoundaryCount += e.BlockNumber; - trieStore.BeginCommit(TrieType.State, 1, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(1, trieNode).Dispose(); reorgBoundaryCount.Should().Be(0); - trieStore.BeginCommit(TrieType.State, 2, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(2, trieNode).Dispose(); reorgBoundaryCount.Should().Be(1); - trieStore.BeginCommit(TrieType.State, 3, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(3, trieNode).Dispose(); reorgBoundaryCount.Should().Be(3); - trieStore.BeginCommit(TrieType.State, 4, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(4, trieNode).Dispose(); reorgBoundaryCount.Should().Be(6); } @@ -154,12 +150,11 @@ public void Should_always_announce_zero_when_not_persisting() long reorgBoundaryCount = 0L; using TrieStore fullTrieStore = CreateTrieStore(); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); fullTrieStore.ReorgBoundaryReached += (_, e) => reorgBoundaryCount += e.BlockNumber; - trieStore.BeginCommit(TrieType.State, 1, trieNode).Dispose(); - trieStore.BeginCommit(TrieType.State, 2, trieNode).Dispose(); - trieStore.BeginCommit(TrieType.State, 3, trieNode).Dispose(); - trieStore.BeginCommit(TrieType.State, 4, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(1, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(2, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(3, trieNode).Dispose(); + fullTrieStore.BeginStateBlockCommit(4, trieNode).Dispose(); reorgBoundaryCount.Should().Be(0L); } @@ -202,9 +197,8 @@ public void Memory_with_two_nodes_is_correct() TrieNode trieNode2 = new(NodeType.Leaf, TestItem.KeccakB); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(true)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1234, null)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1234, null)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); @@ -234,7 +228,10 @@ public void Memory_with_concurrent_commits_is_correct() tree.Set(key, value.ToArray()); } - tree.Commit(0); + using (fullTrieStore.BeginBlockCommit(0)) + { + tree.Commit(); + } fullTrieStore.MemoryUsedByDirtyCache.Should().Be(_scheme == INodeStorage.KeyScheme.Hash ? 591672L : 661820L); fullTrieStore.CommittedNodesCount.Should().Be(1349); @@ -249,15 +246,14 @@ public void Memory_with_two_times_two_nodes_is_correct() TrieNode trieNode4 = new(NodeType.Leaf, TestItem.KeccakB); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(true)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 1234, trieNode2)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(1234, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); } - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 1235, trieNode2)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(1235, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode3)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode4)); @@ -286,21 +282,20 @@ public void Dispatcher_will_try_to_clear_memory() trieNode4.ResolveKey(null!, ref emptyPath, true); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(640)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1234, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1234, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1235, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1235, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode3)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode4)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1236, trieNode2)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(1236, trieNode2)) { } fullTrieStore.MemoryUsedByDirtyCache.Should().Be( trieNode1.GetMemorySize(false) + ExpectedPerNodeKeyMemorySize + @@ -325,15 +320,14 @@ public void Dispatcher_will_try_to_clear_memory_the_soonest_possible() trieNode4.ResolveKey(null!, ref emptyPath, true); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(512)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1234, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1234, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode1)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode2)); } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1235, trieNode2)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1235, trieNode2)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode3)); committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode4)); @@ -350,13 +344,12 @@ public void Dispatcher_will_try_to_clear_memory_the_soonest_possible() public void Dispatcher_will_always_try_to_clear_memory() { TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(512)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; for (int i = 0; i < 1024; i++) { TrieNode fakeRoot = new(NodeType.Leaf, new byte[0]); // 192B fakeRoot.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, i, fakeRoot)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(i, fakeRoot)) { for (int j = 0; j < 1 + i % 3; j++) { @@ -384,16 +377,15 @@ public void Dispatcher_will_save_to_db_everything_from_snapshot_blocks() pruningStrategy: new MemoryLimit(16.MB()), kvStore: memDb, persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 0, a)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(0, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(1, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } storage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -410,15 +402,14 @@ public void Stays_in_memory_until_persisted() NodeStorage storage = new NodeStorage(memDb); using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new MemoryLimit(16.MB())); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 0, a)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(0, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(1, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } // <- do not persist in this test storage.Get(null, TreePath.Empty, a.Keccak).Should().BeNull(); @@ -451,20 +442,19 @@ public void Will_get_persisted_on_snapshot_if_referenced() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4) ); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, a)) { } storage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -488,23 +478,21 @@ public void Will_not_get_dropped_on_snapshot_if_unreferenced_in_later_blocks() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, a)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 7, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, a)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(7, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, a)) { } nodeStorage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -527,23 +515,21 @@ public void Will_get_dropped_on_snapshot_if_it_was_a_transient_node() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 3, a)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(3, a)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); // <- new root } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, b)) { } // should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, b)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, b)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, b)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, b)) { } // should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, b)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, b)) { } memDb[a.Keccak!.Bytes].Should().BeNull(); fullTrieStore.IsNodeCached(null, TreePath.Empty, a.Keccak).Should().BeTrue(); @@ -625,25 +611,28 @@ public void Will_store_storage_on_snapshot() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); + using (fullTrieStore.BeginStateBlockCommit(0, null)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } - using (ICommitter committer = fullTrieStore.GetTrieStore(TestItem.KeccakA).BeginCommit(TrieType.Storage, 1, storage1)) - { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); - } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, a)) + using (fullTrieStore.BeginBlockCommit(1)) { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + using (ICommitter committer = fullTrieStore.GetTrieStore(TestItem.KeccakA).BeginCommit(storage1)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); + } + + using (ICommitter committer = fullTrieStore.GetTrieStore(null).BeginCommit(a)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + } } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, a)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, a)) { } asStorage.Get(null, TreePath.Empty, a.Keccak).Should().NotBeNull(); asStorage.Get(TestItem.KeccakA, TreePath.Empty, storage1.Keccak).Should().NotBeNull(); @@ -676,27 +665,31 @@ public void Will_drop_transient_storage() IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } - using (ICommitter committer = trieStore.BeginCommit(TrieType.Storage, 1, storage1)) + using (fullTrieStore.BeginBlockCommit(1)) { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); - } + using (ICommitter committer = fullTrieStore.GetTrieStore(Hash256.Zero).BeginCommit(storage1)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); + } + + using (ICommitter _ = trieStore.BeginCommit(a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 1, a)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, a)) { } + } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 2, b)) + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, a)) { } + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(2, b)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); // <- new root } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, b)) { } // Should be 'a' to test properly - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, b)) { } // Should be 'a' to test properly + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, b)) { } // Should be 'a' to test properly memDb[a.Keccak!.Bytes].Should().BeNull(); memDb[storage1.Keccak!.Bytes].Should().BeNull(); @@ -744,34 +737,35 @@ public void Will_combine_same_storage() pruningStrategy: new MemoryLimit(16.MB()), persistenceStrategy: new ConstantInterval(4)); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 0, null)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(0, null)) { } - using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage1Nib))).BeginCommit(TrieType.Storage, 1, storage1)) + using (fullTrieStore.BeginBlockCommit(1)) { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); - } + using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage1Nib))).BeginCommit(storage1)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage1)); + } - using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage2Nib))).BeginCommit(TrieType.Storage, 1, storage2)) - { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage2)); - } + using (ICommitter committer = fullTrieStore.GetTrieStore(new Hash256(Nibbles.ToBytes(storage2Nib))).BeginCommit(storage2)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(storage2)); + } - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, 1, branch)) - { - committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); - committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); - committer.CommitNode(ref emptyPath, new NodeCommitInfo(branch)); + using (ICommitter committer = fullTrieStore.GetTrieStore(null).BeginCommit(branch)) + { + committer.CommitNode(ref emptyPath, new NodeCommitInfo(a)); + committer.CommitNode(ref emptyPath, new NodeCommitInfo(b)); + committer.CommitNode(ref emptyPath, new NodeCommitInfo(branch)); + } } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 2, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 3, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 4, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 5, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 6, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 7, branch)) { } - using (ICommitter _ = trieStore.BeginCommit(TrieType.State, 8, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(2, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(3, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(4, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(5, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(6, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(7, branch)) { } + using (ICommitter _ = fullTrieStore.BeginStateBlockCommit(8, branch)) { } storage.Get(null, TreePath.FromNibble(new byte[] { 0 }), a.Keccak).Should().NotBeNull(); storage.Get(new Hash256(Nibbles.ToBytes(storage1Nib)), TreePath.Empty, storage1.Keccak).Should().NotBeNull(); @@ -800,7 +794,7 @@ public async Task Read_only_trie_store_is_allowing_many_thread_to_work_with_the_ IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; trieNode.ResolveKey(trieStore, ref emptyPath, false); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 0, trieNode)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(0, trieNode)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(trieNode)); } @@ -857,7 +851,7 @@ public void ReadOnly_store_returns_copies(bool pruning) using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(pruning)); IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 0, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(0, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -891,13 +885,13 @@ public void After_commit_should_have_has_root() Account account = new(1); StateTree stateTree = new(trieStore, LimboLogs.Instance); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); trieStore.HasRoot(stateTree.RootHash).Should().BeTrue(); stateTree.Get(TestItem.AddressA); account = account.WithChangedBalance(2); stateTree.Set(TestItem.AddressA, account); - stateTree.Commit(0); + stateTree.Commit(); trieStore.HasRoot(stateTree.RootHash).Should().BeTrue(); } @@ -911,13 +905,12 @@ public async Task Will_RemovePastKeys_OnSnapshot() pruningStrategy: new TestPruningStrategy(true, true, 2, 100000), persistenceStrategy: No.Persistence); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TreePath emptyPath = TreePath.Empty; for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i], new byte[2]); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -955,7 +948,7 @@ public async Task Will_Trigger_ReorgBoundaryEvent_On_Prune() for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i], new byte[2]); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -991,7 +984,7 @@ public async Task Will_Not_RemovePastKeys_OnSnapshot_DuringFullPruning() for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i], new byte[2]); - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } @@ -1021,7 +1014,7 @@ public async Task Will_NotRemove_ReCommittedNode() for (int i = 0; i < 64; i++) { TrieNode node = new(NodeType.Leaf, TestItem.Keccaks[i % 4], new byte[2]); - using (ICommitter committer = trieStore.BeginCommit(TrieType.State, i, node)) + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(i, node)) { committer.CommitNode(ref emptyPath, new NodeCommitInfo(node)); } diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs index 001be11d7d7..a2acfd0be71 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs @@ -925,7 +925,8 @@ void CheckChildren() [Test] public void Rlp_is_cloned_when_cloning() { - IScopedTrieStore trieStore = new TrieStore(new MemDb(), NullLogManager.Instance).GetTrieStore(null); + ITrieStore fullTrieStore = new TrieStore(new MemDb(), NullLogManager.Instance); + IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); TrieNode leaf1 = new(NodeType.Leaf); leaf1.Key = Bytes.FromHexString("abc"); @@ -942,10 +943,13 @@ public void Rlp_is_cloned_when_cloning() TreePath path = TreePath.Empty; - using (ICommitter? committer = trieStore.BeginCommit(TrieType.State, 0, leaf2)) + using (IBlockCommitter _ = fullTrieStore.BeginBlockCommit(0)) { - committer.CommitNode(ref path, new NodeCommitInfo(leaf1)); - committer.CommitNode(ref path, new NodeCommitInfo(leaf2)); + using (ICommitter? committer = trieStore.BeginCommit(leaf2)) + { + committer.CommitNode(ref path, new NodeCommitInfo(leaf1)); + committer.CommitNode(ref path, new NodeCommitInfo(leaf2)); + } } TrieNode trieNode = new(NodeType.Branch); diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs index 0d9e05c1e1f..6f61ae6b0c1 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs @@ -61,7 +61,7 @@ public void Single_leaf() using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager); PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(1); @@ -75,7 +75,7 @@ public void Single_leaf_update_same_block() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyA, _longLeaf2); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(1); @@ -92,9 +92,9 @@ public void Single_leaf_update_next_blocks() using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager); PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf2); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -113,7 +113,7 @@ public void Single_leaf_delete_same_block() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(0); @@ -129,9 +129,9 @@ public void Single_leaf_delete_next_block() using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager); PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -147,24 +147,24 @@ public void Single_leaf_and_keep_for_multiple_dispatches_then_delete() MemDb memDb = new(); using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), new ConstantInterval(4), LimboLogs.Instance); PatriciaTree patriciaTree = new(trieStore, _logManager); - patriciaTree.Commit(0); - patriciaTree.Commit(1); - patriciaTree.Commit(2); + trieStore.CommitPatriciaTrie(0, patriciaTree); + trieStore.CommitPatriciaTrie(1, patriciaTree); + trieStore.CommitPatriciaTrie(2, patriciaTree); patriciaTree.Set(_keyA, _longLeaf1); - patriciaTree.Commit(3); - patriciaTree.Commit(4); + trieStore.CommitPatriciaTrie(3, patriciaTree); + trieStore.CommitPatriciaTrie(4, patriciaTree); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(5); + trieStore.CommitPatriciaTrie(5, patriciaTree); patriciaTree.Set(_keyB, _longLeaf2); - patriciaTree.Commit(6); - patriciaTree.Commit(7); - patriciaTree.Commit(8); - patriciaTree.Commit(9); - patriciaTree.Commit(10); - patriciaTree.Commit(11); + trieStore.CommitPatriciaTrie(6, patriciaTree); + trieStore.CommitPatriciaTrie(7, patriciaTree); + trieStore.CommitPatriciaTrie(8, patriciaTree); + trieStore.CommitPatriciaTrie(9, patriciaTree); + trieStore.CommitPatriciaTrie(10, patriciaTree); + trieStore.CommitPatriciaTrie(11, patriciaTree); patriciaTree.Set(_keyB, Array.Empty()); - patriciaTree.Commit(12); - patriciaTree.Commit(13); + trieStore.CommitPatriciaTrie(12, patriciaTree); + trieStore.CommitPatriciaTrie(13, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -184,7 +184,7 @@ public void Branch_with_branch_and_leaf() patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.Set(_keyC, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(6); @@ -218,7 +218,7 @@ public void GetBranchNodesWithPartialPath() patriciaTree.Set(_keysA, _longLeaf1); patriciaTree.Set(_keysB, _longLeaf1); patriciaTree.Set(_keysC, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -252,11 +252,11 @@ public void Branch_with_branch_and_leaf_then_deleted() patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.Set(_keyC, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, Array.Empty()); patriciaTree.Set(_keyB, Array.Empty()); patriciaTree.Set(_keyC, Array.Empty()); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); // leaf (root) @@ -280,7 +280,8 @@ public void Test_add_many(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); + patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -312,7 +313,7 @@ public void Test_try_delete_and_read_missing_nodes(int i) patriciaTree.Set(key.Bytes, Array.Empty()); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -353,7 +354,7 @@ public void Test_update_many(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -378,7 +379,7 @@ public void Test_update_many_next_block(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); for (int j = 0; j < i; j++) { @@ -388,7 +389,7 @@ public void Test_update_many_next_block(int i) _logger.Trace($"Setting {key.Bytes.ToHexString()} = {value.ToHexString()}"); } - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -423,7 +424,7 @@ public void Test_add_and_delete_many_same_block(int i) patriciaTree.Set(key.Bytes, Array.Empty()); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -447,7 +448,7 @@ public void Test_add_and_delete_many_next_block(int i) patriciaTree.Set(key.Bytes, value); } - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); for (int j = 0; j < i; j++) { @@ -455,7 +456,7 @@ public void Test_add_and_delete_many_next_block(int i) patriciaTree.Set(key.Bytes, Array.Empty()); } - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); patriciaTree.UpdateRootHash(); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -493,7 +494,7 @@ public void Two_branches_exactly_same_leaf() patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.Set(_keyC, _longLeaf1); patriciaTree.Set(_keyD, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(8); @@ -519,7 +520,7 @@ public void Two_branches_exactly_same_leaf_then_one_removed() patriciaTree.Set(_keyC, _longLeaf1); patriciaTree.Set(_keyD, _longLeaf1); patriciaTree.Set(_keyA, Array.Empty()); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); // leaf (root) memDb.Keys.Should().HaveCount(6); @@ -545,7 +546,7 @@ public void Extension_with_branch_with_two_different_children() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf2); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); memDb.Keys.Should().HaveCount(4); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); checkTree.Get(_keyA).ToArray().Should().BeEquivalentTo(_longLeaf1); @@ -560,7 +561,7 @@ public void Extension_with_branch_with_two_same_children() PatriciaTree patriciaTree = new(trieStore, _logManager); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); memDb.Keys.Should().HaveCount(4); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); checkTree.Get(_keyA).ToArray().Should().BeEquivalentTo(_longLeaf1); @@ -576,11 +577,11 @@ public void When_branch_with_two_different_children_change_one_and_change_back_n patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf2); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf3); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); // extension // branch @@ -597,11 +598,11 @@ public void When_branch_with_two_same_children_change_one_and_change_back_next_b patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.Set(_keyB, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf3); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); memDb.Keys.Should().HaveCount(4); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -631,7 +632,7 @@ L L - - - - - - - - - - - - - - */ patriciaTree.Set(key2, _longLeaf1); patriciaTree.Set(key3, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); memDb.Keys.Should().HaveCount(7); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -679,10 +680,10 @@ L L - - - - - - - - - - - - - - */ patriciaTree.Set(key2, _longLeaf1); patriciaTree.Set(key3, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(key3, Array.Empty()); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); memDb.Keys.Should().HaveCount(8); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -702,11 +703,11 @@ public void When_two_branches_with_two_same_children_change_one_and_change_back_ patriciaTree.Set(_keyC, _longLeaf1); patriciaTree.Set(_keyD, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(0); + trieStore.CommitPatriciaTrie(0, patriciaTree); patriciaTree.Set(_keyA, _longLeaf3); patriciaTree.Set(_keyA, _longLeaf1); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(1); + trieStore.CommitPatriciaTrie(1, patriciaTree); memDb.Keys.Should().HaveCount(8); PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree); @@ -788,7 +789,7 @@ public void Fuzz_accounts( streamWriter.WriteLine( $"Commit block {blockNumber} | empty: {isEmptyBlock}"); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(blockNumber); + trieStore.CommitPatriciaTrie(blockNumber, patriciaTree); rootQueue.Enqueue(patriciaTree.RootHash); } @@ -930,7 +931,7 @@ public void Fuzz_accounts_with_reorganizations( streamWriter.WriteLine( $"Commit block {blockCount} | empty: {isEmptyBlock}"); patriciaTree.UpdateRootHash(); - patriciaTree.Commit(blockCount); + trieStore.CommitPatriciaTrie(blockNumber, patriciaTree); rootQueue.Enqueue(patriciaTree.RootHash); rootStack.Push(patriciaTree.RootHash); blockCount++; diff --git a/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs b/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs index 28a7d2bb514..1f773711e92 100644 --- a/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/VisitingTests.cs @@ -39,7 +39,7 @@ public void Visitors_state(VisitingOptions options) patriciaTree.Set(raw, Rlp.Encode(new Account(10, (UInt256)(10_000_000 + i)))); } - patriciaTree.Commit(0); + using (trieStore.BeginBlockCommit(0)) { patriciaTree.Commit(); } var visitor = new AppendingVisitor(); @@ -70,6 +70,8 @@ public void Visitors_storage(VisitingOptions options) byte[] value = Enumerable.Range(1, 32).Select(i => (byte)i).ToArray(); Hash256 stateRootHash = Keccak.Zero; + var blockCommit = trieStore.BeginBlockCommit(0); + for (int outi = 0; outi < 64; outi++) { ValueHash256 stateKey = default; @@ -82,7 +84,7 @@ public void Visitors_storage(VisitingOptions options) storageKey.BytesAsSpan[i / 2] = (byte)(1 << (4 * (1 - i % 2))); storage.Set(storageKey, value); } - storage.Commit(0); + storage.Commit(); stateRootHash = storage.RootHash; } @@ -99,7 +101,8 @@ public void Visitors_storage(VisitingOptions options) new Account(10, (UInt256)(10_000_000 + i), stateRootHash, Keccak.OfAnEmptySequenceRlp)); } - stateTree.Commit(0); + stateTree.Commit(); + blockCommit.Dispose(); var visitor = new AppendingVisitor(); diff --git a/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs index 5b23ba6ad92..984ab33ec07 100644 --- a/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs @@ -34,8 +34,8 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) => public INodeStorage.KeyScheme Scheme => @base.Scheme; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - @base.BeginCommit(trieType, blockNumber, root, writeFlags); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => + @base.BeginCommit(root, writeFlags); public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => @base.IsPersisted(in path, in keccak); diff --git a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs index 88b443a3983..2ff2a442f5b 100644 --- a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs +++ b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs @@ -131,7 +131,7 @@ public PatriciaTree( _bufferPool = bufferPool; } - public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlags = WriteFlags.None) + public void Commit(bool skipRoot = false, WriteFlags writeFlags = WriteFlags.None) { if (!_allowCommits) { @@ -140,14 +140,15 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag int maxLevelForConcurrentCommit = _writeBeforeCommit switch { - > 64 * 16 => 1, // we separate at two top levels - > 64 => 0, // we separate at top level + > 4 * 16 * 16 => 2, // we separate at three top levels + > 4 * 16 => 1, // we separate at two top levels + > 4 => 0, // we separate at top level _ => -1 }; _writeBeforeCommit = 0; - using (ICommitter committer = TrieStore.BeginCommit(TrieType, blockNumber, RootRef, writeFlags)) + using (ICommitter committer = TrieStore.BeginCommit(RootRef, writeFlags)) { if (RootRef is not null && RootRef.IsDirty) { @@ -159,14 +160,6 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag SetRootHash(RootRef.Keccak!, true); } } - - if (_logger.IsDebug) Debug(blockNumber); - - [MethodImpl(MethodImplOptions.NoInlining)] - void Debug(long blockNumber) - { - _logger.Debug($"Finished committing {RootRef?.Keccak} in block {blockNumber}"); - } } private void Commit(ICommitter committer, ref TreePath path, NodeCommitInfo nodeCommitInfo, int maxLevelForConcurrentCommit, bool skipSelf = false) @@ -214,7 +207,7 @@ Task CreateTaskForPath(TreePath childPath, TrieNode childNode, int idx) => Task. { if (node.IsChildDirty(i)) { - if (i < 15 && committer.CanSpawnTask()) + if (i < 15 && committer.TryRequestConcurrentQuota()) { childTasks ??= new ArrayPoolList(15); TreePath childPath = path.Append(i); diff --git a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs index 82d83a3e8f3..ece7d00245b 100644 --- a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs @@ -5,7 +5,6 @@ using System.Collections.Concurrent; using System.Numerics; using Nethermind.Core; -using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Trie.Pruning; @@ -34,9 +33,14 @@ public void Dispose() _inner.Dispose(); } - public ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags) + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) { - return _inner.BeginCommit(trieType, blockNumber, address, root, writeFlags); + return _inner.BeginCommit(address, root, writeFlags); + } + + public IBlockCommitter BeginBlockCommit(long blockNumber) + { + return _inner.BeginBlockCommit(blockNumber); } public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) diff --git a/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs b/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs index 3ea7daf397a..c5993090cac 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Diagnostics; + namespace Nethermind.Trie.Pruning { internal class BlockCommitSet @@ -25,5 +27,23 @@ public override string ToString() { return $"{BlockNumber}({Root})"; } + + /// + /// Prunes persisted branches of the current commit set root. + /// + public void Prune() + { + long start = Stopwatch.GetTimestamp(); + + // We assume that the most recent package very likely resolved many persisted nodes and only replaced + // some top level branches. Any of these persisted nodes are held in cache now so we just prune them here + // to avoid the references still being held after we prune the cache. + // We prune them here but just up to two levels deep which makes it a very lightweight operation. + // Note that currently the TrieNode ResolveChild un-resolves any persisted child immediately which + // may make this call unnecessary. + Root?.PrunePersistedRecursively(2); + Metrics.DeepPruningTime = (long)Stopwatch.GetElapsedTime(start).TotalMilliseconds; + } + } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs index 3dec5d49e25..6c4a445d86b 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/IScopedTrieStore.cs @@ -13,7 +13,8 @@ namespace Nethermind.Trie.Pruning; /// public interface IScopedTrieStore : ITrieNodeResolver { - ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None); + // Begins a commit to update the trie store. The `ICommitter` provide `CommitNode` to add node into. + ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None); // Only used by snap provider, so ValueHash instead of Hash bool IsPersisted(in TreePath path, in ValueHash256 keccak); @@ -26,6 +27,6 @@ public interface ICommitter : IDisposable { void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo); - bool CanSpawnTask() => false; + bool TryRequestConcurrentQuota() => false; void ReturnConcurrencyQuota() { } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs index f9f2f59d55e..3178da3fd6d 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ITrieStore.cs @@ -11,10 +11,8 @@ namespace Nethermind.Trie.Pruning /// /// Full traditional trie store. /// - public interface ITrieStore : IDisposable + public interface ITrieStore : IDisposable, ITrieStoreInternal { - bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak); - IReadOnlyTrieStore AsReadOnly(INodeStorage? keyValueStore = null); event EventHandler? ReorgBoundaryReached; @@ -22,22 +20,49 @@ public interface ITrieStore : IDisposable // Used for serving via hash IReadOnlyKeyValueStore TrieNodeRlpStore { get; } - // Used by healing - void Set(Hash256? address, in TreePath path, in ValueHash256 keccak, byte[] rlp); - bool HasRoot(Hash256 stateRoot); IScopedTrieStore GetTrieStore(Hash256? address); + INodeStorage.KeyScheme Scheme { get; } + + /// + /// Begin a block commit for this block number. This call may be blocked if a memory pruning is currently happening. + /// This call is required during block processing for memory pruning and reorg boundary to function. + /// + /// + /// + IBlockCommitter BeginBlockCommit(long blockNumber); + } + /// + /// These methods are to be used by ScopedTrieStore. + /// It should be considered internal to TrieStore. + /// It should not be used directly, nor intercepted. + /// + public interface ITrieStoreInternal + { + // Used by healing + void Set(Hash256? address, in TreePath path, in ValueHash256 keccak, byte[] rlp); + ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags); TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash); byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None); byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None); - INodeStorage.KeyScheme Scheme { get; } - ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags); + bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak); } public interface IPruningTrieStore { public void PersistCache(CancellationToken cancellationToken); } + + /// + /// A block committer identifies the scope at which a commit for a block should happen. + /// The commit started via which is called by + /// Depending on , multiple patricia trie commit may run at the same time. + /// + public interface IBlockCommitter : IDisposable + { + bool TryRequestConcurrencyQuota() => false; + void ReturnConcurrencyQuota() { } + } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs b/src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs new file mode 100644 index 00000000000..c39ddc374bf --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/Pruning/NullCommitter.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Trie.Pruning; + +internal class NullCommitter : ICommitter, IBlockCommitter +{ + public static NullCommitter Instance = new NullCommitter(); + + private NullCommitter() + { + } + + public void Dispose() { } + + public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) { } + public ICommitter GetTrieCommitter(Hash256? address, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => this; +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs index 585050f7150..5e96cb610ea 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Nethermind.Core; @@ -21,7 +22,7 @@ private NullTrieStore() { } public byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => []; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => new NullCommitter(); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => NullCommitter.Instance; public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => true; @@ -30,12 +31,5 @@ public void Set(in TreePath path, in ValueHash256 keccak, byte[] rlp) { } public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256 storageRoot) => this; public INodeStorage.KeyScheme Scheme => INodeStorage.KeyScheme.HalfPath; - - internal class NullCommitter : ICommitter - { - public void Dispose() { } - - public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) { } - } } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs index 4bb2c8d537f..4f66fb73e70 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs @@ -29,8 +29,12 @@ public byte[] LoadRlp(Hash256? address, in TreePath treePath, Hash256 hash, Read public IReadOnlyTrieStore AsReadOnly(INodeStorage nodeStore) => new ReadOnlyTrieStore(_trieStore, nodeStore); - public ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags) => - new NullTrieStore.NullCommitter(); + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => NullCommitter.Instance; + + public IBlockCommitter BeginBlockCommit(long blockNumber) + { + return NullCommitter.Instance; + } public event EventHandler ReorgBoundaryReached { @@ -63,8 +67,7 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address1) => public INodeStorage.KeyScheme Scheme => fullTrieStore.Scheme; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - new NullTrieStore.NullCommitter(); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => NullCommitter.Instance; public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => fullTrieStore.IsPersisted(address, path, in keccak); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs index 650ee399086..66a61af6f74 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs @@ -22,8 +22,8 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address1) => public INodeStorage.KeyScheme Scheme => fullTrieStore.Scheme; - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - fullTrieStore.BeginCommit(trieType, blockNumber, address, root, writeFlags); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => + fullTrieStore.BeginCommit(address, root, writeFlags); public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => fullTrieStore.IsPersisted(address, path, in keccak); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index ac44f42a3e0..b0cacf09bf0 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -27,8 +27,6 @@ public class TrieStore : ITrieStore, IPruningTrieStore private int _isFirst; - private INodeStorage.WriteBatch? _currentBatch = null; - private readonly TrieStoreDirtyNodesCache[] _dirtyNodes = []; private readonly Task[] _dirtyNodesTasks = []; private readonly ConcurrentDictionary[] _persistedHashes = []; @@ -164,8 +162,10 @@ public int CachedNodesCount } } - private void CommitNode(long blockNumber, Hash256? address, ref TreePath path, in NodeCommitInfo nodeCommitInfo, WriteFlags writeFlags = WriteFlags.None) + private void CommitAndInsertToDirtyNodes(long blockNumber, Hash256? address, ref TreePath path, in NodeCommitInfo nodeCommitInfo) { + Debug.Assert(_pruningStrategy.PruningEnabled); + if (_logger.IsTrace) Trace(blockNumber, in nodeCommitInfo); if (!nodeCommitInfo.IsEmptyBlockMarker && !nodeCommitInfo.Node.IsBoundaryProofNode) { @@ -181,18 +181,9 @@ private void CommitNode(long blockNumber, Hash256? address, ref TreePath path, i ThrowNodeHasBeenSeen(blockNumber, node); } - if (_pruningStrategy.PruningEnabled) - { - node = SaveOrReplaceInDirtyNodesCache(address, ref path, nodeCommitInfo, node); - } - + node = SaveOrReplaceInDirtyNodesCache(address, ref path, nodeCommitInfo, node); node.LastSeen = Math.Max(blockNumber, node.LastSeen); - if (!_pruningStrategy.PruningEnabled) - { - PersistNode(address, path, node, blockNumber, writeFlags); - } - IncrementCommittedNodesCount(); } @@ -211,6 +202,29 @@ void Trace(long blockNumber, in NodeCommitInfo nodeCommitInfo) static void ThrowNodeHasBeenSeen(long blockNumber, TrieNode node) => throw new TrieStoreException($"{nameof(TrieNode.LastSeen)} set on {node} committed at {blockNumber}."); } + private void CommitAndPersistNode(Hash256? address, ref TreePath path, in NodeCommitInfo nodeCommitInfo, WriteFlags writeFlags, INodeStorage.WriteBatch? writeBatch) + { + Debug.Assert(!_pruningStrategy.PruningEnabled); + + if (!nodeCommitInfo.IsEmptyBlockMarker && !nodeCommitInfo.Node.IsBoundaryProofNode) + { + TrieNode node = nodeCommitInfo.Node; + + if (node.Keccak is null) + { + ThrowUnknownHash(node); + } + + PersistNode(address, path, node, writeBatch!, writeFlags); + + IncrementCommittedNodesCount(); + } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowUnknownHash(TrieNode node) => throw new TrieStoreException($"The hash of {node} should be known at the time of committing."); + } + private int GetNodeShardIdx(in TreePath path, Hash256 hash) => // When enabled, the shard have dictionaries for tracking past path hash also. // So the same path need to be in the same shard for the remove logic to work. @@ -289,70 +303,61 @@ static void ThrowNodeIsNotSame(TrieNode node, TrieNode cachedNodeCopy) => throw new InvalidOperationException($"The hash of replacement node {cachedNodeCopy} is not the same as the original {node}."); } - public ICommitter BeginCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags) + public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) { - ArgumentOutOfRangeException.ThrowIfNegative(blockNumber); - EnsureCommitSetExistsForBlock(blockNumber); - - int concurrency = _pruningStrategy.PruningEnabled - ? Environment.ProcessorCount - : 0; // The write batch when pruning is not enabled is not concurrent safe + if (_pruningStrategy.PruningEnabled) + { + if (_currentBlockCommitter is null) throw new InvalidOperationException($"With pruning triestore, {nameof(BeginBlockCommit)} must be called."); + } - return new TrieStoreCommitter(this, trieType, blockNumber, address, root, writeFlags, concurrency); + return _currentBlockCommitter is not null + // Note, archive node still use this path. This is because it needs the block commit set to handle reorg announcement. + ? _currentBlockCommitter.GetTrieCommitter(address, root, writeFlags) + // This happens when there are no block involved, such as during snap sync or just calculating patricia root. + : new NonPruningTrieStoreCommitter(this, address, _nodeStorage.StartWriteBatch(), writeFlags); } - private void FinishBlockCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) + public IBlockCommitter BeginBlockCommit(long blockNumber) { - try + if (_pruningStrategy.PruningEnabled) { - if (trieType == TrieType.State) // storage tries happen before state commits + if (_currentBlockCommitter is not null) { - if (_logger.IsTrace) _logger.Trace($"Enqueued blocks {_commitSetQueue?.Count ?? 0}"); - BlockCommitSet set = CurrentPackage; - if (set is not null) - { - if (_logger.IsTrace) _logger.Trace($"Current root (block {blockNumber}): {root}, block {set.BlockNumber}"); - set.Seal(root); - } - - bool shouldPersistSnapshot = _persistenceStrategy.ShouldPersist(set.BlockNumber); - if (shouldPersistSnapshot) - { - _currentBatch ??= _nodeStorage.StartWriteBatch(); - try - { - PersistBlockCommitSet(address, set, _currentBatch, writeFlags: writeFlags); - PruneCurrentSet(); - } - finally - { - // For safety we prefer to commit half of the batch rather than not commit at all. - // Generally hanging nodes are not a problem in the DB but anything missing from the DB is. - _currentBatch?.Dispose(); - _currentBatch = null; - } - } - else - { - PruneCurrentSet(); - } - - CurrentPackage = null; - if (_pruningStrategy.PruningEnabled && Monitor.IsEntered(_dirtyNodesLock)) - { - Monitor.Exit(_dirtyNodesLock); - } + throw new InvalidOperationException("Cannot start a new block commit when an existing one is still not closed"); } + + Monitor.Enter(_dirtyNodesLock); } - finally + + _currentBlockCommitter = new BlockCommitter(this, CreateCommitSet(blockNumber)); + return _currentBlockCommitter; + } + + private void FinishBlockCommit(BlockCommitSet set, TrieNode? root) + { + if (_logger.IsTrace) _logger.Trace($"Enqueued blocks {_commitSetQueue?.Count ?? 0}"); + set.Seal(root); + + bool shouldPersistSnapshot = _persistenceStrategy.ShouldPersist(set.BlockNumber); + if (shouldPersistSnapshot) { - _currentBatch?.Dispose(); - _currentBatch = null; + // For safety we prefer to commit half of the batch rather than not commit at all. + // Generally hanging nodes are not a problem in the DB but anything missing from the DB is. + using INodeStorage.WriteBatch currentBatch = _nodeStorage.StartWriteBatch(); + ParallelPersistBlockCommitSet(set); } + set.Prune(); + + _currentBlockCommitter = null; + + if (_pruningStrategy.PruningEnabled) + Monitor.Exit(_dirtyNodesLock); + Prune(); } + public event EventHandler? ReorgBoundaryReached; public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 keccak, INodeStorage? nodeStorage, ReadFlags readFlags = ReadFlags.None) @@ -544,7 +549,7 @@ private bool SaveSnapshot() { BlockCommitSet blockCommitSet = candidateSets[index]; if (_logger.IsDebug) _logger.Debug($"Elevated pruning for candidate {blockCommitSet.BlockNumber}"); - ParallelPersistBlockCommitSet(null, blockCommitSet, persistedNodeRecorder); + ParallelPersistBlockCommitSet(blockCommitSet, persistedNodeRecorder); } Task deleteTask = shouldDeletePersistedNode ? RemovePastKeys() : Task.CompletedTask; @@ -599,23 +604,6 @@ private void PersistedNodeRecorder(TreePath treePath, Hash256 address, TrieNode } } - /// - /// Prunes persisted branches of the current commit set root. - /// - private void PruneCurrentSet() - { - long start = Stopwatch.GetTimestamp(); - - // We assume that the most recent package very likely resolved many persisted nodes and only replaced - // some top level branches. Any of these persisted nodes are held in cache now so we just prune them here - // to avoid the references still being held after we prune the cache. - // We prune them here but just up to two levels deep which makes it a very lightweight operation. - // Note that currently the TrieNode ResolveChild un-resolves any persisted child immediately which - // may make this call unnecessary. - CurrentPackage?.Root?.PrunePersistedRecursively(2); - Metrics.DeepPruningTime = (long)Stopwatch.GetElapsedTime(start).TotalMilliseconds; - } - /// /// This method is responsible for reviewing the nodes that are directly in the cache and /// removing ones that are either no longer referenced or already persisted. @@ -682,6 +670,9 @@ public void WaitForPruning() private ConcurrentQueue _commitSetQueue; + private ConcurrentQueue CommitSetQueue => + (_commitSetQueue ?? CreateQueueAtomic(ref _commitSetQueue)); + private long _memoryUsedByDirtyCache; private int _committedNodesCount; @@ -690,9 +681,8 @@ public void WaitForPruning() private long _latestPersistedBlockNumber; - private BlockCommitSet? CurrentPackage { get; set; } + private BlockCommitter? _currentBlockCommitter = null; - private bool IsCurrentListSealed => CurrentPackage is null || CurrentPackage.IsSealed; private long LatestCommittedBlockNumber { get; set; } public INodeStorage.KeyScheme Scheme => _nodeStorage.Scheme; @@ -705,22 +695,23 @@ private static ConcurrentQueue CreateQueueAtomic(ref ConcurrentQ return prior ?? instance; } - private void CreateCommitSet(long blockNumber) + private BlockCommitSet CreateCommitSet(long blockNumber) { if (_logger.IsDebug) _logger.Debug($"Beginning new {nameof(BlockCommitSet)} - {blockNumber}"); // TODO: this throws on reorgs, does it not? let us recreate it in test - Debug.Assert(CurrentPackage is null || blockNumber == CurrentPackage.BlockNumber + 1, "Newly begun block is not a successor of the last one"); - Debug.Assert(IsCurrentListSealed, "Not sealed when beginning new block"); + Debug.Assert(!CommitSetQueue.TryPeek(out BlockCommitSet lastSet) || blockNumber == lastSet.BlockNumber + 1, $"Newly begun block is not a successor of the last one."); + Debug.Assert(!CommitSetQueue.TryPeek(out lastSet) || lastSet.IsSealed, "Not sealed when beginning new block"); BlockCommitSet commitSet = new(blockNumber); - (_commitSetQueue ?? CreateQueueAtomic(ref _commitSetQueue)).Enqueue(commitSet); + CommitSetQueue.Enqueue(commitSet); LatestCommittedBlockNumber = Math.Max(blockNumber, LatestCommittedBlockNumber); + // Why are we announcing **before** committing next block?? + // Should it be after commit? AnnounceReorgBoundaries(); DequeueOldCommitSets(); - CurrentPackage = commitSet; - Debug.Assert(ReferenceEquals(CurrentPackage, commitSet), $"Current {nameof(BlockCommitSet)} is not same as the new package just after adding"); + return commitSet; } /// @@ -728,40 +719,10 @@ private void CreateCommitSet(long blockNumber) /// Already persisted nodes are skipped. After this action we are sure that the full state is available /// for the block represented by this commit set. /// - /// /// A commit set of a block which root is to be persisted. - /// The write batch to write to - /// Track persisted hashes in this dictionary if not null + /// Special action to be called on each persist. Used to track which node to remove. /// - private void PersistBlockCommitSet( - Hash256? address, - BlockCommitSet commitSet, - INodeStorage.WriteBatch writeBatch, - Action? persistedNodeRecorder = null, - WriteFlags writeFlags = WriteFlags.None - ) - { - void PersistNode(TrieNode tn, Hash256? address2, TreePath path) - { - persistedNodeRecorder?.Invoke(path, address2, tn); - this.PersistNode(address2, path, tn, commitSet.BlockNumber, writeFlags, writeBatch); - } - - if (_logger.IsDebug) _logger.Debug($"Persisting from root {commitSet.Root} in {commitSet.BlockNumber}"); - - long start = Stopwatch.GetTimestamp(); - TreePath path = TreePath.Empty; - commitSet.Root?.CallRecursively(PersistNode, address, ref path, GetTrieStore(null), true, _logger); - long elapsedMilliseconds = (long)Stopwatch.GetElapsedTime(start).TotalMilliseconds; - Metrics.SnapshotPersistenceTime = elapsedMilliseconds; - - if (_logger.IsDebug) _logger.Debug($"Persisted trie from {commitSet.Root} at {commitSet.BlockNumber} in {elapsedMilliseconds}ms (cache memory {MemoryUsedByDirtyCache})"); - - LastPersistedBlockNumber = commitSet.BlockNumber; - } - private void ParallelPersistBlockCommitSet( - Hash256? address, BlockCommitSet commitSet, Action? persistedNodeRecorder = null, WriteFlags writeFlags = WriteFlags.None @@ -777,7 +738,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) if (path.Length < parallelBoundaryPathLength) { persistedNodeRecorder?.Invoke(path, address2, tn); - PersistNode(address2, path, tn, commitSet.BlockNumber, writeFlags, topLevelWriteBatch); + PersistNode(address2, path, tn, topLevelWriteBatch, writeFlags); } else { @@ -791,7 +752,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) // The first CallRecursive stop at two level, yielding 256 node in parallelStartNodes, which is run concurrently TreePath path = TreePath.Empty; - commitSet.Root?.CallRecursively(TopLevelPersist, address, ref path, GetTrieStore(null), true, _logger, maxPathLength: parallelBoundaryPathLength); + commitSet.Root?.CallRecursively(TopLevelPersist, null, ref path, GetTrieStore(null), true, _logger, maxPathLength: parallelBoundaryPathLength); // The amount of change in the subtrees are not balanced at all. So their writes ares buffered here // which get disposed in parallel instead of being disposed in `PersistNodeStartingFrom`. @@ -815,7 +776,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) Task.WaitAll(parallelStartNodes.Select(entry => Task.Run(() => { (TrieNode trieNode, Hash256? address2, TreePath path2) = entry; - PersistNodeStartingFrom(trieNode, address2, path2, commitSet, persistedNodeRecorder, writeFlags, disposeQueue); + PersistNodeStartingFrom(trieNode, address2, path2, persistedNodeRecorder, writeFlags, disposeQueue); })).ToArray()); disposeQueue.CompleteAdding(); @@ -832,7 +793,7 @@ void TopLevelPersist(TrieNode tn, Hash256? address2, TreePath path) LastPersistedBlockNumber = commitSet.BlockNumber; } - private void PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePath path, BlockCommitSet commitSet, + private void PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePath path, Action? persistedNodeRecorder, WriteFlags writeFlags, BlockingCollection disposeQueue) { @@ -842,7 +803,7 @@ private void PersistNodeStartingFrom(TrieNode tn, Hash256 address2, TreePath pat void DoPersist(TrieNode node, Hash256? address3, TreePath path2) { persistedNodeRecorder?.Invoke(path2, address3, node); - PersistNode(address3, path2, node, commitSet.BlockNumber, writeFlags, writeBatch); + PersistNode(address3, path2, node, writeBatch, writeFlags); persistedNodeCount++; if (persistedNodeCount % 512 == 0) @@ -856,23 +817,20 @@ void DoPersist(TrieNode node, Hash256? address3, TreePath path2) disposeQueue.Add(writeBatch); } - private void PersistNode(Hash256? address, in TreePath path, TrieNode currentNode, long blockNumber, WriteFlags writeFlags = WriteFlags.None, INodeStorage.WriteBatch? writeBatch = null) + private void PersistNode(Hash256? address, in TreePath path, TrieNode currentNode, INodeStorage.WriteBatch writeBatch, WriteFlags writeFlags = WriteFlags.None) { - writeBatch ??= _currentBatch ??= _nodeStorage.StartWriteBatch(); ArgumentNullException.ThrowIfNull(currentNode); if (currentNode.Keccak is not null) { - Debug.Assert(currentNode.LastSeen >= 0, $"Cannot persist a dangling node (without {(nameof(TrieNode.LastSeen))} value set)."); // Note that the LastSeen value here can be 'in the future' (greater than block number // if we replaced a newly added node with an older copy and updated the LastSeen value. // Here we reach it from the old root so it appears to be out of place but it is correct as we need // to prevent it from being removed from cache and also want to have it persisted. - if (_logger.IsTrace) _logger.Trace($"Persisting {nameof(TrieNode)} {currentNode} in snapshot {blockNumber}."); + if (_logger.IsTrace) _logger.Trace($"Persisting {nameof(TrieNode)} {currentNode}."); writeBatch.Set(address, path, currentNode.Keccak, currentNode.FullRlp, writeFlags); currentNode.IsPersisted = true; - currentNode.LastSeen = Math.Max(blockNumber, currentNode.LastSeen); IncrementPersistedNodesCount(); } else @@ -912,19 +870,6 @@ private void DequeueOldCommitSets() } } - private void EnsureCommitSetExistsForBlock(long blockNumber) - { - if (CurrentPackage is null) - { - if (_pruningStrategy.PruningEnabled && !Monitor.IsEntered(_dirtyNodesLock)) - { - Monitor.Enter(_dirtyNodesLock); - } - - CreateCommitSet(blockNumber); - } - } - private void AnnounceReorgBoundaries() { if (LatestCommittedBlockNumber < 1) @@ -993,7 +938,7 @@ private void PersistOnShutdown() { BlockCommitSet blockCommitSet = candidateSets[index]; if (_logger.IsDebug) _logger.Debug($"Persisting on disposal {blockCommitSet} (cache memory at {MemoryUsedByDirtyCache})"); - PersistBlockCommitSet(null, blockCommitSet, writeBatch); + ParallelPersistBlockCommitSet(blockCommitSet); } writeBatch.Dispose(); @@ -1031,10 +976,8 @@ void ClearCommitSetQueue() } commitSetCount++; - using INodeStorage.WriteBatch writeBatch = _nodeStorage.StartWriteBatch(); - PersistBlockCommitSet(null, commitSet, writeBatch); + ParallelPersistBlockCommitSet(commitSet); } - PruneCurrentSet(); } if (!(_commitSetQueue?.IsEmpty ?? true)) @@ -1130,19 +1073,50 @@ public bool HasRoot(Hash256 stateRoot) return true; } - private class TrieStoreCommitter( + private class BlockCommitter( + TrieStore trieStore, + BlockCommitSet commitSet + ) : IBlockCommitter + { + internal TrieNode? StateRoot = null; + private int _concurrency = trieStore._pruningStrategy.PruningEnabled ? Environment.ProcessorCount : 0; + + public void Dispose() + { + trieStore.FinishBlockCommit(commitSet, StateRoot); + } + + public ICommitter GetTrieCommitter(Hash256? address, TrieNode? root, WriteFlags writeFlags) + { + if (address is null) StateRoot = root; + return trieStore._pruningStrategy.PruningEnabled + ? new PruningTrieStoreCommitter(this, trieStore, commitSet.BlockNumber, address, root) + : new NonPruningTrieStoreCommitter(trieStore, address, trieStore._nodeStorage.StartWriteBatch(), writeFlags); + } + + public bool TryRequestConcurrencyQuota() + { + if (Interlocked.Decrement(ref _concurrency) >= 0) + { + return true; + } + + ReturnConcurrencyQuota(); + return false; + } + + public void ReturnConcurrencyQuota() => Interlocked.Increment(ref _concurrency); + } + + private class PruningTrieStoreCommitter( + BlockCommitter blockCommitter, TrieStore trieStore, - TrieType trieType, long blockNumber, Hash256? address, - TrieNode? root, - WriteFlags writeFlags, - int concurrency + TrieNode root ) : ICommitter { private readonly bool _needToResetRoot = root is not null && root.IsDirty; - private int _concurrency = concurrency; - private TrieNode? _root = root; public void Dispose() { @@ -1150,29 +1124,40 @@ public void Dispose() { // During commit it PatriciaTrie, the root may get resolved to an existing node (same keccak). // This ensure that the root that we use here is the same. - _root = trieStore.FindCachedOrUnknown(address, TreePath.Empty, _root?.Keccak); + // This is only needed for state tree as the root need to be put in the block commit set. + if (address == null) blockCommitter.StateRoot = trieStore.FindCachedOrUnknown(address, TreePath.Empty, root?.Keccak); } - - trieStore.FinishBlockCommit(trieType, blockNumber, address, _root, writeFlags); } public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) => - trieStore.CommitNode(blockNumber, address, ref path, nodeCommitInfo, writeFlags: writeFlags); + trieStore.CommitAndInsertToDirtyNodes(blockNumber, address, ref path, nodeCommitInfo); + + public bool TryRequestConcurrentQuota() => blockCommitter.TryRequestConcurrencyQuota(); - public bool CanSpawnTask() + public void ReturnConcurrencyQuota() => blockCommitter.ReturnConcurrencyQuota(); + } + + private class NonPruningTrieStoreCommitter( + TrieStore trieStore, + Hash256? address, + INodeStorage.WriteBatch writeBatch, + WriteFlags writeFlags = WriteFlags.None + ) : ICommitter + { + public void Dispose() { - if (Interlocked.Decrement(ref _concurrency) >= 0) - { - return true; - } + writeBatch.Dispose(); + } - ReturnConcurrencyQuota(); - return false; + public void CommitNode(ref TreePath path, NodeCommitInfo nodeCommitInfo) + { + trieStore.CommitAndPersistNode(address, ref path, nodeCommitInfo, writeFlags: writeFlags, writeBatch: writeBatch); } - public void ReturnConcurrencyQuota() => Interlocked.Increment(ref _concurrency); - } + public bool TryRequestConcurrentQuota() => false; + public void ReturnConcurrencyQuota() { } + } internal static class HashHelpers { diff --git a/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs b/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs index e93f1096d06..499f6f18501 100644 --- a/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs +++ b/src/Nethermind/Nethermind.Trie/TrieStoreWithReadFlags.cs @@ -10,8 +10,8 @@ namespace Nethermind.Trie; public class TrieStoreWithReadFlags(IScopedTrieStore implementation, ReadFlags flags) : TrieNodeResolverWithReadFlags(implementation, flags), IScopedTrieStore { - public ICommitter BeginCommit(TrieType trieType, long blockNumber, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => - implementation.BeginCommit(trieType, blockNumber, root, writeFlags); + public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => + implementation.BeginCommit(root, writeFlags); public bool IsPersisted(in TreePath path, in ValueHash256 keccak) => implementation.IsPersisted(in path, in keccak); From 31c51317cacccac037502f9e0866c2598605c088 Mon Sep 17 00:00:00 2001 From: Ruben Buniatyan Date: Wed, 16 Oct 2024 18:34:00 +0200 Subject: [PATCH 02/13] Update PPA publish prerequisites (#7618) --- .github/workflows/publish-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 210889592d7..1964a1a03f1 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -26,7 +26,7 @@ jobs: echo "Import GPG owner trust" echo ${{ secrets.GPG_OWNERTRUST }} | base64 --decode | gpg --import-ownertrust - name: Install PPA dependencies - run: sudo apt-get update && sudo apt-get install debhelper devscripts -y + run: sudo apt-get update && sudo apt-get install build-essential debhelper devscripts -y - name: Submit package env: PPA_GPG_KEYID: ${{ secrets.PPA_GPG_KEYID }} From ae3e7a68a2176732816cc94f67f4ce111be91f9b Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 17 Oct 2024 02:35:14 +0800 Subject: [PATCH 03/13] Refactor/More proper use of DI for Synchronizer (#7500) Co-authored-by: Lukasz Rozmej Co-authored-by: Alex --- src/Nethermind/Directory.Packages.props | 3 + .../Nethermind.Api/IApiWithBlockchain.cs | 8 + .../Nethermind.Api/IApiWithNetwork.cs | 15 +- .../Nethermind.Api/IApiWithStores.cs | 8 + src/Nethermind/Nethermind.Api/IBasicApi.cs | 18 +- .../Nethermind.Api/NethermindApi.cs | 9 +- .../Synchronization/ISyncConfig.cs | 3 + .../Synchronization/SyncConfig.cs | 1 + .../ContainerBuilderExtensionsTests.cs | 113 +++++ .../ContainerBuilderExtensions.cs | 125 +++++ .../Nethermind.Core/Nethermind.Core.csproj | 3 + src/Nethermind/Nethermind.Db/IDbProvider.cs | 32 ++ src/Nethermind/Nethermind.Db/ITunableDb.cs | 9 +- .../Steps/InitializeNetwork.cs | 43 +- .../Nethermind.Merge.Plugin/MergePlugin.cs | 53 +- .../MergeBlockDownloaderFactory.cs | 81 --- .../Synchronization/MergeSynchronizer.cs | 141 +++--- .../Nethermind.Optimism/OptimismPlugin.cs | 52 +- .../Ethereum/ContextWithMocks.cs | 13 +- .../DbTuner/SyncDbTunerTests.cs | 9 - .../OldStyleFullSynchronizerTests.cs | 54 +- .../MultiSyncModeSelectorTests.Scenario.cs | 5 +- .../SyncThreadTests.cs | 52 +- .../SynchronizerTests.cs | 96 ++-- .../Blocks/IBlockDownloaderFactory.cs | 65 --- .../DbTuner/SyncDbOptimizer.cs | 22 +- .../FastBlocks/BodiesSyncFeed.cs | 6 +- .../FastBlocks/ReceiptsSyncFeed.cs | 4 +- .../FastSync/TreeSync.cs | 8 + .../ISynchronizer.cs | 3 - .../MallocTrimmer.cs | 9 + .../ParallelSync/ISyncFeed.cs | 2 +- .../ParallelSync/MultiSyncModeSelector.cs | 7 +- .../ParallelSync/NoopSyncFeed.cs | 45 ++ .../ParallelSync/SyncDispatcher.cs | 14 + .../ParallelSync/SyncProgressResolver.cs | 23 +- .../SnapSync/ProgressTracker.cs | 8 + .../SnapSync/SnapProvider.cs | 4 +- .../SyncFeedComponent.cs | 30 ++ .../Synchronizer.cs | 464 ++++++++---------- 40 files changed, 911 insertions(+), 749 deletions(-) create mode 100644 src/Nethermind/Nethermind.Core.Test/ContainerBuilderExtensionsTests.cs create mode 100644 src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs delete mode 100644 src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs delete mode 100644 src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs create mode 100644 src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs create mode 100644 src/Nethermind/Nethermind.Synchronization/SyncFeedComponent.cs diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index 253c598a39d..b7fae1de28d 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -6,6 +6,8 @@ + + @@ -36,6 +38,7 @@ + diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index 61bceb25ae4..da24eaa918b 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only #nullable enable +using Autofac; using Nethermind.Blockchain; using Nethermind.Blockchain.Filters; using Nethermind.Blockchain.FullPruning; @@ -100,5 +101,12 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory INodeStorageFactory NodeStorageFactory { get; set; } BackgroundTaskScheduler BackgroundTaskScheduler { get; set; } CensorshipDetector CensorshipDetector { get; set; } + + public ContainerBuilder ConfigureContainerBuilderFromApiWithBlockchain(ContainerBuilder builder) + { + return ConfigureContainerBuilderFromApiWithStores(builder) + .AddPropertiesFrom(this) + .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)); + } } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 0f5d2262535..0b0e05e2a6d 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -2,7 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; +using Autofac; using Nethermind.Consensus; +using Nethermind.Core; using Nethermind.Core.PubSub; using Nethermind.Grpc; using Nethermind.JsonRpc; @@ -16,6 +18,7 @@ using Nethermind.Synchronization; using Nethermind.Synchronization.Peers; using Nethermind.Sockets; +using Nethermind.Synchronization.ParallelSync; namespace Nethermind.Api { @@ -41,12 +44,22 @@ public interface IApiWithNetwork : IApiWithBlockchain IJsonRpcLocalStats? JsonRpcLocalStats { get; set; } ISessionMonitor? SessionMonitor { get; set; } IStaticNodesManager? StaticNodesManager { get; set; } - ISynchronizer? Synchronizer { get; set; } + ISynchronizer? Synchronizer { get; } + ISyncModeSelector SyncModeSelector { get; } + ISyncProgressResolver? SyncProgressResolver { get; } IPivot? Pivot { get; set; } ISyncPeerPool? SyncPeerPool { get; set; } IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool { get; set; } ISyncServer? SyncServer { get; set; } IWebSocketsManager WebSocketsManager { get; set; } ISubscriptionFactory? SubscriptionFactory { get; set; } + + IContainer? ApiWithNetworkServiceContainer { get; set; } + + public ContainerBuilder ConfigureContainerBuilderFromApiWithNetwork(ContainerBuilder builder) + { + return ConfigureContainerBuilderFromApiWithBlockchain(builder) + .AddPropertiesFrom(this); + } } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index fa2911c2ebe..eed8ed62430 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Autofac; using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus; +using Nethermind.Core; using Nethermind.Crypto; using Nethermind.Db.Blooms; using Nethermind.Facade.Find; @@ -30,5 +32,11 @@ public interface IApiWithStores : IBasicApi IReceiptMonitor? ReceiptMonitor { get; set; } IWallet? Wallet { get; set; } IBlockStore? BadBlocksStore { get; set; } + + public ContainerBuilder ConfigureContainerBuilderFromApiWithStores(ContainerBuilder builder) + { + return ConfigureContainerBuilderFromBasicApi(builder) + .AddPropertiesFrom(this); + } } } diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index 3b78f16a865..1ba0adb91f9 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -1,11 +1,14 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; using System.IO.Abstractions; using System.Linq; +using Autofac; using Nethermind.Abi; using Nethermind.Api.Extensions; +using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Core; using Nethermind.Core.Specs; @@ -17,7 +20,6 @@ using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; using Nethermind.Synchronization; -using Nethermind.Synchronization.ParallelSync; namespace Nethermind.Api { @@ -38,10 +40,9 @@ public interface IBasicApi ILogManager LogManager { get; set; } ProtectedPrivateKey? OriginalSignerKey { get; set; } IReadOnlyList Plugins { get; } + [SkipServiceCollection] string SealEngineType { get; set; } ISpecProvider? SpecProvider { get; set; } - ISyncModeSelector SyncModeSelector { get; set; } - ISyncProgressResolver? SyncProgressResolver { get; set; } IBetterPeerStrategy? BetterPeerStrategy { get; set; } ITimestamper Timestamper { get; } ITimerFactory TimerFactory { get; } @@ -57,5 +58,16 @@ public IEnumerable GetConsensusWrapperPlugins() => public IEnumerable GetSynchronizationPlugins() => Plugins.OfType(); + + public ContainerBuilder ConfigureContainerBuilderFromBasicApi(ContainerBuilder builder) + { + builder + .AddPropertiesFrom(this) + .AddSingleton(ConfigProvider.GetConfig()); + + DbProvider!.ConfigureServiceCollection(builder); + + return builder; + } } } diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index 56193492d01..566a094eaa0 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO.Abstractions; +using Autofac; using Nethermind.Abi; using Nethermind.Api.Extensions; using Nethermind.Blockchain; @@ -186,14 +187,14 @@ public ISealEngine SealEngine public ISessionMonitor? SessionMonitor { get; set; } public ISpecProvider? SpecProvider { get; set; } public IPoSSwitcher PoSSwitcher { get; set; } = NoPoS.Instance; - public ISyncModeSelector SyncModeSelector { get; set; } = null!; + public ISyncModeSelector SyncModeSelector => ApiWithNetworkServiceContainer?.Resolve()!; - public ISyncProgressResolver? SyncProgressResolver { get; set; } + public ISyncProgressResolver? SyncProgressResolver => ApiWithNetworkServiceContainer?.Resolve(); public IBetterPeerStrategy? BetterPeerStrategy { get; set; } public IPivot? Pivot { get; set; } public ISyncPeerPool? SyncPeerPool { get; set; } public IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool { get; set; } - public ISynchronizer? Synchronizer { get; set; } + public ISynchronizer? Synchronizer => ApiWithNetworkServiceContainer?.Resolve(); public ISyncServer? SyncServer { get; set; } public IWorldState? WorldState { get; set; } public IReadOnlyStateProvider? ChainHeadStateProvider { get; set; } @@ -243,5 +244,7 @@ public ISealEngine SealEngine public CompositePruningTrigger PruningTrigger { get; } = new(); public IProcessExitSource? ProcessExit { get; set; } public CompositeTxGossipPolicy TxGossipPolicy { get; } = new(); + + public IContainer? ApiWithNetworkServiceContainer { get; set; } } } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs index 35c6c3fc1ac..cfee2a225fd 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs @@ -143,4 +143,7 @@ public interface ISyncConfig : IConfig [ConfigItem(Description = "_Technical._ MultiSyncModeSelector sync mode timer loop interval. Used for testing.", DefaultValue = "1000", HiddenFromDocs = true)] int MultiSyncModeSelectorLoopTimerMs { get; set; } + + [ConfigItem(Description = "_Technical._ MultiSyncModeSelector will wait for header to completely sync first.", DefaultValue = "false", HiddenFromDocs = true)] + bool NeedToWaitForHeader { get; set; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs index d2645336586..c467650ca7a 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs @@ -65,6 +65,7 @@ public string? PivotHash public int MallocTrimIntervalSec { get; set; } = 300; public bool? SnapServingEnabled { get; set; } = null; public int MultiSyncModeSelectorLoopTimerMs { get; set; } = 1000; + public bool NeedToWaitForHeader { get; set; } public bool TrieHealing { get; set; } = true; public override string ToString() diff --git a/src/Nethermind/Nethermind.Core.Test/ContainerBuilderExtensionsTests.cs b/src/Nethermind/Nethermind.Core.Test/ContainerBuilderExtensionsTests.cs new file mode 100644 index 00000000000..28e890c8499 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/ContainerBuilderExtensionsTests.cs @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Autofac; +using FluentAssertions; +using NUnit.Framework; + +namespace Nethermind.Core.Test; + +public class ContainerBuilderExtensionsTests +{ + [Test] + public void AddPropertiesFrom_CanAddProperties() + { + ITestInterface interfaceImplementation = new InterfaceImplementation(); + IContainer sp = new ContainerBuilder() + .AddPropertiesFrom(interfaceImplementation) + .Build(); + + sp.ResolveOptional().Should().NotBeNull(); + sp.ResolveOptional().Should().BeNull(); + sp.ResolveOptional().Should().BeNull(); + sp.ResolveOptional().Should().BeNull(); + } + + [Test] + public void TestRegisterNamedComponent() + { + IContainer sp = new ContainerBuilder() + .AddScoped() + .AddScoped() + .RegisterNamedComponentInItsOwnLifetime("custom", cfg => + { + // Override it in custom + cfg.AddScoped(); + }) + .Build(); + + using (ILifetimeScope scope = sp.BeginLifetimeScope()) + { + scope.Resolve().Property.Should().BeOfType(); + } + + MainComponentDependency customMainComponentDependency = sp.ResolveNamed("custom").Property; + sp.ResolveNamed("custom").Property.Should().BeOfType(); + + sp.Dispose(); + + customMainComponentDependency.WasDisposed.Should().BeTrue(); + } + + private class MainComponent(MainComponentDependency mainComponentDependency, ILifetimeScope scope) : IDisposable + { + public MainComponentDependency Property => mainComponentDependency; + + public void Dispose() + { + scope.Dispose(); + } + } + + private class MainComponentDependency : IDisposable + { + public bool WasDisposed { get; set; } + + public void Dispose() + { + WasDisposed = true; + } + } + + private class MainComponentDependencySubClass : MainComponentDependency + { + } + + private class InterfaceImplementation : ITestInterface + { + public DeclaredService TheService { get; set; } = new DeclaredService(); + public DeclaredButNullService? NullService { get; set; } = null; + public Ignored IgnoredService { get; set; } = new Ignored(); + public DeclaredInBase BaseService { get; set; } = new DeclaredInBase(); + } + + private interface ITestInterface : ITestInterfaceBase + { + DeclaredService TheService { get; set; } + DeclaredButNullService? NullService { get; set; } + + [SkipServiceCollection] + Ignored IgnoredService { get; set; } + } + + private interface ITestInterfaceBase + { + DeclaredInBase BaseService { get; set; } + } + + private class DeclaredInBase { } + private class DeclaredService { } + private class DeclaredButNullService { } + private class Ignored { } + + private class DisposableService : IDisposable + { + public bool WasDisposed { get; set; } = false; + + public void Dispose() + { + WasDisposed = true; + } + } +} diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs new file mode 100644 index 00000000000..1c30a830b55 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Autofac; +using Autofac.Features.AttributeFilters; + +namespace Nethermind.Core; + +public static class ContainerBuilderExtensions +{ + /// + /// Add all properties as singleton. It get them ahead of time instead of lazily to prevent the final service provider + /// from disposing it. To prevent a property from being included, use . + /// + /// + /// + /// + /// + public static ContainerBuilder AddPropertiesFrom(this ContainerBuilder configuration, T source) where T : class + { + Type t = typeof(T); + + IEnumerable properties = t + .GetProperties(BindingFlags.GetProperty | BindingFlags.Instance | BindingFlags.Public | BindingFlags.DeclaredOnly) + .Where(p => p.GetCustomAttribute() == null); + + foreach (PropertyInfo propertyInfo in properties) + { + object? val = propertyInfo.GetValue(source); + if (val != null) + { + configuration.RegisterInstance(val).As(propertyInfo.PropertyType); + } + } + + return configuration; + } + + public static ContainerBuilder AddSingleton(this ContainerBuilder builder) where T : notnull + { + builder.RegisterType() + .As() + .WithAttributeFiltering() + .SingleInstance(); + + return builder; + } + + public static ContainerBuilder AddSingleton(this ContainerBuilder builder, T instance) where T : class + { + builder.RegisterInstance(instance) + .As() + .SingleInstance(); + + return builder; + } + + public static ContainerBuilder AddSingleton(this ContainerBuilder builder) where TImpl : notnull where T : notnull + { + builder.RegisterType() + .As() + .AsSelf() + .WithAttributeFiltering() + .SingleInstance(); + + return builder; + } + + public static ContainerBuilder AddKeyedSingleton(this ContainerBuilder builder, string key, T instance) where T : class + { + builder.RegisterInstance(instance) + .Named(key) + .SingleInstance(); + + return builder; + } + + public static ContainerBuilder AddScoped(this ContainerBuilder builder) where T : notnull + { + builder.RegisterType() + .As() + .AsSelf() + .WithAttributeFiltering() + .InstancePerLifetimeScope(); + + return builder; + } + + public static ContainerBuilder AddScoped(this ContainerBuilder builder) where TImpl : notnull where T : notnull + { + builder.RegisterType() + .As() + .WithAttributeFiltering() + .InstancePerLifetimeScope(); + + return builder; + } + + /// + /// A convenient way of creating a service whose member can be configured indipendent of other instance of the same + /// type (assuming the type is of lifetime scope). This is useful for same type with multiple configuration + /// or a graph of multiple same type. The T is expected to be of a main container of sort that contains the + /// main service of interest. + /// Note: The T should dispose an injected ILifetimeScope on dispose as ILifetimeScope is not automatically disposed + /// when parent scope is disposed. + /// + public static ContainerBuilder RegisterNamedComponentInItsOwnLifetime(this ContainerBuilder builder, string name, Action configurator) where T : notnull + { + builder.Register(ctx => ctx.BeginLifetimeScope(configurator).Resolve()) + .Named(name); + + return builder; + } +} + +/// +/// Mark a property so that it is not picked up by `AddPropertiesFrom`. +/// +public class SkipServiceCollectionAttribute : Attribute +{ +} diff --git a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj index de2832e6ea5..930e96abe48 100644 --- a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj +++ b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj @@ -5,7 +5,10 @@ + + + diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 7786271bc74..6a0380b0446 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -3,6 +3,9 @@ using System; using System.Collections.Generic; +using Autofac; +using Microsoft.Extensions.DependencyInjection; +using Nethermind.Core; namespace Nethermind.Db { @@ -30,5 +33,34 @@ public interface IDbProvider : IDisposable void RegisterDb(string dbName, T db) where T : class, IDb; void RegisterColumnDb(string dbName, IColumnsDb db); IEnumerable> GetAllDbMeta(); + + void ConfigureServiceCollection(ContainerBuilder sc) + { + sc.AddSingleton(this); + + // TODO: Have hooks that automatically get these + string[] dbNames = [ + DbNames.State, + DbNames.Code, + DbNames.Metadata, + DbNames.Blocks, + DbNames.Headers, + DbNames.BlockInfos, + DbNames.BadBlocks, + DbNames.Bloom, + DbNames.Metadata, + ]; + foreach (string dbName in dbNames) + { + var db = GetDb(dbName); + sc.AddKeyedSingleton(dbName, db); + sc.AddKeyedSingleton(dbName, db); + sc.AddKeyedSingleton(dbName, db as ITunableDb ?? new NoopTunableDb()); + } + + IColumnsDb receiptColumnDb = GetColumnDb(DbNames.Receipts); + sc.AddSingleton>(receiptColumnDb); + sc.AddKeyedSingleton(DbNames.Receipts, receiptColumnDb as ITunableDb ?? new NoopTunableDb()); + } } } diff --git a/src/Nethermind/Nethermind.Db/ITunableDb.cs b/src/Nethermind/Nethermind.Db/ITunableDb.cs index 7584364badd..00b0edf0f0c 100644 --- a/src/Nethermind/Nethermind.Db/ITunableDb.cs +++ b/src/Nethermind/Nethermind.Db/ITunableDb.cs @@ -3,7 +3,7 @@ namespace Nethermind.Db; -public interface ITunableDb : IDb +public interface ITunableDb { public void Tune(TuneType type); @@ -18,3 +18,10 @@ enum TuneType HashDb } } + +public class NoopTunableDb : ITunableDb +{ + public void Tune(ITunableDb.TuneType type) + { + } +} diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 7f4636d56be..db2854c0003 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -3,9 +3,13 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; using Nethermind.Blockchain.Synchronization; @@ -126,39 +130,22 @@ private async Task Initialize(CancellationToken cancellationToken) if (_api.Synchronizer is null) { - BlockDownloaderFactory blockDownloaderFactory = new BlockDownloaderFactory( - _api.SpecProvider!, - _api.BlockValidator!, - _api.SealValidator!, - _api.BetterPeerStrategy!, - _api.LogManager); + if (_api.ChainSpec.SealEngineType == SealEngineType.Clique) + _syncConfig.NeedToWaitForHeader = true; // Should this be in chainspec itself? - _api.Synchronizer ??= new Synchronizer( - _api.DbProvider, - _api.NodeStorageFactory.WrapKeyValueStore(_api.DbProvider.StateDb), - _api.SpecProvider!, - _api.BlockTree, - _api.ReceiptStorage!, - _api.SyncPeerPool, - _api.NodeStatsManager!, - _syncConfig, - blockDownloaderFactory, - _api.Pivot, - _api.ProcessExit!, - _api.BetterPeerStrategy, - _api.ChainSpec, - _api.StateReader!, - _api.LogManager); - } + ContainerBuilder builder = new ContainerBuilder(); + _api.ConfigureContainerBuilderFromApiWithNetwork(builder) + .AddSingleton(No.BeaconSync); + builder.RegisterModule(new SynchronizerModule(_syncConfig)); + IContainer container = builder.Build(); - _api.SyncModeSelector = _api.Synchronizer.SyncModeSelector; - _api.SyncProgressResolver = _api.Synchronizer.SyncProgressResolver; + _api.ApiWithNetworkServiceContainer = container; + _api.DisposeStack.Append(container); + } _api.EthSyncingInfo = new EthSyncingInfo(_api.BlockTree, _api.ReceiptStorage!, _syncConfig, - _api.SyncModeSelector, _api.SyncProgressResolver, _api.LogManager); + _api.SyncModeSelector!, _api.SyncProgressResolver!, _api.LogManager); _api.TxGossipPolicy.Policies.Add(new SyncedTxGossipPolicy(_api.SyncModeSelector)); - _api.DisposeStack.Push(_api.SyncModeSelector); - _api.DisposeStack.Push(_api.Synchronizer); ISyncServer syncServer = _api.SyncServer = new SyncServer( _api.TrieStore!.TrieNodeRlpStore, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 9d387b5d200..81b83425ace 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -6,6 +6,10 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Core; +using Autofac.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; using Nethermind.Blockchain; @@ -30,6 +34,8 @@ using Nethermind.Merge.Plugin.Handlers; using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.Synchronization; +using Nethermind.Synchronization; +using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.TxPool; @@ -436,43 +442,24 @@ public Task InitSynchronization() _api.Pivot = _beaconPivot; - MergeBlockDownloaderFactory blockDownloaderFactory = new MergeBlockDownloaderFactory( - _poSSwitcher, - _beaconPivot, - _api.SpecProvider, - _api.BlockValidator!, - _api.SealValidator!, - _syncConfig, - _api.BetterPeerStrategy!, - new FullStateFinder(_api.BlockTree, _api.StateReader), - _api.LogManager); + ContainerBuilder builder = new ContainerBuilder(); - MergeSynchronizer synchronizer = new MergeSynchronizer( - _api.DbProvider, - _api.NodeStorageFactory.WrapKeyValueStore(_api.DbProvider.StateDb), - _api.SpecProvider!, - _api.BlockTree!, - _api.ReceiptStorage!, - _api.SyncPeerPool, - _api.NodeStatsManager!, - _syncConfig, - blockDownloaderFactory, - _beaconPivot, - _poSSwitcher, - _mergeConfig, - _invalidChainTracker, - _api.ProcessExit!, - _api.BetterPeerStrategy, - _api.ChainSpec, - _beaconSync, - _api.StateReader, - _api.LogManager - ); - _api.Synchronizer = synchronizer; + _api.ConfigureContainerBuilderFromApiWithNetwork(builder) + .AddSingleton(_beaconSync) + .AddSingleton(_beaconPivot) + .AddSingleton(_mergeConfig) + .AddSingleton(_invalidChainTracker); + + builder.RegisterModule(new SynchronizerModule(_syncConfig)); + builder.RegisterModule(new MergeSynchronizerModule()); + + IContainer container = builder.Build(); + _api.ApiWithNetworkServiceContainer = container; + _api.DisposeStack.Append(container); PivotUpdator pivotUpdator = new( _api.BlockTree, - synchronizer.SyncModeSelector, + _api.SyncModeSelector, _api.SyncPeerPool, _syncConfig, _blockCacheService, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs deleted file mode 100644 index 8c3b99555bf..00000000000 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; -using Nethermind.Blockchain.Synchronization; -using Nethermind.Consensus; -using Nethermind.Consensus.Validators; -using Nethermind.Core.Specs; -using Nethermind.Logging; -using Nethermind.Synchronization; -using Nethermind.Synchronization.Blocks; -using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; -using Nethermind.Synchronization.Reporting; - - -namespace Nethermind.Merge.Plugin.Synchronization -{ - public class MergeBlockDownloaderFactory : IBlockDownloaderFactory - { - private readonly IPoSSwitcher _poSSwitcher; - private readonly IBeaconPivot _beaconPivot; - private readonly ISpecProvider _specProvider; - private readonly IBlockValidator _blockValidator; - private readonly ISealValidator _sealValidator; - private readonly IBetterPeerStrategy _betterPeerStrategy; - private readonly ILogManager _logManager; - private readonly IFullStateFinder _fullStateFinder; - private readonly ISyncConfig _syncConfig; - - public MergeBlockDownloaderFactory( - IPoSSwitcher poSSwitcher, - IBeaconPivot beaconPivot, - ISpecProvider specProvider, - IBlockValidator blockValidator, - ISealValidator sealValidator, - ISyncConfig syncConfig, - IBetterPeerStrategy betterPeerStrategy, - IFullStateFinder fullStateFinder, - ILogManager logManager) - { - _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); - _beaconPivot = beaconPivot ?? throw new ArgumentNullException(nameof(beaconPivot)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _blockValidator = blockValidator ?? throw new ArgumentNullException(nameof(blockValidator)); - _sealValidator = sealValidator ?? throw new ArgumentNullException(nameof(sealValidator)); - _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - _fullStateFinder = fullStateFinder ?? throw new ArgumentNullException(nameof(fullStateFinder)); ; - _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); ; - } - - public BlockDownloader Create(ISyncFeed syncFeed, IBlockTree blockTree, IReceiptStorage receiptStorage, - ISyncPeerPool syncPeerPool, ISyncReport syncReport) - { - ChainLevelHelper chainLevelHelper = new ChainLevelHelper(blockTree, _beaconPivot, _syncConfig, _logManager); - return new MergeBlockDownloader( - _poSSwitcher, - _beaconPivot, - syncFeed, - syncPeerPool, - blockTree, - _blockValidator, - _sealValidator, - syncReport, - receiptStorage, - _specProvider, - _betterPeerStrategy, - chainLevelHelper, - _fullStateFinder, - _logManager); - } - - public IPeerAllocationStrategyFactory CreateAllocationStrategyFactory() - { - return new MergeBlocksSyncPeerAllocationStrategyFactory(_poSSwitcher, _beaconPivot, _logManager); - } - } -} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 38e65be39e7..ac3ad98313f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -1,113 +1,58 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; +using System; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Autofac.Features.AttributeFilters; using Nethermind.Blockchain.Synchronization; -using Nethermind.Config; -using Nethermind.Consensus; -using Nethermind.Core.Specs; -using Nethermind.Db; +using Nethermind.Core; using Nethermind.Logging; -using Nethermind.Merge.Plugin.InvalidChainTracker; -using Nethermind.Specs.ChainSpecStyle; -using Nethermind.State; -using Nethermind.Stats; using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; -using Nethermind.Trie; -using Nethermind.Trie.Pruning; namespace Nethermind.Merge.Plugin.Synchronization; -public class MergeSynchronizer : Synchronizer +public class MergeSynchronizer( + [KeyFilter(nameof(BeaconHeadersSyncFeed))] SyncFeedComponent beaconHeaderComponent, + ISyncConfig syncConfig, + Synchronizer baseSynchronizer, + ILogManager logManager) + : ISynchronizer { - private readonly IPoSSwitcher _poSSwitcher; - private readonly IMergeConfig _mergeConfig; - private readonly IInvalidChainTracker _invalidChainTracker; - private BeaconHeadersSyncFeed _beaconHeadersFeed = null!; - private readonly IBeaconSyncStrategy _beaconSync; + private readonly CancellationTokenSource? _syncCancellation = new(); + private readonly ILogger _logger = logManager.GetClassLogger(); - public override ISyncModeSelector SyncModeSelector => _syncModeSelector ??= new MultiSyncModeSelector( - SyncProgressResolver, - _syncPeerPool, - _syncConfig, - _beaconSync, - _betterPeerStrategy!, - _logManager); - - public MergeSynchronizer( - IDbProvider dbProvider, - INodeStorage nodeStorage, - ISpecProvider specProvider, - IBlockTree blockTree, - IReceiptStorage receiptStorage, - ISyncPeerPool peerPool, - INodeStatsManager nodeStatsManager, - ISyncConfig syncConfig, - IBlockDownloaderFactory blockDownloaderFactory, - IPivot pivot, - IPoSSwitcher poSSwitcher, - IMergeConfig mergeConfig, - IInvalidChainTracker invalidChainTracker, - IProcessExitSource exitSource, - IBetterPeerStrategy betterPeerStrategy, - ChainSpec chainSpec, - IBeaconSyncStrategy beaconSync, - IStateReader stateReader, - ILogManager logManager) - : base( - dbProvider, - nodeStorage, - specProvider, - blockTree, - receiptStorage, - peerPool, - nodeStatsManager, - syncConfig, - blockDownloaderFactory, - pivot, - exitSource, - betterPeerStrategy, - chainSpec, - stateReader, - logManager) + public event EventHandler? SyncEvent { - _invalidChainTracker = invalidChainTracker; - _poSSwitcher = poSSwitcher; - _mergeConfig = mergeConfig; - _beaconSync = beaconSync; + add => baseSynchronizer.SyncEvent += value; + remove => baseSynchronizer.SyncEvent -= value; } - public override void Start() + public void Start() { - if (!_syncConfig.SynchronizationEnabled) + if (!syncConfig.SynchronizationEnabled) { return; } - base.Start(); + baseSynchronizer.Start(); StartBeaconHeadersComponents(); WireMultiSyncModeSelector(); } - private void StartBeaconHeadersComponents() + public Task StopAsync() { - FastBlocksPeerAllocationStrategyFactory fastFactory = new(); - _beaconHeadersFeed = - new(_poSSwitcher, _blockTree, _syncPeerPool, _syncConfig, _syncReport, _pivot, _mergeConfig, _invalidChainTracker, _logManager); - BeaconHeadersSyncDownloader beaconHeadersDownloader = new(_logManager); - - SyncDispatcher dispatcher = CreateDispatcher( - _beaconHeadersFeed!, - beaconHeadersDownloader, - fastFactory - ); + _syncCancellation?.Cancel(); + return baseSynchronizer.StopAsync(); + } - dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + private void StartBeaconHeadersComponents() + { + beaconHeaderComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -122,6 +67,36 @@ private void StartBeaconHeadersComponents() private void WireMultiSyncModeSelector() { - WireFeedWithModeSelector(_beaconHeadersFeed); + baseSynchronizer.WireFeedWithModeSelector(beaconHeaderComponent.Feed); + } + + public void Dispose() + { + baseSynchronizer.Dispose(); + } +} + +public class MergeSynchronizerModule : Module +{ + protected override void Load(ContainerBuilder builder) + { + builder + .RegisterType() + .As() + .As>() + .InstancePerLifetimeScope(); + + builder + .AddSingleton() + .AddSingleton() + .AddScoped, MergeBlocksSyncPeerAllocationStrategyFactory>() + + .RegisterNamedComponentInItsOwnLifetime>(nameof(BeaconHeadersSyncFeed), ConfigureBeaconHeader); + } + + private void ConfigureBeaconHeader(ContainerBuilder scopeConfig) + { + scopeConfig.AddScoped, BeaconHeadersSyncFeed>() + .AddScoped, BeaconHeadersSyncDownloader>(); } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 5bbd22e36be..76633fbf876 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -2,7 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Linq; using System.Threading.Tasks; +using Autofac; using Nethermind.Api; using Nethermind.Api.Extensions; using Nethermind.Consensus; @@ -23,13 +25,12 @@ using Nethermind.Consensus.Validators; using Nethermind.Core; using Nethermind.Merge.Plugin.Synchronization; -using Nethermind.Synchronization.ParallelSync; using Nethermind.HealthChecks; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; using Nethermind.Serialization.Rlp; using Nethermind.Optimism.Rpc; -using Nethermind.Db; +using Nethermind.Synchronization; namespace Nethermind.Optimism; @@ -151,42 +152,25 @@ public Task InitSynchronization() _api.BetterPeerStrategy = new MergeBetterPeerStrategy(_api.BetterPeerStrategy, _api.PoSSwitcher, _beaconPivot, _api.LogManager); _api.Pivot = _beaconPivot; - MergeBlockDownloaderFactory blockDownloaderFactory = new MergeBlockDownloaderFactory( - _api.PoSSwitcher, - _beaconPivot, - _api.SpecProvider, - _api.BlockValidator!, - _api.SealValidator!, - _syncConfig, - _api.BetterPeerStrategy!, - new FullStateFinder(_api.BlockTree, _api.StateReader!), - _api.LogManager); + ContainerBuilder builder = new ContainerBuilder(); + ((INethermindApi)_api).ConfigureContainerBuilderFromApiWithNetwork(builder) + .AddSingleton(_beaconSync) + .AddSingleton(_beaconPivot) + .AddSingleton(_api.PoSSwitcher) + .AddSingleton(_mergeConfig) + .AddSingleton(_invalidChainTracker); - _api.Synchronizer = new MergeSynchronizer( - _api.DbProvider, - _api.NodeStorageFactory.WrapKeyValueStore(_api.DbProvider.StateDb), - _api.SpecProvider!, - _api.BlockTree!, - _api.ReceiptStorage!, - _api.SyncPeerPool, - _api.NodeStatsManager!, - _syncConfig, - blockDownloaderFactory, - _beaconPivot, - _api.PoSSwitcher, - _mergeConfig, - _invalidChainTracker, - _api.ProcessExit!, - _api.BetterPeerStrategy, - _api.ChainSpec, - _beaconSync, - _api.StateReader!, - _api.LogManager - ); + builder.RegisterModule(new SynchronizerModule(_syncConfig)); + builder.RegisterModule(new MergeSynchronizerModule()); + + IContainer container = builder.Build(); + + _api.ApiWithNetworkServiceContainer = container; + _api.DisposeStack.Append(container); _ = new PivotUpdator( _api.BlockTree, - _api.Synchronizer.SyncModeSelector, + _api.SyncModeSelector, _api.SyncPeerPool, _syncConfig, _blockCacheService, diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs index ef3c4c9a68a..9b0a4f49d77 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.IO.Abstractions; +using Autofac; using Nethermind.Api; using Nethermind.Blockchain; using Nethermind.Blockchain.Filters; @@ -46,6 +47,7 @@ using Nethermind.Trie; using NSubstitute; using Nethermind.Blockchain.Blocks; +using Nethermind.Core; using Nethermind.Facade.Find; namespace Nethermind.Runner.Test.Ethereum @@ -76,7 +78,6 @@ public static NethermindApi ContextWithMocks() StaticNodesManager = Substitute.For(), BloomStorage = Substitute.For(), Sealer = Substitute.For(), - Synchronizer = Substitute.For(), BlockchainProcessor = Substitute.For(), BlockProducer = Substitute.For(), DiscoveryApp = Substitute.For(), @@ -103,7 +104,6 @@ public static NethermindApi ContextWithMocks() EngineSignerStore = Substitute.For(), NodeStatsManager = Substitute.For(), RpcModuleProvider = Substitute.For(), - SyncModeSelector = Substitute.For(), SyncPeerPool = Substitute.For(), PeerDifficultyRefreshPool = Substitute.For(), WebSocketsManager = Substitute.For(), @@ -117,10 +117,15 @@ public static NethermindApi ContextWithMocks() TxValidator = new TxValidator(MainnetSpecProvider.Instance.ChainId), UnclesValidator = Substitute.For(), BlockProductionPolicy = Substitute.For(), - SyncProgressResolver = Substitute.For(), BetterPeerStrategy = Substitute.For(), ReceiptMonitor = Substitute.For(), - BadBlocksStore = Substitute.For() + BadBlocksStore = Substitute.For(), + + ApiWithNetworkServiceContainer = new ContainerBuilder() + .AddSingleton(Substitute.For()) + .AddSingleton(Substitute.For()) + .AddSingleton(Substitute.For()) + .Build(), }; api.WorldStateManager = new ReadOnlyWorldStateManager(api.DbProvider, Substitute.For(), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs index 8c7954d8b92..ffc2251face 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs @@ -53,15 +53,6 @@ public void Setup() _receiptDb); } - [TearDown] - public void TearDown() - { - _blockDb?.Dispose(); - _codeDb?.Dispose(); - _receiptDb?.Dispose(); - _stateDb?.Dispose(); - } - [Test] public void WhenSnapIsOn_TriggerStateDbTune() { diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index d5e3adcbeae..89bd50a3ca4 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -4,7 +4,10 @@ using System; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; @@ -15,6 +18,7 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; using Nethermind.Core.Timers; using Nethermind.Db; @@ -24,6 +28,7 @@ using Nethermind.State; using Nethermind.Stats; using Nethermind.Synchronization.Blocks; +using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; @@ -61,31 +66,34 @@ public async Task Setup() TrieStore trieStore = new(nodeStorage, LimboLogs.Instance); TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); Pivot pivot = new(syncConfig); - BlockDownloaderFactory blockDownloaderFactory = new( - MainnetSpecProvider.Instance, - Always.Valid, - Always.Valid, - new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), - LimboLogs.Instance); IStateReader stateReader = new StateReader(trieStore, _codeDb, LimboLogs.Instance); - _synchronizer = new Synchronizer( - dbProvider, - nodeStorage, - MainnetSpecProvider.Instance, - _blockTree, - _receiptStorage, - _pool, - stats, - syncConfig, - blockDownloaderFactory, - pivot, - Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - stateReader, - LimboLogs.Instance); + ContainerBuilder builder = new ContainerBuilder() + .AddSingleton(nodeStorage) + .AddSingleton(MainnetSpecProvider.Instance) + .AddSingleton(_blockTree) + .AddSingleton(_receiptStorage) + .AddSingleton(_pool) + .AddSingleton(stats) + .AddSingleton(syncConfig) + .AddSingleton(Always.Valid) + .AddSingleton(Always.Valid) + .AddSingleton(pivot) + .AddSingleton(Substitute.For()) + .AddSingleton(bestPeerStrategy) + .AddSingleton(new ChainSpec()) + .AddSingleton(stateReader) + .AddSingleton(No.BeaconSync) + .AddSingleton(LimboLogs.Instance); + dbProvider.ConfigureServiceCollection(builder); + + builder.RegisterModule(new SynchronizerModule(syncConfig)); + + IContainer container = builder.Build(); + + _synchronizer = container.Resolve(); + _syncServer = new SyncServer( trieStore.TrieNodeRlpStore, _codeDb, @@ -94,7 +102,7 @@ public async Task Setup() Always.Valid, Always.Valid, _pool, - _synchronizer.SyncModeSelector, + container.Resolve(), quickConfig, Policy.FullGossip, MainnetSpecProvider.Instance, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs index f44d95d9c8c..87ab10fe7bd 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs @@ -145,7 +145,6 @@ public class ScenarioBuilder private readonly List _overwrites = new(); private readonly List _peers = new(); - private bool _needToWaitForHeaders; public ISyncPeerPool SyncPeerPool { get; set; } = null!; @@ -812,7 +811,7 @@ void Test() } TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); - MultiSyncModeSelector selector = new(SyncProgressResolver, SyncPeerPool, SyncConfig, BeaconSyncStrategy, bestPeerStrategy, LimboLogs.Instance, _needToWaitForHeaders); + MultiSyncModeSelector selector = new(SyncProgressResolver, SyncPeerPool, SyncConfig, BeaconSyncStrategy, bestPeerStrategy, LimboLogs.Instance); selector.Stop(); selector.Update(); selector.Current.Should().Be(syncMode); @@ -840,7 +839,7 @@ void Test() public ScenarioBuilder WhenConsensusRequiresToWaitForHeaders(bool needToWaitForHeaders) { - _needToWaitForHeaders = needToWaitForHeaders; + SyncConfig.NeedToWaitForHeader = needToWaitForHeaders; return this; } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index c86dfbb240b..33e43446a69 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -5,6 +5,9 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.BeaconBlockRoot; using Nethermind.Blockchain.Blocks; @@ -42,6 +45,7 @@ using BlockTree = Nethermind.Blockchain.BlockTree; using Nethermind.Synchronization.SnapSync; using Nethermind.Config; +using Nethermind.Core.Specs; using Nethermind.Specs.ChainSpecStyle; using Nethermind.Trie; @@ -356,28 +360,32 @@ private SyncTestContext CreateSyncManager(int index) TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); Pivot pivot = new(syncConfig); - BlockDownloaderFactory blockDownloaderFactory = new( - MainnetSpecProvider.Instance, - blockValidator, - sealValidator, - new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), - logManager); - Synchronizer synchronizer = new( - dbProvider, - new NodeStorage(dbProvider.StateDb), - MainnetSpecProvider.Instance, - tree, - NullReceiptStorage.Instance, - syncPeerPool, - nodeStatsManager, - syncConfig, - blockDownloaderFactory, - pivot, - Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - stateReader, - logManager); + + ContainerBuilder builder = new ContainerBuilder(); + builder + .AddSingleton(dbProvider) + .AddSingleton(new NodeStorage(dbProvider.StateDb)) + .AddSingleton(MainnetSpecProvider.Instance) + .AddSingleton(tree) + .AddSingleton(NullReceiptStorage.Instance) + .AddSingleton(syncPeerPool) + .AddSingleton(nodeStatsManager) + .AddSingleton(syncConfig) + .AddSingleton(blockValidator) + .AddSingleton(sealValidator) + .AddSingleton(pivot) + .AddSingleton(Substitute.For()) + .AddSingleton(bestPeerStrategy) + .AddSingleton(new ChainSpec()) + .AddSingleton(stateReader) + .AddSingleton(receiptStorage) + .AddSingleton(No.BeaconSync) + .AddSingleton(logManager); + dbProvider.ConfigureServiceCollection(builder); + builder.RegisterModule(new SynchronizerModule(syncConfig)); + IContainer container = builder.Build(); + + Synchronizer synchronizer = container.Resolve(); ISyncModeSelector selector = synchronizer.SyncModeSelector; SyncServer syncServer = new( diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 872499cf73f..47c721d274c 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -8,6 +8,9 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -17,6 +20,7 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; using Nethermind.Core.Timers; using Nethermind.Db; @@ -333,74 +337,48 @@ ISyncConfig GetSyncConfig() => : totalDifficultyBetterPeerStrategy; StateReader reader = new StateReader(trieStore, codeDb, LimboLogs.Instance); - FullStateFinder fullStateFinder = new FullStateFinder(BlockTree, reader); INodeStorage nodeStorage = new NodeStorage(dbProvider.StateDb); SyncPeerPool = new SyncPeerPool(BlockTree, stats, bestPeerStrategy, _logManager, 25); Pivot pivot = new(syncConfig); IInvalidChainTracker invalidChainTracker = new NoopInvalidChainTracker(); + + ContainerBuilder builder = new ContainerBuilder(); + dbProvider.ConfigureServiceCollection(builder); + + builder + .AddSingleton(dbProvider) + .AddSingleton(nodeStorage) + .AddSingleton(MainnetSpecProvider.Instance) + .AddSingleton(BlockTree) + .AddSingleton(NullReceiptStorage.Instance) + .AddSingleton(SyncPeerPool) + .AddSingleton(stats) + .AddSingleton(syncConfig) + .AddSingleton(pivot) + .AddSingleton(poSSwitcher) + .AddSingleton(mergeConfig) + .AddSingleton(invalidChainTracker) + .AddSingleton(Substitute.For()) + .AddSingleton(bestPeerStrategy) + .AddSingleton(new ChainSpec()) + .AddSingleton(No.BeaconSync) + .AddSingleton(reader) + .AddSingleton(Always.Valid) + .AddSingleton(Always.Valid) + .AddSingleton(beaconPivot) + .AddSingleton(_logManager); + + builder.RegisterModule(new SynchronizerModule(syncConfig)); + if (IsMerge(synchronizerType)) { - IBlockDownloaderFactory blockDownloaderFactory = new MergeBlockDownloaderFactory( - poSSwitcher, - beaconPivot, - MainnetSpecProvider.Instance, - Always.Valid, - Always.Valid, - syncConfig, - bestPeerStrategy, - fullStateFinder, - _logManager - ); - Synchronizer = new MergeSynchronizer( - dbProvider, - nodeStorage, - MainnetSpecProvider.Instance, - BlockTree, - NullReceiptStorage.Instance, - SyncPeerPool, - stats, - syncConfig, - blockDownloaderFactory, - pivot, - poSSwitcher, - mergeConfig, - invalidChainTracker, - Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - No.BeaconSync, - reader, - _logManager); - } - else - { - IBlockDownloaderFactory blockDownloaderFactory = new BlockDownloaderFactory( - MainnetSpecProvider.Instance, - Always.Valid, - Always.Valid, - new TotalDifficultyBetterPeerStrategy(_logManager), - _logManager); - - Synchronizer = new Synchronizer( - dbProvider, - nodeStorage, - MainnetSpecProvider.Instance, - BlockTree, - NullReceiptStorage.Instance, - SyncPeerPool, - stats, - syncConfig, - blockDownloaderFactory, - pivot, - Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - reader, - _logManager); + builder.RegisterModule(new MergeSynchronizerModule()); } + IContainer container = builder.Build(); + Synchronizer = container.Resolve(); SyncServer = new SyncServer( trieStore.TrieNodeRlpStore, codeDb, @@ -409,7 +387,7 @@ ISyncConfig GetSyncConfig() => Always.Valid, Always.Valid, SyncPeerPool, - Synchronizer.SyncModeSelector, + container.Resolve(), syncConfig, Policy.FullGossip, MainnetSpecProvider.Instance, diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs deleted file mode 100644 index 64afb7b1061..00000000000 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; -using Nethermind.Consensus; -using Nethermind.Consensus.Validators; -using Nethermind.Core.Specs; -using Nethermind.Logging; -using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; -using Nethermind.Synchronization.Reporting; - -namespace Nethermind.Synchronization.Blocks -{ - public class BlockDownloaderFactory : IBlockDownloaderFactory - { - private readonly ISpecProvider _specProvider; - private readonly IBlockValidator _blockValidator; - private readonly ISealValidator _sealValidator; - private readonly IBetterPeerStrategy _betterPeerStrategy; - private readonly ILogManager _logManager; - - public BlockDownloaderFactory( - ISpecProvider specProvider, - IBlockValidator blockValidator, - ISealValidator sealValidator, - IBetterPeerStrategy betterPeerStrategy, - ILogManager logManager) - { - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _blockValidator = blockValidator ?? throw new ArgumentNullException(nameof(blockValidator)); - _sealValidator = sealValidator ?? throw new ArgumentNullException(nameof(sealValidator)); - _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - } - - public BlockDownloader Create(ISyncFeed syncFeed, IBlockTree blockTree, IReceiptStorage receiptStorage, ISyncPeerPool peerPool, ISyncReport syncReport) - { - return new( - syncFeed, - peerPool, - blockTree, - _blockValidator, - _sealValidator, - syncReport, - receiptStorage, - _specProvider, - _betterPeerStrategy, - _logManager); - } - - public IPeerAllocationStrategyFactory CreateAllocationStrategyFactory() - { - return new BlocksSyncPeerAllocationStrategyFactory(); - } - } - - public interface IBlockDownloaderFactory - { - BlockDownloader Create(ISyncFeed syncFeed, IBlockTree blockTree, IReceiptStorage receiptStorage, ISyncPeerPool syncPeerPool, ISyncReport syncReport); - IPeerAllocationStrategyFactory CreateAllocationStrategyFactory(); - } -} diff --git a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs index e9f5c7c10d5..13a8ecbb4a6 100644 --- a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Autofac.Features.AttributeFilters; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain.Synchronization; using Nethermind.Db; using Nethermind.Synchronization.FastBlocks; @@ -24,26 +26,32 @@ public SyncDbTuner( ISyncFeed? snapSyncFeed, ISyncFeed? bodiesSyncFeed, ISyncFeed? receiptSyncFeed, - ITunableDb? stateDb, - ITunableDb? codeDb, - ITunableDb? blockDb, - ITunableDb? receiptDb + [KeyFilter(DbNames.State)] ITunableDb? stateDb, + [KeyFilter(DbNames.Code)] ITunableDb? codeDb, + [KeyFilter(DbNames.Blocks)] ITunableDb? blockDb, + [KeyFilter(DbNames.Receipts)] ITunableDb? receiptDb ) { + if (syncConfig.TuneDbMode == ITunableDb.TuneType.Default && syncConfig.BlocksDbTuneDbMode == ITunableDb.TuneType.Default) + { + // Do nothing. + return; + } + // Only these three make sense as they are write heavy // Headers is used everywhere, so slowing read might slow the whole sync. // Statesync is read heavy, Forward sync is just plain too slow to saturate IO. - if (snapSyncFeed is not null) + if (snapSyncFeed is not NoopSyncFeed) { snapSyncFeed.StateChanged += SnapStateChanged; } - if (bodiesSyncFeed is not null) + if (bodiesSyncFeed is not NoopSyncFeed) { bodiesSyncFeed.StateChanged += BodiesStateChanged; } - if (receiptSyncFeed is not null) + if (receiptSyncFeed is not NoopSyncFeed) { receiptSyncFeed.StateChanged += ReceiptsStateChanged; } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 9b972b68d67..1152f416054 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -4,6 +4,8 @@ using System; using System.Threading; using System.Threading.Tasks; +using Autofac.Features.AttributeFilters; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; using Nethermind.Consensus.Validators; @@ -50,8 +52,8 @@ public BodiesSyncFeed( ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, - IDbMeta blocksDb, - IDb metadataDb, + [KeyFilter(DbNames.Blocks)] IDbMeta blocksDb, + [KeyFilter(DbNames.Metadata)] IDb metadataDb, ILogManager logManager, long flushDbInterval = DefaultFlushDbInterval) : base(metadataDb, specProvider, logManager.GetClassLogger()) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index e621dc56795..cbfce46215c 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -6,6 +6,8 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Autofac.Features.AttributeFilters; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -57,7 +59,7 @@ public ReceiptsSyncFeed( ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, - IDb metadataDb, + [KeyFilter(DbNames.Metadata)] IDb metadataDb, ILogManager logManager) : base(metadataDb, specProvider, logManager?.GetClassLogger() ?? default) { diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index 0c4b7b35568..1abfe9643bb 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -8,6 +8,8 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Autofac.Features.AttributeFilters; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Core.Caching; @@ -74,6 +76,12 @@ public class TreeSync private long _blockNumber; private readonly SyncMode _syncMode; + public TreeSync([KeyFilter(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) + : this(SyncMode.StateNodes, codeDb, nodeStorage, blockTree, logManager) + { + + } + public TreeSync(SyncMode syncMode, IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) { _syncMode = syncMode; diff --git a/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs b/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs index 512b0a767a5..011659c5238 100644 --- a/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs @@ -15,8 +15,5 @@ public interface ISynchronizer : IDisposable void Start(); Task StopAsync(); - - ISyncProgressResolver SyncProgressResolver { get; } - ISyncModeSelector SyncModeSelector { get; } } } diff --git a/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs b/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs index 63c6e85e36c..0010bc111db 100644 --- a/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs +++ b/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using Nethermind.Blockchain.Synchronization; using Nethermind.Core.Extensions; using Nethermind.Core.Memory; using Nethermind.Logging; @@ -16,6 +17,14 @@ public class MallocTrimmer private readonly MallocHelper _mallocHelper; private readonly ILogger _logger; + public MallocTrimmer( + ISyncModeSelector syncModeSelector, + ISyncConfig syncConfig, + ILogManager logManager + ) : this(syncModeSelector, TimeSpan.FromSeconds(syncConfig.MallocTrimIntervalSec), logManager) + { + } + public MallocTrimmer( ISyncModeSelector syncModeSelector, TimeSpan interval, diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs index 64ac3f4687e..1f87736fb08 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs @@ -30,6 +30,6 @@ public interface ISyncFeed /// Return true if not finished. May not run even if return true if MultiSyncModeSelector said no, probably /// because it's waiting for other sync or something. - bool IsFinished { get; } + bool IsFinished => false; } } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index e9eff01f553..7cfc08deca8 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -8,8 +8,10 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; using Nethermind.Int256; using Nethermind.Logging; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.State.Snap; using Nethermind.Synchronization.Peers; @@ -80,8 +82,7 @@ public MultiSyncModeSelector( ISyncConfig syncConfig, IBeaconSyncStrategy beaconSyncStrategy, IBetterPeerStrategy betterPeerStrategy, - ILogManager logManager, - bool needToWaitForHeaders = false) + ILogManager logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); @@ -89,7 +90,7 @@ public MultiSyncModeSelector( _syncPeerPool = syncPeerPool ?? throw new ArgumentNullException(nameof(syncPeerPool)); _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); _syncProgressResolver = syncProgressResolver ?? throw new ArgumentNullException(nameof(syncProgressResolver)); - _needToWaitForHeaders = needToWaitForHeaders; + _needToWaitForHeaders = syncConfig.NeedToWaitForHeader; if (syncConfig.FastSyncCatchUpHeightDelta <= FastSyncLag) { diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs new file mode 100644 index 00000000000..f04f755c736 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Synchronization.ParallelSync; + +public class NoopSyncFeed : ISyncFeed +{ + public SyncFeedState CurrentState { get; } + +#pragma warning disable CS0067 + public event EventHandler? StateChanged; +#pragma warning disable + + public Task PrepareRequest(CancellationToken token = default) + { + return Task.FromResult(default); + } + + public SyncResponseHandlingResult HandleResponse(T response, PeerInfo? peer = null) + { + return SyncResponseHandlingResult.NotAssigned; + } + + public bool IsMultiFeed { get; } + public AllocationContexts Contexts { get; } + public void Activate() + { + } + + public void Finish() + { + } + + public Task FeedTask => Task.CompletedTask; + public void SyncModeSelectorOnChanged(SyncMode current) + { + } + + public bool IsFinished { get; } = true; +} diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs index 50c38f3384b..2bbdb3e50d4 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs @@ -4,6 +4,8 @@ using System; using System.Threading; using System.Threading.Tasks; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; using Nethermind.Core.Exceptions; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -25,6 +27,18 @@ public class SyncDispatcher private readonly SemaphoreSlim _concurrentProcessingSemaphore; + public SyncDispatcher( + ISyncConfig syncConfig, + ISyncFeed? syncFeed, + ISyncDownloader? downloader, + ISyncPeerPool? syncPeerPool, + IPeerAllocationStrategyFactory? peerAllocationStrategy, + ILogManager? logManager) + : this(syncConfig.MaxProcessingThreads, syncFeed, downloader, syncPeerPool, peerAllocationStrategy, logManager) + { + + } + public SyncDispatcher( int maxNumberOfProcessingThread, ISyncFeed? syncFeed, diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs index 73a325d3149..705611e0f79 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Autofac.Features.AttributeFilters; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; @@ -23,19 +24,19 @@ public class SyncProgressResolver : ISyncProgressResolver // ReSharper disable once NotAccessedField.Local private readonly ILogger _logger; - private readonly ISyncFeed? _headersSyncFeed; - private readonly ISyncFeed? _bodiesSyncFeed; - private readonly ISyncFeed? _receiptsSyncFeed; - private readonly ISyncFeed? _snapSyncFeed; + private readonly ISyncFeed _headersSyncFeed; + private readonly ISyncFeed _bodiesSyncFeed; + private readonly ISyncFeed _receiptsSyncFeed; + private readonly ISyncFeed _snapSyncFeed; public SyncProgressResolver( IBlockTree blockTree, IFullStateFinder fullStateFinder, ISyncConfig syncConfig, - ISyncFeed? headersSyncFeed, - ISyncFeed? bodiesSyncFeed, - ISyncFeed? receiptsSyncFeed, - ISyncFeed? snapSyncFeed, + [KeyFilter(nameof(HeadersSyncFeed))] ISyncFeed headersSyncFeed, + ISyncFeed bodiesSyncFeed, + ISyncFeed receiptsSyncFeed, + ISyncFeed snapSyncFeed, ILogManager logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); @@ -83,11 +84,11 @@ public long FindBestFullState() return _blockTree.FindHeader(blockHash)?.TotalDifficulty == 0 ? null : _blockTree.FindHeader(blockHash)?.TotalDifficulty; } - public bool IsFastBlocksHeadersFinished() => !IsFastBlocks() || !_syncConfig.DownloadHeadersInFastSync || _headersSyncFeed?.IsFinished == true; + public bool IsFastBlocksHeadersFinished() => !IsFastBlocks() || !_syncConfig.DownloadHeadersInFastSync || _headersSyncFeed.IsFinished; - public bool IsFastBlocksBodiesFinished() => !IsFastBlocks() || !_syncConfig.DownloadBodiesInFastSync || _bodiesSyncFeed?.IsFinished == true; + public bool IsFastBlocksBodiesFinished() => !IsFastBlocks() || !_syncConfig.DownloadBodiesInFastSync || _bodiesSyncFeed.IsFinished; - public bool IsFastBlocksReceiptsFinished() => !IsFastBlocks() || !_syncConfig.DownloadReceiptsInFastSync || _receiptsSyncFeed?.IsFinished == true; + public bool IsFastBlocksReceiptsFinished() => !IsFastBlocks() || !_syncConfig.DownloadReceiptsInFastSync || _receiptsSyncFeed.IsFinished; public bool IsSnapGetRangesFinished() => _snapSyncFeed?.IsFinished ?? true; diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index eda6709e66e..b6dba820c0c 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -8,7 +8,10 @@ using System.Linq; using System.Text; using System.Threading; +using Autofac.Features.AttributeFilters; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; +using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; @@ -58,6 +61,11 @@ public class ProgressTracker : IDisposable private readonly Pivot _pivot; + public ProgressTracker(IBlockTree blockTree, [KeyFilter(DbNames.State)] IDb db, ILogManager logManager, ISyncConfig syncConfig) + : this(blockTree, db, logManager, syncConfig.SnapSyncAccountRangePartitionCount) + { + } + public ProgressTracker(IBlockTree blockTree, IDb db, ILogManager logManager, int accountRangePartitionCount = 8) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index afb19297673..51548bfa9f6 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -6,6 +6,8 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using Autofac.Features.AttributeFilters; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.ObjectPool; using Nethermind.Core; using Nethermind.Core.Caching; @@ -33,7 +35,7 @@ public class SnapProvider : ISnapProvider // This is actually close to 97% effective. private readonly ClockKeyCache _codeExistKeyCache = new(1024 * 16); - public SnapProvider(ProgressTracker progressTracker, IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) + public SnapProvider(ProgressTracker progressTracker, [KeyFilter(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) { _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); _progressTracker = progressTracker ?? throw new ArgumentNullException(nameof(progressTracker)); diff --git a/src/Nethermind/Nethermind.Synchronization/SyncFeedComponent.cs b/src/Nethermind/Nethermind.Synchronization/SyncFeedComponent.cs new file mode 100644 index 00000000000..93b4fe0df45 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/SyncFeedComponent.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Autofac; +using Nethermind.Synchronization.Blocks; +using Nethermind.Synchronization.ParallelSync; + +namespace Nethermind.Synchronization; + +/// +/// Container to make it simpler to get a bunch of components within a same named scoped. +/// +public class SyncFeedComponent(Lazy> feed, Lazy> dispatcher, Lazy> blockDownloader, ILifetimeScope lifetimeScope) : IDisposable, IAsyncDisposable +{ + public ISyncFeed Feed => feed.Value; + public SyncDispatcher Dispatcher => dispatcher.Value; + public BlockDownloader BlockDownloader => (BlockDownloader)blockDownloader.Value; + + public void Dispose() + { + lifetimeScope.Dispose(); + } + + public async ValueTask DisposeAsync() + { + await lifetimeScope.DisposeAsync(); + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index ff69883e6f6..826406ca9de 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -4,165 +4,74 @@ using System; using System.Threading; using System.Threading.Tasks; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; +using Autofac; +using Autofac.Features.AttributeFilters; using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Core; using Nethermind.Core.Extensions; -using Nethermind.Core.Specs; -using Nethermind.Db; using Nethermind.Logging; -using Nethermind.Specs.ChainSpecStyle; -using Nethermind.State; using Nethermind.Stats; using Nethermind.Stats.Model; +using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.DbTuner; -using Nethermind.Synchronization.FastBlocks - ; +using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.FastSync; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.StateSync; -using Nethermind.Trie; namespace Nethermind.Synchronization { - public class Synchronizer : ISynchronizer + public class Synchronizer( + ISyncModeSelector syncModeSelector, + ISyncReport syncReport, + ISyncConfig syncConfig, + ILogManager logManager, + INodeStatsManager nodeStatsManager, + [KeyFilter(nameof(FullSyncFeed))] SyncFeedComponent fullSyncComponent, + [KeyFilter(nameof(FastSyncFeed))] SyncFeedComponent fastSyncComponent, + SyncFeedComponent stateSyncComponent, + SyncFeedComponent snapSyncComponent, + [KeyFilter(nameof(HeadersSyncFeed))] SyncFeedComponent fastHeaderComponent, + SyncFeedComponent oldBodiesComponent, + SyncFeedComponent oldReceiptsComponent, +#pragma warning disable CS9113 // Parameter is unread. But it need to be instantiated to function + SyncDbTuner syncDbTuner, + MallocTrimmer mallocTrimmer, +#pragma warning restore CS9113 // Parameter is unread. + IProcessExitSource exitSource) + : ISynchronizer { private const int FeedsTerminationTimeout = 5_000; - private static MallocTrimmer? s_trimmer; - private static SyncDbTuner? s_dbTuner; + private readonly ILogger _logger = logManager.GetClassLogger(); - private readonly ISpecProvider _specProvider; - private readonly IReceiptStorage _receiptStorage; - private readonly IBlockDownloaderFactory _blockDownloaderFactory; - private readonly INodeStatsManager _nodeStatsManager; - - protected readonly ILogger _logger; - protected readonly IBlockTree _blockTree; - protected readonly ISyncConfig _syncConfig; - protected readonly ISyncPeerPool _syncPeerPool; - protected readonly ILogManager _logManager; - protected readonly ISyncReport _syncReport; - protected readonly IPivot _pivot; - - protected CancellationTokenSource? _syncCancellation = new(); + private CancellationTokenSource? _syncCancellation = new(); /* sync events are used mainly for managing sync peers reputation */ public event EventHandler? SyncEvent; - private readonly IDbProvider _dbProvider; - private FastSyncFeed? _fastSyncFeed; - private StateSyncFeed? _stateSyncFeed; - private FullSyncFeed? _fullSyncFeed; - private readonly IProcessExitSource _exitSource; - protected IBetterPeerStrategy _betterPeerStrategy; - private readonly ChainSpec _chainSpec; - - public ISnapProvider SnapProvider { get; } - - private HeadersSyncFeed? _headersSyncFeed; - private HeadersSyncFeed? HeadersSyncFeed => _headersSyncFeed ??= CreateHeadersSyncFeed(); - - private ReceiptsSyncFeed? _receiptsSyncFeed; - private ReceiptsSyncFeed? ReceiptsSyncFeed => _receiptsSyncFeed ??= CreateReceiptsSyncFeed(); - - private BodiesSyncFeed? _bodiesSyncFeed; - private BodiesSyncFeed? BodiesSyncFeed => _bodiesSyncFeed ??= CreateBodiesSyncFeed(); - - private SnapSyncFeed? _snapSyncFeed; - private SnapSyncFeed? SnapSyncFeed => _snapSyncFeed ??= CreateSnapSyncFeed(); - - private ISyncProgressResolver? _syncProgressResolver; - public ISyncProgressResolver SyncProgressResolver => _syncProgressResolver ??= new SyncProgressResolver( - _blockTree, - new FullStateFinder(_blockTree, _stateReader), - _syncConfig, - HeadersSyncFeed, - BodiesSyncFeed, - ReceiptsSyncFeed, - SnapSyncFeed, - _logManager); - - protected ISyncModeSelector? _syncModeSelector; - private readonly IStateReader _stateReader; - private readonly INodeStorage _nodeStorage; - private readonly ProgressTracker _progressTracker; - - public virtual ISyncModeSelector SyncModeSelector => _syncModeSelector ??= new MultiSyncModeSelector( - SyncProgressResolver, - _syncPeerPool!, - _syncConfig, - No.BeaconSync, - _betterPeerStrategy!, - _logManager, - _chainSpec?.SealEngineType == SealEngineType.Clique); - - public Synchronizer( - IDbProvider dbProvider, - INodeStorage nodeStorage, - ISpecProvider specProvider, - IBlockTree blockTree, - IReceiptStorage receiptStorage, - ISyncPeerPool peerPool, - INodeStatsManager nodeStatsManager, - ISyncConfig syncConfig, - IBlockDownloaderFactory blockDownloaderFactory, - IPivot pivot, - IProcessExitSource processExitSource, - IBetterPeerStrategy betterPeerStrategy, - ChainSpec chainSpec, - IStateReader stateReader, - ILogManager logManager) - { - _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); - _nodeStorage = nodeStorage ?? throw new ArgumentNullException(nameof(nodeStorage)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); - _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); - _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); - _blockDownloaderFactory = blockDownloaderFactory ?? throw new ArgumentNullException(nameof(blockDownloaderFactory)); - _pivot = pivot ?? throw new ArgumentNullException(nameof(pivot)); - _syncPeerPool = peerPool ?? throw new ArgumentNullException(nameof(peerPool)); - _nodeStatsManager = nodeStatsManager ?? throw new ArgumentNullException(nameof(nodeStatsManager)); - _exitSource = processExitSource ?? throw new ArgumentNullException(nameof(processExitSource)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); - _chainSpec = chainSpec ?? throw new ArgumentNullException(nameof(chainSpec)); - _stateReader = stateReader ?? throw new ArgumentNullException(nameof(_stateReader)); - - _syncReport = new SyncReport(_syncPeerPool!, nodeStatsManager!, _syncConfig, _pivot, logManager); - - _progressTracker = new( - blockTree, - dbProvider.StateDb, - logManager, - _syncConfig.SnapSyncAccountRangePartitionCount); - SnapProvider = new SnapProvider(_progressTracker, dbProvider.CodeDb, nodeStorage, logManager); - } + public ISyncModeSelector SyncModeSelector => syncModeSelector; public virtual void Start() { - if (!_syncConfig.SynchronizationEnabled) + if (!syncConfig.SynchronizationEnabled) { return; } StartFullSyncComponents(); - if (_syncConfig.FastSync) + if (syncConfig.FastSync) { StartFastBlocksComponents(); StartFastSyncComponents(); - if (_syncConfig.SnapSync) + if (syncConfig.SnapSync) { StartSnapSyncComponents(); } @@ -170,72 +79,20 @@ public virtual void Start() StartStateSyncComponents(); } - if (_syncConfig.TuneDbMode != ITunableDb.TuneType.Default || _syncConfig.BlocksDbTuneDbMode != ITunableDb.TuneType.Default) + if (syncConfig.ExitOnSynced) { - SetupDbOptimizer(); - } - - if (_syncConfig.ExitOnSynced) - { - _exitSource.WatchForExit(SyncModeSelector, _logManager, TimeSpan.FromSeconds(_syncConfig.ExitOnSyncedWaitTimeSec)); + exitSource.WatchForExit(SyncModeSelector, logManager, TimeSpan.FromSeconds(syncConfig.ExitOnSyncedWaitTimeSec)); } WireMultiSyncModeSelector(); - s_trimmer ??= new MallocTrimmer(SyncModeSelector, TimeSpan.FromSeconds(_syncConfig.MallocTrimIntervalSec), _logManager); - SyncModeSelector.Changed += _syncReport.SyncModeSelectorOnChanged; - } - - private HeadersSyncFeed? CreateHeadersSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync) return null; - return new HeadersSyncFeed(_blockTree, _syncPeerPool, _syncConfig, _syncReport, _logManager); - } - - private BodiesSyncFeed? CreateBodiesSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || !_syncConfig.DownloadBodiesInFastSync) return null; - return new BodiesSyncFeed(_specProvider, _blockTree, _syncPeerPool, _syncConfig, _syncReport, _dbProvider.BlocksDb, _dbProvider.MetadataDb, _logManager); - } - - private ReceiptsSyncFeed? CreateReceiptsSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || !_syncConfig.DownloadBodiesInFastSync || !_syncConfig.DownloadReceiptsInFastSync) return null; - return new ReceiptsSyncFeed(_specProvider, _blockTree, _receiptStorage, _syncPeerPool, _syncConfig, _syncReport, _dbProvider.MetadataDb, _logManager); - } - - private SnapSyncFeed? CreateSnapSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.SnapSync) return null; - return new SnapSyncFeed(SnapProvider, _logManager); - } - - private void SetupDbOptimizer() - { - s_dbTuner ??= new SyncDbTuner( - _syncConfig, - SnapSyncFeed, - BodiesSyncFeed, - ReceiptsSyncFeed, - _dbProvider.StateDb as ITunableDb, - _dbProvider.CodeDb as ITunableDb, - _dbProvider.BlocksDb as ITunableDb, - _dbProvider.ReceiptsDb as ITunableDb); + SyncModeSelector.Changed += syncReport.SyncModeSelectorOnChanged; } private void StartFullSyncComponents() { - _fullSyncFeed = new FullSyncFeed(); - BlockDownloader fullSyncBlockDownloader = _blockDownloaderFactory.Create(_fullSyncFeed, _blockTree, _receiptStorage, _syncPeerPool, _syncReport); - fullSyncBlockDownloader.SyncEvent += DownloaderOnSyncEvent; - - SyncDispatcher dispatcher = CreateDispatcher( - _fullSyncFeed, - fullSyncBlockDownloader, - _blockDownloaderFactory.CreateAllocationStrategyFactory() - ); - - dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + fullSyncComponent.BlockDownloader.SyncEvent += DownloaderOnSyncEvent; + fullSyncComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -250,17 +107,8 @@ private void StartFullSyncComponents() private void StartFastSyncComponents() { - _fastSyncFeed = new FastSyncFeed(_syncConfig); - BlockDownloader downloader = _blockDownloaderFactory.Create(_fastSyncFeed, _blockTree, _receiptStorage, _syncPeerPool, _syncReport); - downloader.SyncEvent += DownloaderOnSyncEvent; - - SyncDispatcher dispatcher = CreateDispatcher( - _fastSyncFeed, - downloader, - _blockDownloaderFactory.CreateAllocationStrategyFactory() - ); - - dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + fastSyncComponent.BlockDownloader.SyncEvent += DownloaderOnSyncEvent; + fastSyncComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -275,15 +123,7 @@ private void StartFastSyncComponents() private void StartStateSyncComponents() { - TreeSync treeSync = new(SyncMode.StateNodes, _dbProvider.CodeDb, _nodeStorage, _blockTree, _logManager); - _stateSyncFeed = new StateSyncFeed(treeSync, _logManager); - SyncDispatcher stateSyncDispatcher = CreateDispatcher( - _stateSyncFeed, - new StateSyncDownloader(_logManager), - new StateSyncAllocationStrategyFactory() - ); - - Task syncDispatcherTask = stateSyncDispatcher.Start(_syncCancellation.Token).ContinueWith(t => + Task syncDispatcherTask = stateSyncComponent.Dispatcher.Start(_syncCancellation.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -296,15 +136,10 @@ private void StartStateSyncComponents() }); } + private void StartSnapSyncComponents() { - SyncDispatcher dispatcher = CreateDispatcher( - SnapSyncFeed, - new SnapSyncDownloader(_logManager), - new SnapSyncAllocationStrategyFactory() - ); - - Task _ = dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + Task _ = snapSyncComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -319,14 +154,7 @@ private void StartSnapSyncComponents() private void StartFastBlocksComponents() { - FastBlocksPeerAllocationStrategyFactory fastFactory = new(); - SyncDispatcher headersDispatcher = CreateDispatcher( - HeadersSyncFeed, - new HeadersSyncDownloader(_logManager), - fastFactory - ); - - Task headersTask = headersDispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + Task headersTask = fastHeaderComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -338,18 +166,11 @@ private void StartFastBlocksComponents() } }); - if (_syncConfig.DownloadHeadersInFastSync) + if (syncConfig.DownloadHeadersInFastSync) { - if (_syncConfig.DownloadBodiesInFastSync) + if (syncConfig.DownloadBodiesInFastSync) { - - SyncDispatcher bodiesDispatcher = CreateDispatcher( - BodiesSyncFeed!, - new BodiesSyncDownloader(_logManager), - fastFactory - ); - - Task bodiesTask = bodiesDispatcher.Start(_syncCancellation.Token).ContinueWith(t => + Task bodiesTask = oldBodiesComponent.Dispatcher.Start(_syncCancellation.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -362,15 +183,9 @@ private void StartFastBlocksComponents() }); } - if (_syncConfig.DownloadReceiptsInFastSync) + if (syncConfig.DownloadReceiptsInFastSync) { - SyncDispatcher receiptsDispatcher = CreateDispatcher( - ReceiptsSyncFeed!, - new ReceiptsSyncDispatcher(_logManager), - fastFactory - ); - - Task receiptsTask = receiptsDispatcher.Start(_syncCancellation.Token).ContinueWith(t => + Task receiptsTask = oldReceiptsComponent.Dispatcher.Start(_syncCancellation.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -385,17 +200,6 @@ private void StartFastBlocksComponents() } } - protected SyncDispatcher CreateDispatcher(ISyncFeed feed, ISyncDownloader downloader, IPeerAllocationStrategyFactory peerAllocationStrategyFactory) - { - return new( - _syncConfig.MaxProcessingThreads, - feed!, - downloader, - _syncPeerPool, - peerAllocationStrategyFactory, - _logManager); - } - private static NodeStatsEventType Convert(SyncEvent syncEvent) { return syncEvent switch @@ -410,7 +214,7 @@ private static NodeStatsEventType Convert(SyncEvent syncEvent) private void DownloaderOnSyncEvent(object? sender, SyncEventArgs e) { - _nodeStatsManager.ReportSyncEvent(e.Peer.Node, Convert(e.SyncEvent)); + nodeStatsManager.ReportSyncEvent(e.Peer.Node, Convert(e.SyncEvent)); SyncEvent?.Invoke(this, e); } @@ -421,27 +225,27 @@ public Task StopAsync() return Task.WhenAny( Task.Delay(FeedsTerminationTimeout), Task.WhenAll( - _fastSyncFeed?.FeedTask ?? Task.CompletedTask, - _stateSyncFeed?.FeedTask ?? Task.CompletedTask, - SnapSyncFeed?.FeedTask ?? Task.CompletedTask, - _fullSyncFeed?.FeedTask ?? Task.CompletedTask, - HeadersSyncFeed?.FeedTask ?? Task.CompletedTask, - BodiesSyncFeed?.FeedTask ?? Task.CompletedTask, - ReceiptsSyncFeed?.FeedTask ?? Task.CompletedTask)); + fullSyncComponent.Feed.FeedTask, + fastSyncComponent.Feed.FeedTask, + stateSyncComponent.Feed.FeedTask, + snapSyncComponent.Feed.FeedTask, + fastHeaderComponent.Feed.FeedTask, + oldBodiesComponent.Feed.FeedTask, + oldReceiptsComponent.Feed.FeedTask)); } private void WireMultiSyncModeSelector() { - WireFeedWithModeSelector(_fastSyncFeed); - WireFeedWithModeSelector(_stateSyncFeed); - WireFeedWithModeSelector(SnapSyncFeed); - WireFeedWithModeSelector(_fullSyncFeed); - WireFeedWithModeSelector(HeadersSyncFeed); - WireFeedWithModeSelector(BodiesSyncFeed); - WireFeedWithModeSelector(ReceiptsSyncFeed); + WireFeedWithModeSelector(fullSyncComponent.Feed); + WireFeedWithModeSelector(fastSyncComponent.Feed); + WireFeedWithModeSelector(stateSyncComponent.Feed); + WireFeedWithModeSelector(snapSyncComponent.Feed); + WireFeedWithModeSelector(fastHeaderComponent.Feed); + WireFeedWithModeSelector(oldBodiesComponent.Feed); + WireFeedWithModeSelector(oldReceiptsComponent.Feed); } - protected void WireFeedWithModeSelector(ISyncFeed? feed) + public void WireFeedWithModeSelector(ISyncFeed? feed) { if (feed is null) return; SyncModeSelector.Changed += ((sender, args) => @@ -454,17 +258,141 @@ protected void WireFeedWithModeSelector(ISyncFeed? feed) public void Dispose() { CancellationTokenExtensions.CancelDisposeAndClear(ref _syncCancellation); - _syncReport.Dispose(); - _fastSyncFeed?.Dispose(); - _stateSyncFeed?.Dispose(); - _stateSyncFeed = null; - SnapSyncFeed?.Dispose(); - _snapSyncFeed = null; - _fullSyncFeed?.Dispose(); - HeadersSyncFeed?.Dispose(); - BodiesSyncFeed?.Dispose(); - ReceiptsSyncFeed?.Dispose(); - _progressTracker.Dispose(); } } } + +public class SynchronizerModule(ISyncConfig syncConfig) : Module +{ + protected override void Load(ContainerBuilder builder) + { + base.Load(builder); + + builder + .AddSingleton() + + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + + // For blocks. There are two block scope, Fast and Full + .AddScoped>() + .AddScoped, BlockDownloader>() + .AddScoped, BlocksSyncPeerAllocationStrategyFactory>() + .AddScoped>() + + // For headers. There are two header scope, Fast and Beacon + .AddScoped>() + .AddScoped, HeadersSyncDownloader>() + .AddScoped, FastBlocksPeerAllocationStrategyFactory>() + .AddScoped>() + + // SyncProgress resolver need one header sync batch feed, which is the fast header one. + .Register(ctx => ctx + .ResolveNamed>(nameof(HeadersSyncFeed)) + .Feed) + .Named>(nameof(HeadersSyncFeed)); + + ConfigureSnapComponent(builder); + ConfigureReceiptSyncComponent(builder); + ConfigureBodiesSyncComponent(builder); + ConfigureStateSyncComponent(builder); + + builder + .RegisterNamedComponentInItsOwnLifetime>(nameof(HeadersSyncFeed), ConfigureFastHeader) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FastSyncFeed), ConfigureFastSync) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FullSyncFeed), ConfigureFullSync); + } + + private void ConfigureFullSync(ContainerBuilder scopeConfig) + { + scopeConfig.AddScoped, FullSyncFeed>(); + } + + private void ConfigureFastSync(ContainerBuilder scopeConfig) + { + if (syncConfig.FastSync) + { + scopeConfig.AddScoped, FastSyncFeed>(); + } + else + { + scopeConfig.AddScoped, NoopSyncFeed>(); + } + } + + private void ConfigureFastHeader(ContainerBuilder scopeConfig) + { + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync) + { + scopeConfig.AddScoped, NoopSyncFeed>(); + } + else + { + scopeConfig.AddScoped, HeadersSyncFeed>(); + } + } + + private void ConfigureSnapComponent(ContainerBuilder serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton(); + + ConfigureSingletonSyncFeed(serviceCollection); + + if (!syncConfig.FastSync || !syncConfig.SnapSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + } + } + + private void ConfigureReceiptSyncComponent(ContainerBuilder serviceCollection) + { + ConfigureSingletonSyncFeed(serviceCollection); + + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || + !syncConfig.DownloadBodiesInFastSync || + !syncConfig.DownloadReceiptsInFastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + } + } + + private void ConfigureBodiesSyncComponent(ContainerBuilder serviceCollection) + { + ConfigureSingletonSyncFeed(serviceCollection); + + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || + !syncConfig.DownloadBodiesInFastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + } + + } + + private void ConfigureStateSyncComponent(ContainerBuilder serviceCollection) + { + serviceCollection + .AddSingleton(); + + ConfigureSingletonSyncFeed(serviceCollection); + + // Disable it by setting noop + if (!syncConfig.FastSync) serviceCollection.AddSingleton, NoopSyncFeed>(); + } + + private static void ConfigureSingletonSyncFeed(ContainerBuilder serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory + { + serviceCollection + .AddSingleton, TFeed>() + .AddSingleton>() + .AddSingleton, TDownloader>() + .AddSingleton, TAllocationStrategy>() + .AddSingleton>(); + } + +} From 8f61588b6ff1f97af31d1553f7a457c7840b93f6 Mon Sep 17 00:00:00 2001 From: ak88 Date: Wed, 16 Oct 2024 22:31:23 +0200 Subject: [PATCH 04/13] Infinite loop fix in spec tests if nonce is set very high (#7615) --- src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs index 6f8afe58043..fe5fb25c019 100644 --- a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs @@ -326,12 +326,8 @@ private void InitializeTestState(BlockchainTest test, IWorldState stateProvider, stateProvider.Set(new StorageCell(accountState.Key, storageItem.Key), storageItem.Value); } - stateProvider.CreateAccount(accountState.Key, accountState.Value.Balance); + stateProvider.CreateAccount(accountState.Key, accountState.Value.Balance, accountState.Value.Nonce); stateProvider.InsertCode(accountState.Key, accountState.Value.Code, specProvider.GenesisSpec); - for (int i = 0; i < accountState.Value.Nonce; i++) - { - stateProvider.IncrementNonce(accountState.Key); - } } stateProvider.Commit(specProvider.GenesisSpec); From 04f6db39bdb62158736c47f2bf4bec32448a6218 Mon Sep 17 00:00:00 2001 From: Ruben Buniatyan Date: Thu, 17 Oct 2024 00:05:41 +0200 Subject: [PATCH 05/13] Fix Python issues in workflows (#7620) --- .github/workflows/publish-packages.yml | 2 ++ .github/workflows/update-fast-sync.yml | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 1964a1a03f1..5b152ec6ee7 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -65,6 +65,8 @@ jobs: run: | sudo apt-get update sudo apt-get install debhelper devscripts ubuntu-dev-tools -y + - name: Set up Python + uses: actions/setup-python@v5 - name: Install launchpadlib run: pip install launchpadlib --upgrade - name: Copy to other series diff --git a/.github/workflows/update-fast-sync.yml b/.github/workflows/update-fast-sync.yml index 3e23d2056f7..577cafe9fb6 100644 --- a/.github/workflows/update-fast-sync.yml +++ b/.github/workflows/update-fast-sync.yml @@ -16,10 +16,12 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 - name: Install dependencies run: | - pip3 install setuptools --break-system-packages - pip3 install emoji --break-system-packages + pip install setuptools + pip install emoji - name: Update config files run: python3 scripts/syncSettings.py ${{ secrets.ETHERSCAN_API_KEY }} - name: Create GitHub app token From 81cb4f5c2e3351b371eb50c942e600704026221b Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 17 Oct 2024 18:49:22 +0800 Subject: [PATCH 06/13] Fix opstack cant start (#7623) --- src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 76633fbf876..33615392a99 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -158,7 +158,7 @@ public Task InitSynchronization() .AddSingleton(_beaconPivot) .AddSingleton(_api.PoSSwitcher) .AddSingleton(_mergeConfig) - .AddSingleton(_invalidChainTracker); + .AddSingleton(_invalidChainTracker); builder.RegisterModule(new SynchronizerModule(_syncConfig)); builder.RegisterModule(new MergeSynchronizerModule()); From 53244735ca77361c108f560f04cea8fae900bf6b Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 17 Oct 2024 22:41:38 +0800 Subject: [PATCH 07/13] Fix assertion failure on debug (#7627) --- src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index b0cacf09bf0..5187107106c 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -700,7 +700,7 @@ private BlockCommitSet CreateCommitSet(long blockNumber) if (_logger.IsDebug) _logger.Debug($"Beginning new {nameof(BlockCommitSet)} - {blockNumber}"); // TODO: this throws on reorgs, does it not? let us recreate it in test - Debug.Assert(!CommitSetQueue.TryPeek(out BlockCommitSet lastSet) || blockNumber == lastSet.BlockNumber + 1, $"Newly begun block is not a successor of the last one."); + Debug.Assert(!CommitSetQueue.TryPeek(out BlockCommitSet lastSet) || blockNumber == lastSet.BlockNumber + 1 || lastSet.BlockNumber == 0, $"Newly begun block is not a successor of the last one."); Debug.Assert(!CommitSetQueue.TryPeek(out lastSet) || lastSet.IsSealed, "Not sealed when beginning new block"); BlockCommitSet commitSet = new(blockNumber); From e1a2c827396be713fa1689c498a8314b2473aa9a Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 18 Oct 2024 18:10:11 +0800 Subject: [PATCH 08/13] Consolidate block body decoder logic (#7616) --- .../Messages/BlockBodiesMessageSerializer.cs | 92 +----- .../BlockBodyDecoder.cs | 123 ++++++++ .../BlockDecoder.cs | 270 ++---------------- 3 files changed, 141 insertions(+), 344 deletions(-) create mode 100644 src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs index c037fe6b16e..714a3999dcc 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Messages/BlockBodiesMessageSerializer.cs @@ -5,14 +5,13 @@ using DotNetty.Buffers; using Nethermind.Core; using Nethermind.Core.Buffers; -using Nethermind.Core.ConsensusRequests; using Nethermind.Serialization.Rlp; namespace Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages { public class BlockBodiesMessageSerializer : IZeroInnerMessageSerializer { - private readonly BlockBodyDecoder _blockBodyDecoder = new(); + private readonly BlockBodyDecoder _blockBodyDecoder = BlockBodyDecoder.Instance; public void Serialize(IByteBuffer byteBuffer, BlockBodiesMessage message) { @@ -53,94 +52,5 @@ public BlockBodiesMessage Deserialize(IByteBuffer byteBuffer) return new() { Bodies = new(bodies, memoryOwner) }; } - - private class BlockBodyDecoder : IRlpValueDecoder - { - private readonly TxDecoder _txDecoder = TxDecoder.Instance; - private readonly HeaderDecoder _headerDecoder = new(); - private readonly WithdrawalDecoder _withdrawalDecoderDecoder = new(); - private readonly ConsensusRequestDecoder _requestsDecoder = ConsensusRequestDecoder.Instance; - - public int GetLength(BlockBody item, RlpBehaviors rlpBehaviors) - { - return Rlp.LengthOfSequence(GetBodyLength(item)); - } - - public int GetBodyLength(BlockBody b) => - Rlp.LengthOfSequence(GetTxLength(b.Transactions)) + - Rlp.LengthOfSequence(GetUnclesLength(b.Uncles)) - + (b.Withdrawals is not null ? Rlp.LengthOfSequence(GetWithdrawalsLength(b.Withdrawals)) : 0) - + (b.Requests is not null ? Rlp.LengthOfSequence(GetRequestsLength(b.Requests)) : 0); - - private int GetTxLength(Transaction[] transactions) => transactions.Sum(t => _txDecoder.GetLength(t, RlpBehaviors.None)); - - private int GetUnclesLength(BlockHeader[] headers) => headers.Sum(t => _headerDecoder.GetLength(t, RlpBehaviors.None)); - - private int GetWithdrawalsLength(Withdrawal[] withdrawals) => withdrawals.Sum(t => _withdrawalDecoderDecoder.GetLength(t, RlpBehaviors.None)); - - private int GetRequestsLength(ConsensusRequest[] requests) => requests.Sum(t => _requestsDecoder.GetLength(t, RlpBehaviors.None)); - - public BlockBody? Decode(ref Rlp.ValueDecoderContext ctx, RlpBehaviors rlpBehaviors = RlpBehaviors.None) - { - int sequenceLength = ctx.ReadSequenceLength(); - int startingPosition = ctx.Position; - if (sequenceLength == 0) - { - return null; - } - - // quite significant allocations (>0.5%) here based on a sample 3M blocks sync - // (just on these delegates) - Transaction[] transactions = ctx.DecodeArray(_txDecoder); - BlockHeader[] uncles = ctx.DecodeArray(_headerDecoder); - Withdrawal[]? withdrawals = null; - ConsensusRequest[]? requests = null; - if (ctx.PeekNumberOfItemsRemaining(startingPosition + sequenceLength, 1) > 0) - { - withdrawals = ctx.DecodeArray(_withdrawalDecoderDecoder); - } - - if (ctx.PeekNumberOfItemsRemaining(startingPosition + sequenceLength, 1) > 0) - { - requests = ctx.DecodeArray(_requestsDecoder); - } - - return new BlockBody(transactions, uncles, withdrawals, requests); - } - - public void Serialize(RlpStream stream, BlockBody body) - { - stream.StartSequence(GetBodyLength(body)); - stream.StartSequence(GetTxLength(body.Transactions)); - foreach (Transaction? txn in body.Transactions) - { - stream.Encode(txn); - } - - stream.StartSequence(GetUnclesLength(body.Uncles)); - foreach (BlockHeader? uncle in body.Uncles) - { - stream.Encode(uncle); - } - - if (body.Withdrawals is not null) - { - stream.StartSequence(GetWithdrawalsLength(body.Withdrawals)); - foreach (Withdrawal? withdrawal in body.Withdrawals) - { - stream.Encode(withdrawal); - } - } - - if (body.Requests is not null) - { - stream.StartSequence(GetRequestsLength(body.Requests)); - foreach (ConsensusRequest? request in body.Requests) - { - stream.Encode(request); - } - } - } - } } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs new file mode 100644 index 00000000000..6e38f63dc38 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockBodyDecoder.cs @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Linq; +using System.Runtime.CompilerServices; +using Nethermind.Core; +using Nethermind.Core.ConsensusRequests; + +namespace Nethermind.Serialization.Rlp; + +public class BlockBodyDecoder : IRlpValueDecoder +{ + private readonly TxDecoder _txDecoder = TxDecoder.Instance; + private readonly HeaderDecoder _headerDecoder = new(); + private readonly WithdrawalDecoder _withdrawalDecoderDecoder = new(); + private readonly ConsensusRequestDecoder _requestsDecoder = ConsensusRequestDecoder.Instance; + + private static BlockBodyDecoder? _instance = null; + public static BlockBodyDecoder Instance => _instance ??= new BlockBodyDecoder(); + + // Cant set to private because of `Rlp.RegisterDecoder`. + public BlockBodyDecoder() + { + } + + public int GetLength(BlockBody item, RlpBehaviors rlpBehaviors) + { + return Rlp.LengthOfSequence(GetBodyLength(item)); + } + + public int GetBodyLength(BlockBody b) + { + (int txs, int uncles, int? withdrawals, int? requests) = GetBodyComponentLength(b); + return Rlp.LengthOfSequence(txs) + + Rlp.LengthOfSequence(uncles) + + (withdrawals is not null ? Rlp.LengthOfSequence(withdrawals.Value) : 0) + + (requests is not null ? Rlp.LengthOfSequence(requests.Value) : 0); + } + + public (int Txs, int Uncles, int? Withdrawals, int? Requests) GetBodyComponentLength(BlockBody b) => + ( + GetTxLength(b.Transactions), + GetUnclesLength(b.Uncles), + (b.Withdrawals is not null ? GetWithdrawalsLength(b.Withdrawals) : null), + (b.Requests is not null ? GetRequestsLength(b.Requests) : null) + ); + + private int GetTxLength(Transaction[] transactions) => transactions.Sum(t => _txDecoder.GetLength(t, RlpBehaviors.None)); + + private int GetUnclesLength(BlockHeader[] headers) => headers.Sum(t => _headerDecoder.GetLength(t, RlpBehaviors.None)); + + private int GetWithdrawalsLength(Withdrawal[] withdrawals) => withdrawals.Sum(t => _withdrawalDecoderDecoder.GetLength(t, RlpBehaviors.None)); + + private int GetRequestsLength(ConsensusRequest[] requests) => requests.Sum(t => _requestsDecoder.GetLength(t, RlpBehaviors.None)); + + public BlockBody? Decode(ref Rlp.ValueDecoderContext ctx, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + int sequenceLength = ctx.ReadSequenceLength(); + int startingPosition = ctx.Position; + if (sequenceLength == 0) + { + return null; + } + + return DecodeUnwrapped(ref ctx, startingPosition + sequenceLength); + } + + public BlockBody? DecodeUnwrapped(ref Rlp.ValueDecoderContext ctx, int lastPosition) + { + + // quite significant allocations (>0.5%) here based on a sample 3M blocks sync + // (just on these delegates) + Transaction[] transactions = ctx.DecodeArray(_txDecoder); + BlockHeader[] uncles = ctx.DecodeArray(_headerDecoder); + Withdrawal[]? withdrawals = null; + ConsensusRequest[]? requests = null; + if (ctx.PeekNumberOfItemsRemaining(lastPosition, 1) > 0) + { + withdrawals = ctx.DecodeArray(_withdrawalDecoderDecoder); + } + + if (ctx.PeekNumberOfItemsRemaining(lastPosition, 1) > 0) + { + requests = ctx.DecodeArray(_requestsDecoder); + } + + return new BlockBody(transactions, uncles, withdrawals, requests); + } + + public void Serialize(RlpStream stream, BlockBody body) + { + stream.StartSequence(GetBodyLength(body)); + stream.StartSequence(GetTxLength(body.Transactions)); + foreach (Transaction? txn in body.Transactions) + { + stream.Encode(txn); + } + + stream.StartSequence(GetUnclesLength(body.Uncles)); + foreach (BlockHeader? uncle in body.Uncles) + { + stream.Encode(uncle); + } + + if (body.Withdrawals is not null) + { + stream.StartSequence(GetWithdrawalsLength(body.Withdrawals)); + foreach (Withdrawal? withdrawal in body.Withdrawals) + { + stream.Encode(withdrawal); + } + } + + if (body.Requests is not null) + { + stream.StartSequence(GetRequestsLength(body.Requests)); + foreach (ConsensusRequest? request in body.Requests) + { + stream.Encode(request); + } + } + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs index 0530f141fd8..89d1cfd7a83 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/BlockDecoder.cs @@ -12,9 +12,7 @@ namespace Nethermind.Serialization.Rlp public class BlockDecoder : IRlpValueDecoder, IRlpStreamDecoder { private readonly HeaderDecoder _headerDecoder = new(); - private readonly TxDecoder _txDecoder = TxDecoder.Instance; - private readonly WithdrawalDecoder _withdrawalDecoder = new(); - private readonly ConsensusRequestDecoder _consensusRequestsDecoder = ConsensusRequestDecoder.Instance; + private readonly BlockBodyDecoder _blockBodyDecoder = BlockBodyDecoder.Instance; public Block? Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { @@ -29,191 +27,27 @@ public class BlockDecoder : IRlpValueDecoder, IRlpStreamDecoder return null; } - int sequenceLength = rlpStream.ReadSequenceLength(); - int blockCheck = rlpStream.Position + sequenceLength; - - BlockHeader header = Rlp.Decode(rlpStream); - - int transactionsSequenceLength = rlpStream.ReadSequenceLength(); - int transactionsCheck = rlpStream.Position + transactionsSequenceLength; - List transactions = new(); - while (rlpStream.Position < transactionsCheck) - { - transactions.Add(Rlp.Decode(rlpStream, rlpBehaviors)); - } - - rlpStream.Check(transactionsCheck); - - int unclesSequenceLength = rlpStream.ReadSequenceLength(); - int unclesCheck = rlpStream.Position + unclesSequenceLength; - List uncleHeaders = new(); - while (rlpStream.Position < unclesCheck) - { - uncleHeaders.Add(Rlp.Decode(rlpStream, rlpBehaviors)); - } - - rlpStream.Check(unclesCheck); - - List? withdrawals = DecodeWithdrawals(rlpStream, blockCheck); - List? requests = DecodeRequests(rlpStream, blockCheck); - - if ((rlpBehaviors & RlpBehaviors.AllowExtraBytes) != RlpBehaviors.AllowExtraBytes) - { - rlpStream.Check(blockCheck); - } - - return new(header, transactions, uncleHeaders, withdrawals, requests); - } - - - private List? DecodeWithdrawals(RlpStream rlpStream, int blockCheck) - { - List? withdrawals = null; - if (rlpStream.Position != blockCheck) - { - bool lengthWasRead = true; - try - { - rlpStream.PeekNextRlpLength(); - } - catch - { - lengthWasRead = false; - } - - if (lengthWasRead) - { - int withdrawalsLength = rlpStream.ReadSequenceLength(); - int withdrawalsCheck = rlpStream.Position + withdrawalsLength; - withdrawals = new(); - - while (rlpStream.Position < withdrawalsCheck) - { - withdrawals.Add(Rlp.Decode(rlpStream)); - } - - rlpStream.Check(withdrawalsCheck); - } - } - - return withdrawals; + Span contentSpan = rlpStream.PeekNextItem(); + Rlp.ValueDecoderContext ctx = new Rlp.ValueDecoderContext(contentSpan); + Block? decoded = Decode(ref ctx, rlpBehaviors); + rlpStream.Position += contentSpan.Length; + return decoded; } - private List? DecodeRequests(RlpStream rlpStream, int blockCheck) - { - List? requests = null; - if (rlpStream.Position != blockCheck) - { - bool lengthWasRead = true; - try - { - rlpStream.PeekNextRlpLength(); - } - catch - { - lengthWasRead = false; - } - - if (lengthWasRead) - { - int requestsLength = rlpStream.ReadSequenceLength(); - int requestsCheck = rlpStream.Position + requestsLength; - requests = new(); - - while (rlpStream.Position < requestsCheck) - { - requests.Add(Rlp.Decode(rlpStream)); - } - - rlpStream.Check(requestsCheck); - } - } - - return requests; - } - - private (int Total, int Txs, int Uncles, int? Withdrawals, int? Requests) GetContentLength(Block item, RlpBehaviors rlpBehaviors) { - int contentLength = _headerDecoder.GetLength(item.Header, rlpBehaviors); - - int txLength = GetTxLength(item, rlpBehaviors); - contentLength += Rlp.LengthOfSequence(txLength); - - int unclesLength = GetUnclesLength(item, rlpBehaviors); - contentLength += Rlp.LengthOfSequence(unclesLength); - - int? withdrawalsLength = null; - if (item.Withdrawals is not null) - { - withdrawalsLength = GetWithdrawalsLength(item, rlpBehaviors); - - if (withdrawalsLength.HasValue) - contentLength += Rlp.LengthOfSequence(withdrawalsLength.Value); - } - - int? consensusRequestsLength = null; - if (item.Requests is not null) - { - consensusRequestsLength = GetConsensusRequestsLength(item, rlpBehaviors); - - if (consensusRequestsLength.HasValue) - contentLength += Rlp.LengthOfSequence(consensusRequestsLength.Value); - } - - return (contentLength, txLength, unclesLength, withdrawalsLength, consensusRequestsLength); - } - - private int GetUnclesLength(Block item, RlpBehaviors rlpBehaviors) - { - int unclesLength = 0; - for (int i = 0; i < item.Uncles.Length; i++) - { - unclesLength += _headerDecoder.GetLength(item.Uncles[i], rlpBehaviors); - } - - return unclesLength; - } - - private int GetTxLength(Block item, RlpBehaviors rlpBehaviors) - { - int txLength = 0; - for (int i = 0; i < item.Transactions.Length; i++) - { - txLength += _txDecoder.GetLength(item.Transactions[i], rlpBehaviors); - } - - return txLength; - } - - private int? GetWithdrawalsLength(Block item, RlpBehaviors rlpBehaviors) - { - if (item.Withdrawals is null) - return null; - - var withdrawalLength = 0; - - for (int i = 0, count = item.Withdrawals.Length; i < count; i++) - { - withdrawalLength += _withdrawalDecoder.GetLength(item.Withdrawals[i], rlpBehaviors); - } - - return withdrawalLength; - } - - private int? GetConsensusRequestsLength(Block item, RlpBehaviors rlpBehaviors) - { - if (item.Requests is null) - return null; + int headerLength = _headerDecoder.GetLength(item.Header, rlpBehaviors); - var consensusRequestsLength = 0; + (int txs, int uncles, int? withdrawals, int? requests) = _blockBodyDecoder.GetBodyComponentLength(item.Body); - for (int i = 0, count = item.Requests.Length; i < count; i++) - { - consensusRequestsLength += _consensusRequestsDecoder.GetLength(item.Requests[i], rlpBehaviors); - } + int contentLength = + headerLength + + Rlp.LengthOfSequence(txs) + + Rlp.LengthOfSequence(uncles) + + (withdrawals is not null ? Rlp.LengthOfSequence(withdrawals.Value) : 0) + + (requests is not null ? Rlp.LengthOfSequence(requests.Value) : 0); - return consensusRequestsLength; + return (contentLength, txs, uncles, withdrawals, requests); } public int GetLength(Block? item, RlpBehaviors rlpBehaviors) @@ -238,78 +72,8 @@ public int GetLength(Block? item, RlpBehaviors rlpBehaviors) int blockCheck = decoderContext.Position + sequenceLength; BlockHeader header = Rlp.Decode(ref decoderContext); - - int transactionsSequenceLength = decoderContext.ReadSequenceLength(); - int transactionsCheck = decoderContext.Position + transactionsSequenceLength; - List transactions = new(); - while (decoderContext.Position < transactionsCheck) - { - transactions.Add(Rlp.Decode(ref decoderContext, rlpBehaviors)); - } - - decoderContext.Check(transactionsCheck); - - int unclesSequenceLength = decoderContext.ReadSequenceLength(); - int unclesCheck = decoderContext.Position + unclesSequenceLength; - List uncleHeaders = new(); - while (decoderContext.Position < unclesCheck) - { - uncleHeaders.Add(Rlp.Decode(ref decoderContext, rlpBehaviors)); - } - - decoderContext.Check(unclesCheck); - - List? withdrawals = DecodeWithdrawals(ref decoderContext, blockCheck); - List? requests = DecodeRequests(ref decoderContext, blockCheck); - - if ((rlpBehaviors & RlpBehaviors.AllowExtraBytes) != RlpBehaviors.AllowExtraBytes) - { - decoderContext.Check(blockCheck); - } - - return new(header, transactions, uncleHeaders, withdrawals, requests); - } - - private List? DecodeWithdrawals(ref Rlp.ValueDecoderContext decoderContext, int blockCheck) - { - List? withdrawals = null; - - if (decoderContext.Position != blockCheck) - { - int withdrawalsLength = decoderContext.ReadSequenceLength(); - int withdrawalsCheck = decoderContext.Position + withdrawalsLength; - withdrawals = new(); - - while (decoderContext.Position < withdrawalsCheck) - { - withdrawals.Add(Rlp.Decode(ref decoderContext)); - } - - decoderContext.Check(withdrawalsCheck); - } - - return withdrawals; - } - - private List? DecodeRequests(ref Rlp.ValueDecoderContext decoderContext, int blockCheck) - { - List? requests = null; - - if (decoderContext.Position != blockCheck) - { - int requestsLength = decoderContext.ReadSequenceLength(); - int requestsCheck = decoderContext.Position + requestsLength; - requests = new(); - - while (decoderContext.Position < requestsCheck) - { - requests.Add(Rlp.Decode(ref decoderContext)); - } - - decoderContext.Check(requestsCheck); - } - - return requests; + BlockBody body = _blockBodyDecoder.DecodeUnwrapped(ref decoderContext, blockCheck); + return new(header, body); } public Rlp Encode(Block? item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) From addad41d1a48a09c9190bb51e137b0f9049fe23c Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Sat, 19 Oct 2024 00:22:25 +0800 Subject: [PATCH 09/13] Fix commitset assertion (#7631) --- .../Nethermind.Trie/Pruning/TrieStore.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index 5187107106c..b5c6bbcab66 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -673,6 +673,10 @@ public void WaitForPruning() private ConcurrentQueue CommitSetQueue => (_commitSetQueue ?? CreateQueueAtomic(ref _commitSetQueue)); +#if DEBUG + private BlockCommitSet? _lastCommitSet = null; +#endif + private long _memoryUsedByDirtyCache; private int _committedNodesCount; @@ -700,11 +704,18 @@ private BlockCommitSet CreateCommitSet(long blockNumber) if (_logger.IsDebug) _logger.Debug($"Beginning new {nameof(BlockCommitSet)} - {blockNumber}"); // TODO: this throws on reorgs, does it not? let us recreate it in test - Debug.Assert(!CommitSetQueue.TryPeek(out BlockCommitSet lastSet) || blockNumber == lastSet.BlockNumber + 1 || lastSet.BlockNumber == 0, $"Newly begun block is not a successor of the last one."); - Debug.Assert(!CommitSetQueue.TryPeek(out lastSet) || lastSet.IsSealed, "Not sealed when beginning new block"); +#if DEBUG + Debug.Assert(_lastCommitSet == null || blockNumber == _lastCommitSet.BlockNumber + 1 || _lastCommitSet.BlockNumber == 0, $"Newly begun block is not a successor of the last one."); + Debug.Assert(_lastCommitSet == null || _lastCommitSet.IsSealed, "Not sealed when beginning new block"); +#endif BlockCommitSet commitSet = new(blockNumber); CommitSetQueue.Enqueue(commitSet); + +#if DEBUG + _lastCommitSet = commitSet; +#endif + LatestCommittedBlockNumber = Math.Max(blockNumber, LatestCommittedBlockNumber); // Why are we announcing **before** committing next block?? // Should it be after commit? From 07def8d8291bdf59370e1f3544ab0c0b049b3ae8 Mon Sep 17 00:00:00 2001 From: yevh Date: Sat, 19 Oct 2024 09:15:09 +0200 Subject: [PATCH 10/13] Add Trivy scanner workflow (#7632) Co-authored-by: Ruben Buniatyan --- .github/workflows/trivy.yml | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/trivy.yml diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml new file mode 100644 index 00000000000..b5c7b5616f1 --- /dev/null +++ b/.github/workflows/trivy.yml @@ -0,0 +1,46 @@ +name: Trivy scanner + +on: + pull_request: + branches: [master] + push: + branches: [master] + schedule: + - cron: '29 19 * * 4' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: Build + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + actions: read + env: + IMAGE_TAG: nethermind:${{ github.sha }} + steps: + - name: Check out repository + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 #v4.2.1 + + - name: Build Docker image + run: docker build -t $IMAGE_TAG . + + - name: Scan + uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 #v0.28.0 + with: + image-ref: ${{ env.IMAGE_TAG }} + format: template + template: '@/contrib/sarif.tpl' + output: trivy-results.sarif + severity: CRITICAL,HIGH + env: + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db + + - name: Upload scan results + uses: github/codeql-action/upload-sarif@cf5b0a9041d3c1d336516f1944c96d96598193cc #v2.19.1 + with: + sarif_file: trivy-results.sarif From 70a47807e92dfe1156af993487e013c148271ffd Mon Sep 17 00:00:00 2001 From: yerke26 <160018032+yerke26@users.noreply.github.com> Date: Sun, 20 Oct 2024 19:32:31 +0300 Subject: [PATCH 11/13] Fix BeaconBlockRootHandler (#7604) --- .../Ethereum.Test.Base/BlockchainTestBase.cs | 2 +- .../AuraBlockProcessorTests.cs | 2 +- .../AuRaContractGasLimitOverrideTests.cs | 2 +- .../Transactions/TxCertifierFilterTests.cs | 2 +- .../Transactions/TxPermissionFilterTest.cs | 2 +- .../BeaconBlockRootHandlerTests.cs | 156 ++++++++++++++++++ .../BlockProcessorTests.cs | 4 +- .../Producers/DevBlockproducerTests.cs | 2 +- .../Nethermind.Blockchain.Test/ReorgTests.cs | 2 +- .../BeaconBlockRoot/BeaconBlockRootHandler.cs | 6 +- .../IBeaconBlockRootHandler.cs | 1 + .../CliqueBlockProducerTests.cs | 4 +- .../InitializeBlockchainAuRa.cs | 2 +- .../StartBlockProducerAuRa.cs | 2 +- .../CliquePlugin.cs | 2 +- .../NethDevPlugin.cs | 2 +- .../Processing/ReadOnlyChainProcessingEnv.cs | 2 +- .../Producers/BlockProducerEnvFactory.cs | 2 +- .../Blockchain/TestBlockchain.cs | 5 +- .../SimulateReadOnlyBlocksProcessingEnv.cs | 2 +- .../Steps/InitializeBlockchain.cs | 2 +- .../EthModuleBenchmarks.cs | 2 +- .../Modules/Trace/ParityStyleTracerTests.cs | 2 +- .../AuRaMergeEngineModuleTests.cs | 2 +- .../AuRaMergeBlockProducerEnvFactory.cs | 2 +- .../InitializeBlockchainAuRaMerge.cs | 2 +- .../EngineModuleTests.HelperFunctions.cs | 2 +- .../EngineModuleTests.Setup.cs | 2 +- .../InitializeBlockchainOptimism.cs | 2 +- .../OptimismBlockProducerEnvFactory.cs | 2 +- .../ReadOnlyChainProcessingEnv.cs | 2 +- .../SyncThreadTests.cs | 4 +- 32 files changed, 193 insertions(+), 37 deletions(-) create mode 100644 src/Nethermind/Nethermind.Blockchain.Test/BeaconBlockRootHandlerTests.cs diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs index fe5fb25c019..66df7e618ee 100644 --- a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs @@ -171,7 +171,7 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? stateProvider, receiptStorage, transactionProcessor, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), new BlockhashStore(specProvider, stateProvider), _logManager); diff --git a/src/Nethermind/Nethermind.AuRa.Test/AuraBlockProcessorTests.cs b/src/Nethermind/Nethermind.AuRa.Test/AuraBlockProcessorTests.cs index 5ca0f63f74d..2da9aabbc1c 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/AuraBlockProcessorTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/AuraBlockProcessorTests.cs @@ -158,7 +158,7 @@ void Process(AuRaBlockProcessor auRaBlockProcessor, int blockNumber, Hash256 sta new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider), stateProvider, NullReceiptStorage.Instance, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), LimboLogs.Instance, Substitute.For(), new WithdrawalProcessor(stateProvider, LimboLogs.Instance), diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs index bcbe3b8be48..6f2f7b13d6d 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs @@ -97,7 +97,7 @@ protected override BlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - new BeaconBlockRootHandler(TxProcessor), + new BeaconBlockRootHandler(TxProcessor, State), LimboLogs.Instance, BlockTree, NullWithdrawalProcessor.Instance, diff --git a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs index c754ce285f8..cda399944da 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs @@ -154,7 +154,7 @@ protected override BlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - new BeaconBlockRootHandler(TxProcessor), + new BeaconBlockRootHandler(TxProcessor, State), LimboLogs.Instance, BlockTree, NullWithdrawalProcessor.Instance, diff --git a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs index de6de6eb464..da04745095f 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs @@ -296,7 +296,7 @@ protected override BlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - new BeaconBlockRootHandler(TxProcessor), + new BeaconBlockRootHandler(TxProcessor, State), LimboLogs.Instance, BlockTree, NullWithdrawalProcessor.Instance, diff --git a/src/Nethermind/Nethermind.Blockchain.Test/BeaconBlockRootHandlerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/BeaconBlockRootHandlerTests.cs new file mode 100644 index 00000000000..8f3e5461c7d --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain.Test/BeaconBlockRootHandlerTests.cs @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain.BeaconBlockRoot; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Eip2930; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; +using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Int256; +using Nethermind.Specs.Forks; +using Nethermind.State; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Blockchain.Test; + +public class BeaconBlockRootHandlerTests +{ + private BeaconBlockRootHandler _beaconBlockRootHandler; + private ITransactionProcessor _transactionProcessor; + private IWorldState _worldState; + + [SetUp] + public void Setup() + { + _worldState = Substitute.For(); + _transactionProcessor = Substitute.For(); + _beaconBlockRootHandler = new BeaconBlockRootHandler(_transactionProcessor, _worldState); + } + + [Test] + public void Test_BeaconRootsAccessList_IsBeaconBlockRootAvailableFalse() + { + BlockHeader header = Build.A.BlockHeader.WithNumber(1).WithParentBeaconBlockRoot(Hash256.Zero).TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + _worldState.AccountExists(Arg.Any
()).Returns(true); + + (Address? toAddress, AccessList? accessList) result = _beaconBlockRootHandler + .BeaconRootsAccessList(block, Shanghai.Instance); + + Assert.That(result.accessList, Is.Null); + Assert.That(result.toAddress, Is.Null); + } + + [Test] + public void Test_BeaconRootsAccessList_HeaderIsGenesis() + { + BlockHeader header = Build.A.BlockHeader.WithParentBeaconBlockRoot(Hash256.Zero).TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + _worldState.AccountExists(Arg.Any
()).Returns(true); + + (Address? toAddress, AccessList? accessList) result = _beaconBlockRootHandler + .BeaconRootsAccessList(block, Cancun.Instance); + + Assert.That(result.accessList, Is.Null); + Assert.That(result.toAddress, Is.Null); + } + + [Test] + public void Test_BeaconRootsAccessList_ParentBeaconBlockRootIsNull() + { + BlockHeader header = Build.A.BlockHeader.WithNumber(1).TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + _worldState.AccountExists(Arg.Any
()).Returns(true); + + (Address? toAddress, AccessList? accessList) result = _beaconBlockRootHandler + .BeaconRootsAccessList(block, Cancun.Instance); + + Assert.That(result.accessList, Is.Null); + Assert.That(result.toAddress, Is.Null); + } + + [Test] + public void Test_BeaconRootsAccessList_canInsertBeaconRootIsTrue_AccountNotExist() + { + BlockHeader header = Build.A.BlockHeader.WithNumber(1).WithParentBeaconBlockRoot(Hash256.Zero).TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + _worldState.AccountExists(Arg.Any
()).Returns(false); + + (Address? toAddress, AccessList? accessList) result = _beaconBlockRootHandler + .BeaconRootsAccessList(block, Cancun.Instance); + + Assert.That(result.accessList, Is.Null); + Assert.That(result.toAddress, Is.Null); + } + + [Test] + public void Test_BeaconRootsAccessList_canInsertBeaconRootIsTrue_AccountExists() + { + BlockHeader header = Build.A.BlockHeader.WithNumber(1).WithParentBeaconBlockRoot(Hash256.Zero).TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + _worldState.AccountExists(Arg.Any
()).Returns(true); + + (Address? toAddress, AccessList? accessList) result = _beaconBlockRootHandler + .BeaconRootsAccessList(block, Cancun.Instance); + + Assert.That(result.accessList, Is.Not.Null); + Assert.That(result.accessList.Count.AddressesCount, Is.EqualTo(1)); + Assert.That(result.accessList.Count.StorageKeysCount, Is.EqualTo(1)); + } + + [Test] + public void Test_BeaconRootsAccessList_canInsertBeaconRootIsTrue_AccountExists_IncludeStorageCellsIsFalse() + { + BlockHeader header = Build.A.BlockHeader.WithNumber(1).WithParentBeaconBlockRoot(Hash256.Zero).TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + _worldState.AccountExists(Arg.Any
()).Returns(true); + + (Address? toAddress, AccessList? accessList) result = _beaconBlockRootHandler + .BeaconRootsAccessList(block, Cancun.Instance, false); + + Assert.That(result.accessList, Is.Not.Null); + Assert.That(result.accessList.Count.AddressesCount, Is.EqualTo(1)); + Assert.That(result.accessList.Count.StorageKeysCount, Is.EqualTo(0)); + } + + [Test] + public void Test_StoreBeaconRoot_AccessListIsNull() + { + BlockHeader header = Build.A.BlockHeader.TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + + _beaconBlockRootHandler.StoreBeaconRoot(block, Cancun.Instance); + + _transactionProcessor.DidNotReceive().Execute(Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Test] + public void Test_StoreBeaconRoot_AccessListNotNull() + { + BlockHeader header = Build.A.BlockHeader.WithNumber(1).WithParentBeaconBlockRoot(Hash256.Zero).TestObject; + Block block = Build.A.Block.WithHeader(header).TestObject; + _worldState.AccountExists(Arg.Any
()).Returns(true); + + _beaconBlockRootHandler.StoreBeaconRoot(block, Cancun.Instance); + + Transaction transaction = new() + { + Value = UInt256.Zero, + Data = header.ParentBeaconBlockRoot!.Bytes.ToArray(), + To = Eip4788Constants.BeaconRootsAddress, + SenderAddress = Address.SystemUser, + GasLimit = 30_000_000L, + GasPrice = UInt256.Zero, + AccessList = new AccessList.Builder().AddAddress(Eip4788Constants.BeaconRootsAddress).Build() + }; + + transaction.Hash = transaction.CalculateHash(); + _transactionProcessor.Received().Execute(Arg.Is(t => + t.Hash == transaction.Hash), header, NullTxTracer.Instance); + } +} diff --git a/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs index 60c7c37b66e..a9c11d23a2d 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs @@ -50,7 +50,7 @@ public void Prepared_block_contains_author_field() stateProvider, NullReceiptStorage.Instance, transactionProcessor, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), Substitute.For(), LimboLogs.Instance); @@ -81,7 +81,7 @@ public void Recovers_state_on_cancel() stateProvider, NullReceiptStorage.Instance, transactionProcessor, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), Substitute.For(), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs index 096dcd6db9a..1453c55dadd 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs @@ -76,7 +76,7 @@ public void Test() stateProvider, NullReceiptStorage.Instance, txProcessor, - new BeaconBlockRootHandler(txProcessor), + new BeaconBlockRootHandler(txProcessor, stateProvider), new BlockhashStore(specProvider, stateProvider), LimboLogs.Instance); BlockchainProcessor blockchainProcessor = new( diff --git a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs index 88274cfd17b..0f87b9ea846 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs @@ -78,7 +78,7 @@ public void Setup() stateProvider, NullReceiptStorage.Instance, transactionProcessor, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), new BlockhashStore(MainnetSpecProvider.Instance, stateProvider), LimboLogs.Instance); _blockchainProcessor = new BlockchainProcessor( diff --git a/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/BeaconBlockRootHandler.cs b/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/BeaconBlockRootHandler.cs index c464091ba49..a9a519e4492 100644 --- a/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/BeaconBlockRootHandler.cs +++ b/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/BeaconBlockRootHandler.cs @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; using Nethermind.Core; using Nethermind.Core.Eip2930; using Nethermind.Core.Specs; @@ -9,9 +8,10 @@ using Nethermind.Evm.Tracing; using Nethermind.Evm.TransactionProcessing; using Nethermind.Int256; +using Nethermind.State; namespace Nethermind.Blockchain.BeaconBlockRoot; -public class BeaconBlockRootHandler(ITransactionProcessor processor) : IBeaconBlockRootHandler +public class BeaconBlockRootHandler(ITransactionProcessor processor, IWorldState stateProvider) : IBeaconBlockRootHandler { private const long GasLimit = 30_000_000L; @@ -26,7 +26,7 @@ public class BeaconBlockRootHandler(ITransactionProcessor processor) : IBeaconBl spec.Eip4788ContractAddress ?? Eip4788Constants.BeaconRootsAddress : null; - if (eip4788ContractAddress is null) + if (eip4788ContractAddress is null || !stateProvider.AccountExists(eip4788ContractAddress)) { return (null, null); } diff --git a/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/IBeaconBlockRootHandler.cs b/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/IBeaconBlockRootHandler.cs index d6e3ec0f5ed..05aea2133ea 100644 --- a/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/IBeaconBlockRootHandler.cs +++ b/src/Nethermind/Nethermind.Blockchain/BeaconBlockRoot/IBeaconBlockRootHandler.cs @@ -4,6 +4,7 @@ using Nethermind.Core; using Nethermind.Core.Eip2930; using Nethermind.Core.Specs; +using Nethermind.State; namespace Nethermind.Blockchain.BeaconBlockRoot; public interface IBeaconBlockRootHandler diff --git a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs index 83e2f3b5372..1a56439105b 100644 --- a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs +++ b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs @@ -140,7 +140,7 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f stateProvider, NullReceiptStorage.Instance, transactionProcessor, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), new BlockhashStore(goerliSpecProvider, stateProvider), nodeLogManager); @@ -161,7 +161,7 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f minerStateProvider, NullReceiptStorage.Instance, minerTransactionProcessor, - new BeaconBlockRootHandler(minerTransactionProcessor), + new BeaconBlockRootHandler(minerTransactionProcessor, minerStateProvider), new BlockhashStore(goerliSpecProvider, minerStateProvider), nodeLogManager); diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs index d2a3952e8f0..cf7ac81bbc7 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs @@ -108,7 +108,7 @@ protected virtual AuRaBlockProcessor NewAuraBlockProcessor(ITxFilter txFilter, B new BlockProcessor.BlockValidationTransactionsExecutor(_api.TransactionProcessor, worldState), worldState, _api.ReceiptStorage!, - new BeaconBlockRootHandler(_api.TransactionProcessor!), + new BeaconBlockRootHandler(_api.TransactionProcessor!, worldState), _api.LogManager, _api.BlockTree!, NullWithdrawalProcessor.Instance, diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs index 5fa09b539d8..2b3d2d5578e 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs @@ -155,7 +155,7 @@ private BlockProcessor CreateBlockProcessor(IReadOnlyTxProcessingScope changeabl _api.BlockProducerEnvFactory.TransactionsExecutorFactory.Create(changeableTxProcessingEnv), changeableTxProcessingEnv.WorldState, _api.ReceiptStorage, - new BeaconBlockRootHandler(changeableTxProcessingEnv.TransactionProcessor), + new BeaconBlockRootHandler(changeableTxProcessingEnv.TransactionProcessor, changeableTxProcessingEnv.WorldState), _api.LogManager, _api.BlockTree, NullWithdrawalProcessor.Instance, diff --git a/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs b/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs index 9167e80f900..153a2941487 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs @@ -113,7 +113,7 @@ public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) scope.WorldState, NullReceiptStorage.Instance, getFromApi.TransactionProcessor, - new BeaconBlockRootHandler(scope.TransactionProcessor), + new BeaconBlockRootHandler(scope.TransactionProcessor, scope.WorldState), new BlockhashStore(getFromApi.SpecProvider, scope.WorldState), getFromApi.LogManager, new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(scope.WorldState, getFromApi.LogManager))); diff --git a/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs b/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs index 215ae44e976..7c1be4c3c90 100644 --- a/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs +++ b/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs @@ -83,7 +83,7 @@ public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) scope.WorldState, NullReceiptStorage.Instance, scope.TransactionProcessor, - new BeaconBlockRootHandler(scope.TransactionProcessor), + new BeaconBlockRootHandler(scope.TransactionProcessor, scope.WorldState), new BlockhashStore(getFromApi.SpecProvider, scope.WorldState), getFromApi.LogManager); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs index 0b278165205..97954e1ed5c 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs @@ -69,7 +69,7 @@ IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor scope.WorldState, receiptStorage, scope.TransactionProcessor, - new BeaconBlockRootHandler(scope.TransactionProcessor), + new BeaconBlockRootHandler(scope.TransactionProcessor, scope.WorldState), new BlockhashStore(specProvider, scope.WorldState), logManager); } diff --git a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs index f896766bfbd..85e5697a8be 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs @@ -155,7 +155,7 @@ protected virtual BlockProcessor CreateBlockProcessor( readOnlyTxProcessingEnv.WorldState, receiptStorage, readOnlyTxProcessingEnv.TransactionProcessor, - new BeaconBlockRootHandler(readOnlyTxProcessingEnv.TransactionProcessor), + new BeaconBlockRootHandler(readOnlyTxProcessingEnv.TransactionProcessor, readOnlyTxProcessingEnv.WorldState), new BlockhashStore(_specProvider, readOnlyTxProcessingEnv.WorldState), logManager, new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(readOnlyTxProcessingEnv.WorldState, logManager)), diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index 2af6892ee13..fb5118b4861 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -200,7 +200,7 @@ protected virtual async Task Build(ISpecProvider? specProvider = HeaderValidator = new HeaderValidator(BlockTree, Always.Valid, SpecProvider, LogManager); _canonicalityMonitor ??= new ReceiptCanonicalityMonitor(ReceiptStorage, LogManager); - BeaconBlockRootHandler = new BeaconBlockRootHandler(TxProcessor); + BeaconBlockRootHandler = new BeaconBlockRootHandler(TxProcessor, State); BlockValidator = new BlockValidator( new TxValidator(SpecProvider.ChainId), @@ -216,7 +216,6 @@ protected virtual async Task Build(ISpecProvider? specProvider = BloomStorage bloomStorage = new(new BloomConfig(), new MemDb(), new InMemoryDictionaryFileStoreFactory()); ReceiptsRecovery receiptsRecovery = new(new EthereumEcdsa(SpecProvider.ChainId), SpecProvider); LogFinder = new LogFinder(BlockTree, ReceiptStorage, ReceiptStorage, bloomStorage, LimboLogs.Instance, receiptsRecovery); - BeaconBlockRootHandler = new BeaconBlockRootHandler(TxProcessor); BlockProcessor = CreateBlockProcessor(); BlockchainProcessor chainProcessor = new(BlockTree, BlockProcessor, BlockPreprocessorStep, StateReader, LogManager, Consensus.Processing.BlockchainProcessor.Options.Default); @@ -390,7 +389,7 @@ protected virtual IBlockProcessor CreateBlockProcessor() => State, ReceiptStorage, TxProcessor, - new BeaconBlockRootHandler(TxProcessor), + new BeaconBlockRootHandler(TxProcessor, State), new BlockhashStore(SpecProvider, State), LogManager, preWarmer: CreateBlockCachePreWarmer(), diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs index 5196c1850ac..c86b147817e 100644 --- a/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs @@ -115,7 +115,7 @@ public IBlockProcessor GetProcessor(bool validate, UInt256? blobBaseFeeOverride) StateProvider, NullReceiptStorage.Instance, _transactionProcessor, - new BeaconBlockRootHandler(_transactionProcessor), + new BeaconBlockRootHandler(_transactionProcessor, StateProvider), new BlockhashStore(SpecProvider, StateProvider), _logManager); } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs index 92aea789e83..630809f8b53 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs @@ -238,7 +238,7 @@ protected virtual BlockProcessor CreateBlockProcessor(BlockCachePreWarmer? preWa worldState, _api.ReceiptStorage, _api.TransactionProcessor, - new BeaconBlockRootHandler(_api.TransactionProcessor), + new BeaconBlockRootHandler(_api.TransactionProcessor, worldState), new BlockhashStore(_api.SpecProvider!, worldState), _api.LogManager, preWarmer: preWarmer diff --git a/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs b/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs index 885496c6812..02a8abacccd 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs @@ -104,7 +104,7 @@ TransactionProcessor transactionProcessor stateProvider, NullReceiptStorage.Instance, transactionProcessor, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), new BlockhashStore(specProvider, stateProvider), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs index f9b10675107..1c4023f547f 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs @@ -73,7 +73,7 @@ public void Setup() stateProvider, NullReceiptStorage.Instance, transactionProcessor, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, stateProvider), new BlockhashStore(specProvider, stateProvider), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs b/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs index b2d1e4c26ab..663c941b485 100644 --- a/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs +++ b/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs @@ -153,7 +153,7 @@ protected override IBlockProcessor CreateBlockProcessor() State, ReceiptStorage, TxProcessor, - new BeaconBlockRootHandler(TxProcessor), + new BeaconBlockRootHandler(TxProcessor, State), new BlockhashStore(SpecProvider, State), LogManager, WithdrawalProcessor, diff --git a/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs index 741930723bf..faf77f16d32 100644 --- a/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs @@ -75,7 +75,7 @@ protected override BlockProcessor CreateBlockProcessor( TransactionsExecutorFactory.Create(readOnlyTxProcessingEnv), readOnlyTxProcessingEnv.WorldState, receiptStorage, - new BeaconBlockRootHandler(readOnlyTxProcessingEnv.TransactionProcessor), + new BeaconBlockRootHandler(readOnlyTxProcessingEnv.TransactionProcessor, readOnlyTxProcessingEnv.WorldState), logManager, _blockTree, new Consensus.Withdrawals.BlockProductionWithdrawalProcessor( diff --git a/src/Nethermind/Nethermind.Merge.AuRa/InitializationSteps/InitializeBlockchainAuRaMerge.cs b/src/Nethermind/Nethermind.Merge.AuRa/InitializationSteps/InitializeBlockchainAuRaMerge.cs index d37d4fb9b1a..6fec278699c 100644 --- a/src/Nethermind/Nethermind.Merge.AuRa/InitializationSteps/InitializeBlockchainAuRaMerge.cs +++ b/src/Nethermind/Nethermind.Merge.AuRa/InitializationSteps/InitializeBlockchainAuRaMerge.cs @@ -43,7 +43,7 @@ protected override AuRaBlockProcessor NewAuraBlockProcessor(ITxFilter txFilter, new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, worldState), worldState, _api.ReceiptStorage!, - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, worldState), _api.LogManager, _api.BlockTree!, new AuraWithdrawalProcessor(withdrawalContractFactory.Create(transactionProcessor), _api.LogManager), diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs index 2de5457185a..abfce411741 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs @@ -153,7 +153,7 @@ private static ExecutionPayloadV4 CreateBlockRequestV4(MergeTestBlockchain chain ExecutionPayloadV4 blockRequestV4 = CreateBlockRequestInternal(parent, miner, withdrawals, blobGasUsed, excessBlobGas, transactions: transactions, parentBeaconBlockRoot: parentBeaconBlockRoot, requests: requests); blockRequestV4.TryGetBlock(out Block? block); - var beaconBlockRootHandler = new BeaconBlockRootHandler(chain.TxProcessor); + var beaconBlockRootHandler = new BeaconBlockRootHandler(chain.TxProcessor, chain.WorldStateManager.GlobalWorldState); beaconBlockRootHandler.StoreBeaconRoot(block!, chain.SpecProvider.GetSpec(block!.Header)); Snapshot before = chain.State.TakeSnapshot(); var blockHashStore = new BlockhashStore(chain.SpecProvider, chain.State); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs index a95d1a9479b..cffbfafb516 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs @@ -255,7 +255,7 @@ protected override IBlockProcessor CreateBlockProcessor() State, ReceiptStorage, TxProcessor, - new BeaconBlockRootHandler(TxProcessor), + new BeaconBlockRootHandler(TxProcessor, State), new BlockhashStore(SpecProvider, State), LogManager, WithdrawalProcessor, diff --git a/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs b/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs index 5f53d0f31eb..f7ce6c9b833 100644 --- a/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs +++ b/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs @@ -100,7 +100,7 @@ protected override BlockProcessor CreateBlockProcessor(BlockCachePreWarmer? preW api.ReceiptStorage, transactionProcessor, new BlockhashStore(api.SpecProvider, api.WorldState), - new BeaconBlockRootHandler(transactionProcessor), + new BeaconBlockRootHandler(transactionProcessor, api.WorldState), api.LogManager, api.SpecHelper, contractRewriter, diff --git a/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs index 88ee38dd5e3..3056aa4fbad 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs @@ -89,7 +89,7 @@ protected override BlockProcessor CreateBlockProcessor( receiptStorage, readOnlyTxProcessingEnv.TransactionProcessor, new BlockhashStore(specProvider, readOnlyTxProcessingEnv.WorldState), - new BeaconBlockRootHandler(readOnlyTxProcessingEnv.TransactionProcessor), + new BeaconBlockRootHandler(readOnlyTxProcessingEnv.TransactionProcessor, readOnlyTxProcessingEnv.WorldState), logManager, _specHelper, new Create2DeployerContractRewriter(_specHelper, _specProvider, _blockTree), diff --git a/src/Nethermind/Nethermind.Optimism/ReadOnlyChainProcessingEnv.cs b/src/Nethermind/Nethermind.Optimism/ReadOnlyChainProcessingEnv.cs index 3e1e3cbe011..05e6b898ffe 100644 --- a/src/Nethermind/Nethermind.Optimism/ReadOnlyChainProcessingEnv.cs +++ b/src/Nethermind/Nethermind.Optimism/ReadOnlyChainProcessingEnv.cs @@ -65,7 +65,7 @@ IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor receiptStorage, scope.TransactionProcessor, new BlockhashStore(specProvider, scope.WorldState), - new BeaconBlockRootHandler(scope.TransactionProcessor), + new BeaconBlockRootHandler(scope.TransactionProcessor, scope.WorldState), logManager, opSpecHelper, contractRewriter, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 33e43446a69..ada2c15efb3 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -307,7 +307,7 @@ private SyncTestContext CreateSyncManager(int index) stateProvider, receiptStorage, txProcessor, - new BeaconBlockRootHandler(txProcessor), + new BeaconBlockRootHandler(txProcessor, stateProvider), new BlockhashStore(specProvider, stateProvider), logManager); @@ -331,7 +331,7 @@ private SyncTestContext CreateSyncManager(int index) devState, receiptStorage, devTxProcessor, - new BeaconBlockRootHandler(devTxProcessor), + new BeaconBlockRootHandler(devTxProcessor, devState), new BlockhashStore(specProvider, devState), logManager); From a1ca0fb34a83b40d5013f55ae468723885e72587 Mon Sep 17 00:00:00 2001 From: Jorge Mederos <46798594+jmederosalvarado@users.noreply.github.com> Date: Mon, 21 Oct 2024 07:57:23 +0200 Subject: [PATCH 12/13] Add Linea L2 Configs (#7555) --- scripts/syncSettings.py | 10 + src/Nethermind/Chains/linea-mainnet.json | 923 +++++++++++++++++ src/Nethermind/Chains/linea-sepolia.json | 942 ++++++++++++++++++ .../configs/linea-mainnet.cfg | 27 + .../configs/linea-mainnet_archive.cfg | 24 + .../configs/linea-sepolia.cfg | 27 + .../configs/linea-sepolia_archive.cfg | 24 + src/bench_precompiles | 2 +- 8 files changed, 1978 insertions(+), 1 deletion(-) create mode 100644 src/Nethermind/Chains/linea-mainnet.json create mode 100644 src/Nethermind/Chains/linea-sepolia.json create mode 100644 src/Nethermind/Nethermind.Runner/configs/linea-mainnet.cfg create mode 100644 src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.cfg create mode 100644 src/Nethermind/Nethermind.Runner/configs/linea-sepolia.cfg create mode 100644 src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.cfg diff --git a/scripts/syncSettings.py b/scripts/syncSettings.py index 6042a327a54..d137de19890 100644 --- a/scripts/syncSettings.py +++ b/scripts/syncSettings.py @@ -83,6 +83,16 @@ "url": "https://sepolia.optimism.io", "blockReduced": 8192, "multiplierRequirement": 10000 + }, + "linea-mainnet": { + "url": "https://rpc.linea.build", + "blockReduced": 8192, + "multiplierRequirement": 10000 + }, + "linea-sepolia": { + "url": "https://rpc.sepolia.linea.build", + "blockReduced": 8192, + "multiplierRequirement": 10000 } } diff --git a/src/Nethermind/Chains/linea-mainnet.json b/src/Nethermind/Chains/linea-mainnet.json new file mode 100644 index 00000000000..404bd586fc5 --- /dev/null +++ b/src/Nethermind/Chains/linea-mainnet.json @@ -0,0 +1,923 @@ +{ + "name": "Linea Mainnet", + "dataDir": "linea-mainnet", + "engine": { + "clique": { + "params": { + "period": 1, + "epoch": 30000 + } + } + }, + "params": { + "chainId": "59144", + "maxCodeSize": "0x6000", + "maxCodeSizeTransition": "0x0", + "maximumExtraDataSize": "0xffff", + "eip150Transition": "0x0", + "eip160Transition": "0x0", + "eip161abcTransition": "0x0", + "eip161dTransition": "0x0", + "eip155Transition": "0x0", + "eip140Transition": "0x0", + "eip211Transition": "0x0", + "eip214Transition": "0x0", + "eip658Transition": "0x0", + "eip145Transition": "0x0", + "eip1014Transition": "0x0", + "eip1052Transition": "0x0", + "eip1283Transition": "0x0", + "eip1283DisableTransition": "0x0", + "eip152Transition": "0x0", + "eip1108Transition": "0x0", + "eip1344Transition": "0x0", + "eip1884Transition": "0x0", + "eip2028Transition": "0x0", + "eip2200Transition": "0x0", + "eip2565Transition": "0x0", + "eip2929Transition": "0x0", + "eip2930Transition": "0x0", + "eip1559Transition": "0x0", + "eip1559FeeCollectorTransition": "0x0", + "eip3198Transition": "0x0", + "eip3529Transition": "0x0", + "eip3541Transition": "0x0" + }, + "genesis": { + "seal": { + "ethereum": { + "nonce": "0x0000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "number": "0x0", + "difficulty": "0x1", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x6391BFF3", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x00000000000000000000000000000000000000000000000000000000000000008F81e2E3F8b46467523463835F965fFE476E1c9E0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas": "0x8", + "gasLimit": "0x3A2C940" + }, + "nodes": [ + "enode://069800db9e6e0ec9cadca670994ef1aea2cfd3d88133e63ecadbc1cdbd1a5847b09838ee08d8b5f02a9c32ee13abeb4d4104bb5514e5322c9d7ee19f41ff3e51@3.132.73.210:31002", + "enode://a8e03a71eab12ec4b47bb6e19169d8e4dc7a58373a2476969bbe463f2dded6003037fa4dd5f71e15027f7fc8d7340956fbbefed67ddd116ac19a7f74da034b61@3.132.73.210:31003", + "enode://97706526cf79df9d930003644f9156805f6c8bd964fc79e083444f7014ce10c9bdd2c5049e63b58040dca1d4c82ebef970822198cf0714de830cff4111534ff1@18.223.198.165:31004", + "enode://24e1c654a801975a96b7f54ebd7452ab15777fc635c1db25bdbd4425fdb04e7f4768e9e838a87ab724320a765e41631d5d37758c933ad0e8668693558125c8aa@18.223.198.165:31000", + "enode://27010891d960f73d272a553f72b6336c6698db3ade98d631f09c764e57674a797be5ebc6829ddbb65ab564f439ebc75215d20aa98b6f351d12ea623e7d139ac3@3.132.73.210:31001", + "enode://228e1b8a4931e46f383e30721dac21fb8fb4e5e1b32c870e13b25478c82db3dc1cd9e7ceb93d302a766466b55638cc9c5cbfc43aa48fa41ced19baf365951f76@3.1.142.64:31002", + "enode://c22eb0d40fc3ad5ea710aeddea906567778166bfe18c157955e8c39b23a46c45db18a0fa2ba07f2b64c81178a8c796aec2a29151533920ead06fcdfc6d8d03c6@47.128.192.57:31004", + "enode://8ce733abe39fd7ae0a278b9893f85c1193c611a3886168690dd843435460f22cc4d61f9e8d0ace7f5905836a665319a31cccdaacdada2acc69972c382ecce7db@3.1.142.64:31003", + "enode://b7c1b2bed65a855f7a2104aac9a14674dfdf018fdac763415b373b29ce18cdb81d36328ba4e5c9f12629f3a50c3e8f9ee048f22dbdbe93a82813da89c6b81334@51.20.235.126:31004", + "enode://95270e0550848a72fb141cf27f1c4ea10714edde365b411dc0fa06c81c0f282ce155eb9fa472b6b8bb9ee98395eeaf4c5a7b02a01fe58b37ea98ba152eda4c37@13.50.94.193:31000", + "enode://72013391755f24f08567b932feeeec4c893c06e0b1fb480890c83bf87fd277ad86a5ab9cb586db9ae9970371a2f8cb0c96f6c9f69045abca0fb801db7f047138@51.20.235.126:31001" + ], + "accounts": { + "0000000000000000000000000000000000000001": { + "balance": "0x1", + "builtin": { + "name": "ecrecover", + "pricing": { + "linear": { + "base": 3000, + "word": 0 + } + } + } + }, + "0000000000000000000000000000000000000002": { + "balance": "0x1", + "builtin": { + "name": "sha256", + "pricing": { + "linear": { + "base": 60, + "word": 12 + } + } + } + }, + "0000000000000000000000000000000000000003": { + "balance": "0x1", + "builtin": { + "name": "ripemd160", + "pricing": { + "linear": { + "base": 600, + "word": 120 + } + } + } + }, + "0000000000000000000000000000000000000004": { + "balance": "0x1", + "builtin": { + "name": "identity", + "pricing": { + "linear": { + "base": 15, + "word": 3 + } + } + } + }, + "0000000000000000000000000000000000000005": { + "balance": "0x1", + "builtin": { + "name": "modexp", + "pricing": { + "modexp": { + "divisor": 20 + } + } + } + }, + "0000000000000000000000000000000000000006": { + "balance": "0x1", + "builtin": { + "name": "alt_bn128_add", + "pricing": { + "linear": { + "base": 500, + "word": 0 + } + } + } + }, + "0000000000000000000000000000000000000007": { + "balance": "0x1", + "builtin": { + "name": "alt_bn128_mul", + "pricing": { + "linear": { + "base": 40000, + "word": 0 + } + } + } + }, + "0000000000000000000000000000000000000008": { + "balance": "0x1", + "builtin": { + "name": "alt_bn128_pairing", + "pricing": { + "alt_bn128_pairing": { + "base": 100000, + "pair": 80000 + } + } + } + }, + + "F06B7BD371e46e96DF807d45ED1298BeeE8894BA": { + "balance": "0x8AC7230489E80000" + }, + "508Ca82Df566dCD1B0DE8296e70a96332cD644ec": { + "balance": "0x33B2E3C15095D385E180000" + }, + "0000000000000000000000000000000000000000": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000009": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000010": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000011": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000012": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000013": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000014": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000015": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000016": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000017": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000018": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000019": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000020": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000021": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000022": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000023": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000024": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000025": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000026": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000027": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000028": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000029": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000030": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000031": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000032": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000033": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000034": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000035": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000036": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000037": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000038": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000039": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000040": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000041": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000042": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000043": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000044": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000045": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000046": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000047": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000048": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000049": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000050": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000051": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000052": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000053": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000054": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000055": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000056": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000057": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000058": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000059": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000060": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000061": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000062": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000063": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000064": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000065": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000066": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000067": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000068": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000069": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000070": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000071": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000072": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000073": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000074": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000075": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000076": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000077": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000078": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000079": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000080": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000081": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000082": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000083": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000084": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000085": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000086": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000087": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000088": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000089": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000090": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000091": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000092": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000093": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000094": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000095": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000096": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000097": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000098": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000099": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009f": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000aa": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ab": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ac": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ad": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ae": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000af": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ba": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000be": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bf": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ca": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ce": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cf": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000da": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000db": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000dc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000dd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000de": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000df": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ea": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000eb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ec": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ed": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ee": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ef": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fa": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fe": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ff": { + "balance": "0x1" + } + } +} diff --git a/src/Nethermind/Chains/linea-sepolia.json b/src/Nethermind/Chains/linea-sepolia.json new file mode 100644 index 00000000000..d79985bd2b2 --- /dev/null +++ b/src/Nethermind/Chains/linea-sepolia.json @@ -0,0 +1,942 @@ +{ + "name": "Linea Sepolia", + "dataDir": "linea-sepolia", + "engine": { + "clique": { + "params": { + "period": 1, + "epoch": 30000 + } + } + }, + "params": { + "chainId": "59141", + "maxCodeSize": "0x6000", + "maxCodeSizeTransition": "0x0", + "maximumExtraDataSize": "0xffff", + "eip150Transition": "0x0", + "eip160Transition": "0x0", + "eip161abcTransition": "0x0", + "eip161dTransition": "0x0", + "eip155Transition": "0x0", + "eip140Transition": "0x0", + "eip211Transition": "0x0", + "eip214Transition": "0x0", + "eip658Transition": "0x0", + "eip145Transition": "0x0", + "eip1014Transition": "0x0", + "eip1052Transition": "0x0", + "eip1283Transition": "0x0", + "eip1283DisableTransition": "0x0", + "eip152Transition": "0x0", + "eip1108Transition": "0x0", + "eip1344Transition": "0x0", + "eip1884Transition": "0x0", + "eip2028Transition": "0x0", + "eip2200Transition": "0x0", + "eip2565Transition": "0x0", + "eip2929Transition": "0x0", + "eip2930Transition": "0x0", + "eip1559Transition": "0x0", + "eip1559FeeCollectorTransition": "0x0", + "eip3198Transition": "0x0", + "eip3529Transition": "0x0", + "eip3541Transition": "0x0" + }, + "genesis": { + "seal": { + "ethereum": { + "nonce": "0x0000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "number": "0x0", + "difficulty": "0x1", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x6391BFF3", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000a27342f1b74c0cfb2cda74bac1628d0c1a9752f20000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas": "0x8", + "gasLimit": "0x3A2C940" + }, + "nodes": [ + "enode://6f20afbe4397e51b717a7c1ad3095e79aee48c835eebd9237a3e8a16951ade1fe0e66e981e30ea269849fcb6ba03d838da37f524fabd2a557474194a2e2604fa@18.221.100.27:31002", + "enode://ce1e0d8e0500cb5c0ac56bdcdafb2d6320c3a2c5125b5ccf12f5dfc9b47ee74acbcafc32559017613136c9c36a0ce74ba4f83b7fb8244f099f3b15708d9d3129@3.23.75.47:31000", + "enode://1b026a5eb0ae74300f58987d235ef0e3a550df963345cb3574be3b0b54378bd11f14dfd515a8976f2c2d2826090e9507b8ccc24f896a9ffffffcabcfd996a733@3.129.120.128:31001" + ], + "accounts": { + "0000000000000000000000000000000000000001": { + "balance": "0x1", + "builtin": { + "name": "ecrecover", + "pricing": { + "linear": { + "base": 3000, + "word": 0 + } + } + } + }, + "0000000000000000000000000000000000000002": { + "balance": "0x1", + "builtin": { + "name": "sha256", + "pricing": { + "linear": { + "base": 60, + "word": 12 + } + } + } + }, + "0000000000000000000000000000000000000003": { + "balance": "0x1", + "builtin": { + "name": "ripemd160", + "pricing": { + "linear": { + "base": 600, + "word": 120 + } + } + } + }, + "0000000000000000000000000000000000000004": { + "balance": "0x1", + "builtin": { + "name": "identity", + "pricing": { + "linear": { + "base": 15, + "word": 3 + } + } + } + }, + "0000000000000000000000000000000000000005": { + "balance": "0x1", + "builtin": { + "name": "modexp", + "pricing": { + "modexp": { + "divisor": 20 + } + } + } + }, + "0000000000000000000000000000000000000006": { + "balance": "0x1", + "builtin": { + "name": "alt_bn128_add", + "pricing": { + "linear": { + "base": 500, + "word": 0 + } + } + } + }, + "0000000000000000000000000000000000000007": { + "balance": "0x1", + "builtin": { + "name": "alt_bn128_mul", + "pricing": { + "linear": { + "base": 40000, + "word": 0 + } + } + } + }, + "0000000000000000000000000000000000000008": { + "balance": "0x1", + "builtin": { + "name": "alt_bn128_pairing", + "pricing": { + "alt_bn128_pairing": { + "base": 100000, + "pair": 80000 + } + } + } + }, + + "Ed8587afbDBb2504EC31583B3377447d6fA322B2": { + "balance": "0x8AC7230489E80000" + }, + "971e727e956690b9957be6d51Ec16E73AcAC83A7": { + "balance": "0x33B2E3C9FD0803CE8000000" + }, + "47C63d1E391FcB3dCdC40C4d7fA58ADb172f8c38": { + "balance": "0x8AC7230489E80000" + }, + "5948ccC8A59B464c6DcC87db4004Cf0a8d600C93": { + "balance": "0x8AC7230489E80000" + }, + "73259D1d7534D1b68542e3240Cf76A03a5C2530F": { + "balance": "0x8AC7230489E80000" + }, + "857dBFEbD7D29a75FF06d1423c418Da3b6E07292": { + "balance": "0x8AC7230489E80000" + }, + "C702eAfe94b232ed50A7d4aC204306b97cdDAB0B": { + "balance": "0x8AC7230489E80000" + }, + "Cc791070a5aA20f7C77CFd02E1713AfA30A23021": { + "balance": "0x8AC7230489E80000" + }, + "Cd6366E24283D5D2fA10350Ac75d8a0B798E1FAe": { + "balance": "0x8AC7230489E80000" + }, + "d83C3dC7257Eb2E304CBEB50b0B98bc2dd2cdCd9": { + "balance": "0x8AC7230489E80000" + }, + "df5443dBe811fF9a121466dDa3C96292a3F6f43D": { + "balance": "0x8AC7230489E80000" + }, + "0000000000000000000000000000000000000000": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000009": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000000f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000010": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000011": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000012": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000013": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000014": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000015": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000016": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000017": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000018": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000019": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000001f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000020": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000021": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000022": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000023": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000024": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000025": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000026": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000027": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000028": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000029": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000002f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000030": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000031": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000032": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000033": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000034": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000035": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000036": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000037": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000038": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000039": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000003f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000040": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000041": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000042": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000043": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000044": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000045": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000046": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000047": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000048": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000049": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000004f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000050": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000051": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000052": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000053": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000054": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000055": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000056": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000057": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000058": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000059": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000005f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000060": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000061": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000062": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000063": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000064": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000065": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000066": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000067": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000068": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000069": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000006f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000070": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000071": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000072": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000073": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000074": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000075": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000076": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000077": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000078": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000079": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000007f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000080": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000081": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000082": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000083": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000084": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000085": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000086": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000087": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000088": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000089": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000008f": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000090": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000091": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000092": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000093": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000094": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000095": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000096": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000097": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000098": { + "balance": "0x1" + }, + "0000000000000000000000000000000000000099": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009a": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009b": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009c": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009d": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009e": { + "balance": "0x1" + }, + "000000000000000000000000000000000000009f": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000a9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000aa": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ab": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ac": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ad": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ae": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000af": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000b9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ba": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000be": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000bf": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000c9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ca": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ce": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000cf": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000d9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000da": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000db": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000dc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000dd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000de": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000df": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000e9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ea": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000eb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ec": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ed": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ee": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ef": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f0": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f1": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f2": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f3": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f4": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f5": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f6": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f7": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f8": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000f9": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fa": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fb": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fc": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fd": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000fe": { + "balance": "0x1" + }, + "00000000000000000000000000000000000000ff": { + "balance": "0x1" + } + } +} diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-mainnet.cfg new file mode 100644 index 00000000000..acb645db219 --- /dev/null +++ b/src/Nethermind/Nethermind.Runner/configs/linea-mainnet.cfg @@ -0,0 +1,27 @@ +{ + "Init": { + "ChainSpecPath": "chainspec/linea-mainnet.json", + "GenesisHash": "0xb6762a65689107b2326364aefc18f94cda413209fab35c00d4af51eaa20ffbc6", + "BaseDbPath": "nethermind_db/linea-mainnet", + "LogFileName": "linea-mainnet.logs.txt" + }, + "Merge": { + "Enabled": false + }, + "TxPool": { + "BlobsSupport": "Disabled" + }, + "Metrics": { + "NodeName": "LineaMainnet" + }, + "Sync": { + "SnapSync": true, + "PivotNumber": "0x9d6cba", + "PivotHash": "0xfd9805546776bc4b4369264fef5582db4c5ee4929c27af02132e2609c4a6bb94", + "PivotTotalDifficulty": "0x13ad975" + }, + "JsonRpc": { + "Enabled": true, + "Port": 8545 + } +} diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.cfg new file mode 100644 index 00000000000..775ee7ac150 --- /dev/null +++ b/src/Nethermind/Nethermind.Runner/configs/linea-mainnet_archive.cfg @@ -0,0 +1,24 @@ +{ + "Init": { + "ChainSpecPath": "chainspec/linea-mainnet.json", + "GenesisHash": "0xb6762a65689107b2326364aefc18f94cda413209fab35c00d4af51eaa20ffbc6", + "BaseDbPath": "nethermind_db/linea-mainnet_archive", + "LogFileName": "linea-mainnet_archive.logs.txt" + }, + "Merge": { + "Enabled": false + }, + "Pruning": { + "Mode": "None" + }, + "Metrics": { + "NodeName": "LineaMainnetArchive" + }, + "TxPool": { + "BlobsSupport": "Disabled" + }, + "JsonRpc": { + "Enabled": true, + "Port": 8545 + } +} diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-sepolia.cfg new file mode 100644 index 00000000000..83063766558 --- /dev/null +++ b/src/Nethermind/Nethermind.Runner/configs/linea-sepolia.cfg @@ -0,0 +1,27 @@ +{ + "Init": { + "ChainSpecPath": "chainspec/linea-sepolia.json", + "GenesisHash": "0x65a64c825d7c13ce1bf077801d0b6b2a89853e19e4a89a5433d7d85d2102a20b", + "BaseDbPath": "nethermind_db/linea-sepolia", + "LogFileName": "linea-sepolia.logs.txt" + }, + "Merge": { + "Enabled": false + }, + "TxPool": { + "BlobsSupport": "Disabled" + }, + "Metrics": { + "NodeName": "LineaSepolia" + }, + "Sync": { + "SnapSync": true, + "PivotNumber": "4477400", + "PivotHash": "0xacbec6fad90a6cec8bdbff5f6041c90dad06027b0097c677a6a8c5c28382f375", + "PivotTotalDifficulty": "8954801" + }, + "JsonRpc": { + "Enabled": true, + "Port": 8545 + } +} diff --git a/src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.cfg new file mode 100644 index 00000000000..ecca09dde1b --- /dev/null +++ b/src/Nethermind/Nethermind.Runner/configs/linea-sepolia_archive.cfg @@ -0,0 +1,24 @@ +{ + "Init": { + "ChainSpecPath": "chainspec/linea-sepolia.json", + "GenesisHash": "0x65a64c825d7c13ce1bf077801d0b6b2a89853e19e4a89a5433d7d85d2102a20b", + "BaseDbPath": "nethermind_db/linea-sepolia_archive", + "LogFileName": "linea-sepolia.logs.txt" + }, + "Merge": { + "Enabled": false + }, + "Pruning": { + "Mode": "None" + }, + "Metrics": { + "NodeName": "LineaSepoliaArchive" + }, + "TxPool": { + "BlobsSupport": "Disabled" + }, + "JsonRpc": { + "Enabled": true, + "Port": 8545 + } +} diff --git a/src/bench_precompiles b/src/bench_precompiles index c1089350ef0..9c9705ed4e7 160000 --- a/src/bench_precompiles +++ b/src/bench_precompiles @@ -1 +1 @@ -Subproject commit c1089350ef0daa3f0bd437ce9d1626de973f9a93 +Subproject commit 9c9705ed4e7f264abc8f343a848016e7bf6501b3 From 631f879f6718b02a8bf878e992690221add00867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Chodo=C5=82a?= <43241881+kamilchodola@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:49:53 +0200 Subject: [PATCH 13/13] Add linea to sync supported chains (#7628) --- .github/workflows/sync-supported-chains.yml | 18 +++++++++++++++--- .../workflow_config/sync_testnets_matrix.json | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync-supported-chains.yml b/.github/workflows/sync-supported-chains.yml index 4e7047b9298..aac65b6aaa3 100644 --- a/.github/workflows/sync-supported-chains.yml +++ b/.github/workflows/sync-supported-chains.yml @@ -12,6 +12,10 @@ on: description: "Docker image to be used by action" default: "" required: false + network_filter: + description: "Usefull for manual execution on only specified networks - provide partial or full name. Will execute action only on networks which contains phrase." + default: "" + required: false env: DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: "1" @@ -29,7 +33,14 @@ jobs: path: nethermind - name: Set Matrix id: set-matrix - run: echo "matrix=$(jq -c . nethermind/scripts/workflow_config/sync_testnets_matrix.json)" >> $GITHUB_OUTPUT + run: | + matrix=$(cat nethermind/scripts/workflow_config/sync_testnets_matrix.json) + + if [ -n "${{ github.event.inputs.network_filter }}" ]; then + matrix=$(echo "$matrix" | jq --arg filter "${{ github.event.inputs.network_filter }}" '[.[] | select(.network | contains($filter))]') + fi + + echo "matrix=$(echo "$matrix" | jq -c .)" >> $GITHUB_OUTPUT create_a_runner: needs: [setup-matrix] @@ -188,7 +199,7 @@ jobs: required_count["Processed"]=20 network="${{ matrix.config.network }}" - if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" ]]; then + if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" && "$network" != "linea-mainnet" && "$network" != "linea-sepolia" ]]; then good_logs["Synced Chain Head"]=0 required_count["Synced Chain Head"]=20 fi @@ -250,7 +261,7 @@ jobs: done - name: Get Consensus Logs - if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet' + if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet' && matrix.config.network != 'linea-mainnet' && matrix.config.network != 'linea-sepolia' run: | network="${{ matrix.config.network }}" if [[ "$network" == base-* || "$network" == op-* ]]; then @@ -265,6 +276,7 @@ jobs: - name: Destroy VM if: always() + continue-on-error: true id: run-linode-action uses: kamilchodola/linode-github-runner/.github/actions/linode-machine-manager@main with: diff --git a/scripts/workflow_config/sync_testnets_matrix.json b/scripts/workflow_config/sync_testnets_matrix.json index 758c25f29a7..116ca8e9da0 100644 --- a/scripts/workflow_config/sync_testnets_matrix.json +++ b/scripts/workflow_config/sync_testnets_matrix.json @@ -86,5 +86,21 @@ "checkpoint-sync-url": "", "timeout": 180, "agent": "g6-standard-6" + }, + { + "network": "linea-mainnet", + "cl": "", + "cl_image": "", + "checkpoint-sync-url": "", + "timeout": 180, + "agent": "g6-standard-6" + }, + { + "network": "linea-sepolia", + "cl": "", + "cl_image": "", + "checkpoint-sync-url": "", + "timeout": 180, + "agent": "g6-standard-6" } ]