Add data gas calculations

src/Nethermind/Directory.Packages.props

@@ -9,7 +9,7 @@
     <PackageVersion Include="AspNetCore.HealthChecks.UI.InMemory.Storage" Version="6.0.5" />
     <PackageVersion Include="BenchmarkDotNet" Version="0.13.5" />
     <PackageVersion Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.5" />
-    <PackageVersion Include="Ckzg.Bindings" Version="0.1.2.402" />
+    <PackageVersion Include="Ckzg.Bindings" Version="0.1.2.437" />
     <PackageVersion Include="Colorful.Console" Version="1.2.15" />
     <PackageVersion Include="CommandLineParser" Version="2.9.1" />
     <PackageVersion Include="ConcurrentHashSet" Version="1.3.0" />

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -134,7 +134,7 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             ITxValidator txValidator = new TxValidator(TestBlockchainIds.ChainId);
             IHeaderValidator headerValidator = new HeaderValidator(blockTree, Sealer, specProvider, _logManager);
             IUnclesValidator unclesValidator = new UnclesValidator(blockTree, headerValidator, _logManager);
-            IBlockValidator blockValidator = new BlockValidator(txValidator, headerValidator, unclesValidator, specProvider, _logManager);
+            IBlockValidator blockValidator = new BlockValidator(txValidator, headerValidator, unclesValidator, specProvider, blockTree, _logManager);
             IStorageProvider storageProvider = new StorageProvider(trieStore, stateProvider, _logManager);
             IVirtualMachine virtualMachine = new VirtualMachine(
                 blockhashProvider,
@@ -151,12 +151,14 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
                         stateProvider,
                         storageProvider,
                         virtualMachine,
+                        blockTree,
                         _logManager),
                     stateProvider),
                 stateProvider,
                 storageProvider,
                 receiptStorage,
                 NullWitnessCollector.Instance,
+                blockTree,
                 _logManager);
 
             IBlockchainProcessor blockchainProcessor = new BlockchainProcessor(

src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs

@@ -4,13 +4,15 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using Nethermind.Blockchain;
 using Nethermind.Consensus.Validators;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Specs;
 using Nethermind.Crypto;
 using Nethermind.Db;
+using Nethermind.Db.Blooms;
 using Nethermind.Int256;
 using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
@@ -20,6 +22,7 @@
 using Nethermind.Specs.Forks;
 using Nethermind.Specs.Test;
 using Nethermind.State;
+using Nethermind.State.Repositories;
 using Nethermind.Trie.Pruning;
 using NUnit.Framework;
 
@@ -73,11 +76,16 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
                 specProvider,
                 _logManager);
 
+            IDb blockInfoDb = new MemDb();
+            IBlockTree blockTree = new BlockTree(new MemDb(), new MemDb(), blockInfoDb,
+                new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, _logManager);
+
             TransactionProcessor transactionProcessor = new(
                 specProvider,
                 stateProvider,
                 storageProvider,
                 virtualMachine,
+                blockTree,
                 _logManager);
 
             InitializeTestState(test, stateProvider, storageProvider, specProvider);
@@ -91,8 +99,8 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
             header.MixHash = test.CurrentRandom;
 
             Stopwatch stopwatch = Stopwatch.StartNew();
-            var txValidator = new TxValidator((MainnetSpecProvider.Instance.ChainId));
-            var spec = specProvider.GetSpec((ForkActivation)test.CurrentNumber);
+            TxValidator? txValidator = new((MainnetSpecProvider.Instance.ChainId));
+            IReleaseSpec? spec = specProvider.GetSpec((ForkActivation)test.CurrentNumber);
             if (test.Transaction.ChainId == null)
                 test.Transaction.ChainId = MainnetSpecProvider.Instance.ChainId;
             bool isValid = txValidator.IsWellFormed(test.Transaction, spec);

src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.TestAccountAbstractionRpcBlockchain.cs

@@ -193,6 +193,7 @@ protected override BlockProcessor CreateBlockProcessor()
                     Storage,
                     ReceiptStorage,
                     NullWitnessCollector.Instance,
+                    BlockTree,
                     LogManager);
 
                 var parser = new AbiDefinitionParser();
@@ -282,6 +283,7 @@ private IBlockValidator CreateBlockValidator()
                     headerValidator,
                     Always.Valid,
                     SpecProvider,
+                    BlockTree,
                     LogManager);
             }
 

src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs

@@ -24,6 +24,7 @@
 using System.Threading.Tasks;
 using System.Threading;
 using FluentAssertions;
+using Nethermind.Blockchain.Find;
 using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Rewards;
 using Nethermind.Consensus.Withdrawals;
@@ -52,6 +53,7 @@ public void Prepared_block_contains_author_field()
                 new StorageProvider(trieStore, stateProvider, LimboLogs.Instance),
                 NullReceiptStorage.Instance,
                 NullWitnessCollector.Instance,
+                Substitute.For<IBlockFinder>(),
                 LimboLogs.Instance);
 
             BlockHeader header = Build.A.BlockHeader.WithAuthor(TestItem.AddressD).TestObject;
@@ -84,6 +86,7 @@ public void Can_store_a_witness()
                 new StorageProvider(trieStore, stateProvider, LimboLogs.Instance),
                 NullReceiptStorage.Instance,
                 witnessCollector,
+                Substitute.For<IBlockFinder>(),
                 LimboLogs.Instance);
 
             BlockHeader header = Build.A.BlockHeader.WithAuthor(TestItem.AddressD).TestObject;
@@ -114,6 +117,7 @@ public void Recovers_state_on_cancel()
                 new StorageProvider(trieStore, stateProvider, LimboLogs.Instance),
                 NullReceiptStorage.Instance,
                 NullWitnessCollector.Instance,
+                Substitute.For<IBlockFinder>(),
                 LimboLogs.Instance);
 
             BlockHeader header = Build.A.BlockHeader.WithNumber(1).WithAuthor(TestItem.AddressD).TestObject;

src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs

@@ -69,6 +69,7 @@ public void Test()
                 stateProvider,
                 storageProvider,
                 virtualMachine,
+                blockTree,
                 LimboLogs.Instance);
             BlockProcessor blockProcessor = new(
                 specProvider,
@@ -79,6 +80,7 @@ public void Test()
                 storageProvider,
                 NullReceiptStorage.Instance,
                 NullWitnessCollector.Instance,
+                blockTree,
                 LimboLogs.Instance);
             BlockchainProcessor blockchainProcessor = new(
                 blockTree,

src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs

@@ -77,6 +77,7 @@ public void Setup()
                 stateProvider,
                 storageProvider,
                 virtualMachine,
+                _blockTree,
                 LimboLogs.Instance);
 
             BlockProcessor blockProcessor = new(
@@ -88,6 +89,7 @@ public void Setup()
                 storageProvider,
                 NullReceiptStorage.Instance,
                 new WitnessCollector(memDbProvider.StateDb, LimboLogs.Instance),
+                _blockTree,
                 LimboLogs.Instance);
             _blockchainProcessor = new BlockchainProcessor(
                 _blockTree,

src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs

@@ -158,7 +158,7 @@ public static IEnumerable EnoughShardBlobTransactionsSelectedTestCases
                     tx.BlobVersionedHashes = new byte[1][];
                 });
                 maxTransactionsSelected.Transactions[1].BlobVersionedHashes =
-                    new byte[Eip4844Constants.MaxBlobsPerBlock - 1][];
+                    new byte[Eip4844Constants.MaxDataGasPerTransaction / Eip4844Constants.DataGasPerBlob - 1][];
                 maxTransactionsSelected.ExpectedSelectedTransactions.AddRange(
                     maxTransactionsSelected.Transactions.OrderBy(t => t.Nonce).Take(2));
                 yield return new TestCaseData(maxTransactionsSelected).SetName("Enough transactions selected");
@@ -168,7 +168,8 @@ public static IEnumerable EnoughShardBlobTransactionsSelectedTestCases
                 Transaction[] expectedSelectedTransactions =
                     enoughTransactionsSelected.Transactions.OrderBy(t => t.Nonce).ToArray();
                 expectedSelectedTransactions[0].Type = TxType.Blob;
-                expectedSelectedTransactions[0].BlobVersionedHashes = new byte[Eip4844Constants.MaxBlobsPerBlock][];
+                expectedSelectedTransactions[0].BlobVersionedHashes =
+                    new byte[Eip4844Constants.MaxDataGasPerTransaction / Eip4844Constants.DataGasPerBlob][];
                 expectedSelectedTransactions[1].Type = TxType.Blob;
                 expectedSelectedTransactions[1].BlobVersionedHashes = new byte[1][];
                 enoughTransactionsSelected.ExpectedSelectedTransactions.AddRange(

src/Nethermind/Nethermind.Blockchain.Test/Validators/BlockValidatorTests.cs

@@ -4,6 +4,7 @@
 using Nethermind.Consensus.Validators;
 using Nethermind.Core;
 using Nethermind.Core.Specs;
+using Nethermind.Core.Test;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Logging;
 using Nethermind.Specs;
@@ -23,7 +24,7 @@ public void When_more_uncles_than_allowed_returns_false()
             releaseSpec.MaximumUncleCount = 0;
             ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, releaseSpec));
 
-            BlockValidator blockValidator = new(txValidator, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+            BlockValidator blockValidator = new(txValidator, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
             bool noiseRemoved = blockValidator.ValidateSuggestedBlock(Build.A.Block.TestObject);
             Assert.True(noiseRemoved);
 

src/Nethermind/Nethermind.Blockchain.Test/Validators/ShardBlobBlockValidatorTests.cs

@@ -4,7 +4,9 @@
 using Nethermind.Consensus.Validators;
 using Nethermind.Core;
 using Nethermind.Core.Specs;
+using Nethermind.Core.Test;
 using Nethermind.Core.Test.Builders;
+using Nethermind.Evm;
 using Nethermind.Logging;
 using Nethermind.Specs.Forks;
 using Nethermind.Specs.Test;
@@ -18,7 +20,7 @@ public class ShardBlobBlockValidatorTests
     public void Not_null_ExcessDataGas_is_invalid_pre_cancun()
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Shanghai.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block
             .WithWithdrawalsRoot(TestItem.KeccakA)
             .WithWithdrawals(TestItem.WithdrawalA_1Eth)
@@ -30,7 +32,7 @@ public void Not_null_ExcessDataGas_is_invalid_pre_cancun()
     public void Null_ExcessDataGas_is_invalid_post_cancun()
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Cancun.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block
             .WithWithdrawalsRoot(TestItem.KeccakA)
             .WithWithdrawals(TestItem.WithdrawalA_1Eth)
@@ -39,18 +41,18 @@ public void Null_ExcessDataGas_is_invalid_post_cancun()
     }
 
     [TestCase(0, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerBlock - 1, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerBlock, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerBlock + 1, ExpectedResult = false)]
+    [TestCase(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob - 1, ExpectedResult = true)]
+    [TestCase(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob, ExpectedResult = true)]
+    [TestCase(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob + 1, ExpectedResult = false)]
     public bool Blobs_per_block_count_is_valid(int blobsCount)
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Cancun.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         return blockValidator.ValidateSuggestedBlock(
             Build.A.Block
                 .WithWithdrawalsRoot(TestItem.KeccakA)
                 .WithWithdrawals(TestItem.WithdrawalA_1Eth)
-                .WithExcessDataGas(1)
+                .WithExcessDataGas(IntrinsicGasCalculator.CalculateExcessDataGas(0, blobsCount, specProvider.GenesisSpec)!.Value)
                 .WithTransactions(Build.A.Transaction.WithBlobVersionedHashes(blobsCount).TestObject)
                 .TestObject);
     }

src/Nethermind/Nethermind.Blockchain.Test/Validators/TxValidatorTests.cs

@@ -307,9 +307,9 @@ public bool MaxFeePerDataGas_should_be_set_for_blob_tx_only(TxType txType, bool
     [TestCase(Eip4844Constants.MinBlobsPerTransaction - 1, ExpectedResult = false)]
     [TestCase(Eip4844Constants.MinBlobsPerTransaction, ExpectedResult = true)]
     [TestCase(Eip4844Constants.MinBlobsPerTransaction + 1, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerTransaction - 1, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerTransaction, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerTransaction + 1, ExpectedResult = false)]
+    [TestCase(Eip4844Constants.MaxDataGasPerTransaction / Eip4844Constants.DataGasPerBlob - 1, ExpectedResult = true)]
+    [TestCase(Eip4844Constants.MaxDataGasPerTransaction / Eip4844Constants.DataGasPerBlob, ExpectedResult = true)]
+    [TestCase(Eip4844Constants.MaxDataGasPerTransaction / Eip4844Constants.DataGasPerBlob + 1, ExpectedResult = false)]
     public bool Blobs_count_should_be_within_constraints(int blobsCount)
     {
         byte[] sigData = new byte[65];

src/Nethermind/Nethermind.Blockchain.Test/Validators/WithdrawalValidatorTests.cs

@@ -5,6 +5,7 @@
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Specs;
+using Nethermind.Core.Test;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Logging;
 using Nethermind.Specs.Forks;
@@ -20,7 +21,7 @@ public class WithdrawalValidatorTests
     public void Not_null_withdrawals_are_invalid_pre_shanghai()
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, London.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(new Withdrawal[] { TestItem.WithdrawalA_1Eth, TestItem.WithdrawalB_2Eth }).TestObject);
         Assert.False(isValid);
     }
@@ -29,7 +30,7 @@ public void Not_null_withdrawals_are_invalid_pre_shanghai()
     public void Null_withdrawals_are_invalid_post_shanghai()
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Shanghai.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.TestObject);
         Assert.False(isValid);
     }
@@ -38,7 +39,7 @@ public void Null_withdrawals_are_invalid_post_shanghai()
     public void Withdrawals_with_incorrect_withdrawals_root_are_invalid()
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Shanghai.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         Withdrawal[] withdrawals = { TestItem.WithdrawalA_1Eth, TestItem.WithdrawalB_2Eth };
         Keccak withdrawalRoot = new WithdrawalTrie(withdrawals).RootHash;
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(TestItem.KeccakD).TestObject);
@@ -49,7 +50,7 @@ public void Withdrawals_with_incorrect_withdrawals_root_are_invalid()
     public void Empty_withdrawals_are_valid_post_shanghai()
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Shanghai.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         Withdrawal[] withdrawals = { };
         Keccak withdrawalRoot = new WithdrawalTrie(withdrawals).RootHash;
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(withdrawalRoot).TestObject);
@@ -60,7 +61,7 @@ public void Empty_withdrawals_are_valid_post_shanghai()
     public void Correct_withdrawals_block_post_shanghai()
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Shanghai.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, NullBlockFinder.Instance, LimboLogs.Instance);
         Withdrawal[] withdrawals = { TestItem.WithdrawalA_1Eth, TestItem.WithdrawalB_2Eth };
         Keccak withdrawalRoot = new WithdrawalTrie(withdrawals).RootHash;
         bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block.WithWithdrawals(withdrawals).WithWithdrawalsRoot(withdrawalRoot).TestObject);

src/Nethermind/Nethermind.Blockchain/BlockTree.cs

@@ -23,6 +23,7 @@
 using Nethermind.Serialization.Rlp;
 using Nethermind.State.Repositories;
 using Nethermind.Db.Blooms;
+using Nethermind.Evm.TransactionProcessing;
 
 namespace Nethermind.Blockchain
 {
@@ -2098,5 +2099,10 @@ public void ForkChoiceUpdated(Keccak? finalizedBlockHash, Keccak? safeBlockHash)
                 _metadataDb.Set(MetadataDbKeys.SafeBlockHash, Rlp.Encode(SafeHash!).Bytes);
             }
         }
+
+        BlockHeader? IParentBlockHeaderFinder.FindParentHeader(BlockHeader block) =>
+            this.FindParentHeader(block,
+                BlockTreeLookupOptions.TotalDifficultyNotNeeded | BlockTreeLookupOptions.DoNotCreateLevelIfMissing |
+                BlockTreeLookupOptions.AllowInvalid);
     }
 }

src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs

@@ -4,10 +4,11 @@
 using System;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
+using Nethermind.Evm.TransactionProcessing;
 
 namespace Nethermind.Blockchain.Find
 {
-    public interface IBlockFinder
+    public interface IBlockFinder : IParentBlockHeaderFinder
     {
         Keccak HeadHash { get; }