Add JSON-RPC endpoints for EIP-4844 (#5558)

Add JSON-RPC endpoints for EIP-4844

src/Nethermind/Directory.Packages.props

@@ -9,7 +9,7 @@
     <PackageVersion Include="AspNetCore.HealthChecks.UI.InMemory.Storage" Version="6.0.5" />
     <PackageVersion Include="BenchmarkDotNet" Version="0.13.5" />
     <PackageVersion Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.5" />
-    <PackageVersion Include="Ckzg.Bindings" Version="0.1.2.112" />
+    <PackageVersion Include="Ckzg.Bindings" Version="0.1.2.402" />
     <PackageVersion Include="Colorful.Console" Version="1.2.15" />
     <PackageVersion Include="CommandLineParser" Version="2.9.1" />
     <PackageVersion Include="ConcurrentHashSet" Version="1.3.0" />

src/Nethermind/Nethermind.Consensus/Validators/BlockValidator.cs

@@ -195,9 +195,16 @@ private bool ValidateWithdrawals(Block block, IReleaseSpec spec, out string? err
 
     private bool ValidateBlobs(Block block, IReleaseSpec spec, out string? error)
     {
-        if (spec.IsEip4844Enabled ^ block.ExcessDataGas is not null)
+        if (spec.IsEip4844Enabled && block.ExcessDataGas is null)
         {
-            error = $"ExcessDataGas cannot be null in block {block.Hash} when EIP-4844 activated.";
+            error = "ExcessDataGas field is not set.";
+            if (_logger.IsWarn) _logger.Warn(error);
+            return false;
+        }
+
+        if (!spec.IsEip4844Enabled && block.ExcessDataGas is not null)
+        {
+            error = "ExcessDataGas field should not have value.";
             if (_logger.IsWarn) _logger.Warn(error);
             return false;
         }

src/Nethermind/Nethermind.Consensus/Validators/TxValidator.cs

@@ -146,7 +146,7 @@ transaction.BlobKzgs is not null ||
                      i < transaction.BlobVersionedHashes!.Length;
                      i++, n += Ckzg.Ckzg.BytesPerCommitment)
                 {
-                    if (!KzgPolynomialCommitments.TryComputeCommitmentV1(
+                    if (!KzgPolynomialCommitments.TryComputeCommitmentHashV1(
                             commitements[n..(n + Ckzg.Ckzg.BytesPerCommitment)], hash) ||
                         !hash.SequenceEqual(transaction.BlobVersionedHashes![i]))
                     {

src/Nethermind/Nethermind.Core.Test/Builders/TransactionBuilder.cs

@@ -89,6 +89,9 @@ public TransactionBuilder<T> WithMaxFeePerGas(UInt256 feeCap)
             return this;
         }
 
+        public TransactionBuilder<T> WithMaxFeePerGasIfSupports1559(UInt256 feeCap) =>
+            TestObjectInternal.Supports1559 ? WithMaxFeePerGas(feeCap) : this;
+
         public TransactionBuilder<T> WithMaxPriorityFeePerGas(UInt256 maxPriorityFeePerGas)
         {
             TestObjectInternal.GasPrice = maxPriorityFeePerGas;
@@ -166,6 +169,35 @@ public TransactionBuilder<T> WithBlobs(byte[] blobs)
 
             return this;
         }
+
+        public TransactionBuilder<T> WithShardBlobTxTypeAndFields(int blobCount)
+        {
+            if (blobCount is 0)
+            {
+                return this;
+            }
+
+            TestObjectInternal.Type = TxType.Blob;
+            TestObjectInternal.MaxFeePerDataGas ??= 1;
+            TestObjectInternal.Blobs = new byte[Ckzg.Ckzg.BytesPerBlob * blobCount];
+            TestObjectInternal.BlobKzgs = new byte[Ckzg.Ckzg.BytesPerCommitment * blobCount];
+            TestObjectInternal.BlobProofs = new byte[Ckzg.Ckzg.BytesPerProof * blobCount];
+            TestObjectInternal.BlobVersionedHashes = new byte[blobCount][];
+            for (int i = 0; i < blobCount; i++)
+            {
+                TestObjectInternal.BlobVersionedHashes[i] = new byte[32];
+                TestObjectInternal.Blobs[Ckzg.Ckzg.BytesPerBlob * i] = 1;
+                KzgPolynomialCommitments.KzgifyBlob(
+                    TestObjectInternal.Blobs.AsSpan(Ckzg.Ckzg.BytesPerBlob * i, Ckzg.Ckzg.BytesPerBlob * (i + 1)),
+                    TestObjectInternal.BlobKzgs.AsSpan(Ckzg.Ckzg.BytesPerCommitment * i, Ckzg.Ckzg.BytesPerCommitment * (i + 1)),
+                    TestObjectInternal.BlobProofs.AsSpan(Ckzg.Ckzg.BytesPerProof * i, Ckzg.Ckzg.BytesPerProof * (i + 1)),
+                    TestObjectInternal.BlobVersionedHashes[i].AsSpan());
+            }
+
+
+            return this;
+        }
+
         public TransactionBuilder<T> WithBlobKzgs(byte[] blobKzgs)
         {
             TestObjectInternal.BlobKzgs = blobKzgs;

src/Nethermind/Nethermind.Core/Extensions/Bytes.cs

@@ -518,6 +518,18 @@ public StateSmall(byte[] bytes, bool withZeroX)
             public readonly bool WithZeroX;
         }
 
+        private readonly struct StateSmallMemory
+        {
+            public StateSmallMemory(Memory<byte> bytes, bool withZeroX)
+            {
+                Bytes = bytes;
+                WithZeroX = withZeroX;
+            }
+
+            public readonly Memory<byte> Bytes;
+            public readonly bool WithZeroX;
+        }
+
         private struct StateOld
         {
             public StateOld(byte[] bytes, int leadingZeros, bool withZeroX, bool withEip55Checksum)
@@ -579,6 +591,37 @@ public static string ByteArrayToHexViaLookup32Safe(byte[] bytes, bool withZeroX)
             });
         }
 
+        [DebuggerStepThrough]
+        public static string ByteArrayToHexViaLookup32Safe(Memory<byte> bytes, bool withZeroX)
+        {
+            if (bytes.Length == 0)
+            {
+                return withZeroX ? "0x" : string.Empty;
+            }
+
+            int length = bytes.Length * 2 + (withZeroX ? 2 : 0);
+            StateSmallMemory stateToPass = new(bytes, withZeroX);
+
+            return string.Create(length, stateToPass, static (chars, state) =>
+            {
+                ref char charsRef = ref MemoryMarshal.GetReference(chars);
+
+                Memory<byte> bytes = state.Bytes;
+                if (bytes.Length == 0)
+                {
+                    if (state.WithZeroX)
+                    {
+                        chars[1] = 'x';
+                        chars[0] = '0';
+                    }
+
+                    return;
+                }
+
+                OutputBytesToCharHex(ref bytes.Span[0], state.Bytes.Length, ref charsRef, state.WithZeroX, leadingZeros: 0);
+            });
+        }
+
         [DebuggerStepThrough]
         private static string ByteArrayToHexViaLookup32(byte[] bytes, bool withZeroX, bool skipLeadingZeros,
             bool withEip55Checksum)

src/Nethermind/Nethermind.Crypto/KzgPolynomialCommitments.cs

@@ -27,7 +27,7 @@ public static class KzgPolynomialCommitments
 
     private static Task? _initializeTask;
 
-    public static Task Initialize(ILogger? logger = null) => _initializeTask ??= Task.Run(() =>
+    public static Task InitializeAsync(ILogger? logger = null) => _initializeTask ??= Task.Run(() =>
     {
         if (_ckzgSetup != IntPtr.Zero) return;
 
@@ -52,7 +52,7 @@ public static Task Initialize(ILogger? logger = null) => _initializeTask ??= Tas
     /// <param name="hashBuffer">Holds the output, can safely contain any data before the call.</param>
     /// <returns>Result of the attempt</returns>
     /// <exception cref="ArgumentException"></exception>
-    public static bool TryComputeCommitmentV1(ReadOnlySpan<byte> commitment, Span<byte> hashBuffer)
+    public static bool TryComputeCommitmentHashV1(ReadOnlySpan<byte> commitment, Span<byte> hashBuffer)
     {
         if (commitment.Length != Ckzg.Ckzg.BytesPerCommitment)
         {
@@ -98,4 +98,14 @@ public static bool AreProofsValid(byte[] blobs, byte[] commitments, byte[] proof
             return false;
         }
     }
+
+    /// <summary>
+    /// Method to genereate correct data for tests only, not safe
+    /// </summary>
+    public static void KzgifyBlob(ReadOnlySpan<byte> blob, Span<byte> commitment, Span<byte> proof, Span<byte> hashV1)
+    {
+        Ckzg.Ckzg.BlobToKzgCommitment(commitment, blob, _ckzgSetup);
+        Ckzg.Ckzg.ComputeBlobKzgProof(proof, blob, commitment, _ckzgSetup);
+        TryComputeCommitmentHashV1(commitment, hashV1);
+    }
 }

src/Nethermind/Nethermind.Evm.Test/PointEvaluationPrecompileTests.cs

@@ -24,7 +24,7 @@ public class PointEvaluationPrecompileTests
     private static readonly byte[] _predefinedFailureAnswer = Array.Empty<byte>();
 
     [OneTimeSetUp]
-    public Task OneTimeSetUp() => KzgPolynomialCommitments.Initialize();
+    public Task OneTimeSetUp() => KzgPolynomialCommitments.InitializeAsync();
 
     [TestCaseSource(nameof(OutputTests))]
     public bool Test_PointEvaluationPrecompile_Produces_Correct_Outputs(byte[] input)

src/Nethermind/Nethermind.Evm/Precompiles/PointEvaluationPrecompile.cs

@@ -43,7 +43,7 @@ static bool IsValid(in ReadOnlyMemory<byte> inputData)
             ReadOnlySpan<byte> proof = inputDataSpan[144..192];
             Span<byte> hash = stackalloc byte[32];
 
-            return KzgPolynomialCommitments.TryComputeCommitmentV1(commitment, hash)
+            return KzgPolynomialCommitments.TryComputeCommitmentHashV1(commitment, hash)
                    && hash.SequenceEqual(versionedHash)
                    && KzgPolynomialCommitments.VerifyProof(commitment, z, y, proof);
         }

src/Nethermind/Nethermind.Init/Steps/InitializePrecompiles.cs

@@ -27,7 +27,7 @@ public async Task Execute(CancellationToken cancellationToken)
 
             try
             {
-                await KzgPolynomialCommitments.Initialize(logger);
+                await KzgPolynomialCommitments.InitializeAsync(logger);
             }
             catch (Exception e)
             {

src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs

@@ -108,7 +108,7 @@ public interface IJsonRpcConfig : IConfig
 
         [ConfigItem(
             Description = "Defines method names of Json RPC service requests to NOT log. Example: {\"eth_blockNumber\"} will not log \"eth_blockNumber\" requests.",
-            DefaultValue = "[engine_newPayloadV1, engine_newPayloadV2, engine_forkchoiceUpdatedV1, engine_forkchoiceUpdatedV2]")]
+            DefaultValue = "[engine_newPayloadV1, engine_newPayloadV2, engine_newPayloadV3, engine_forkchoiceUpdatedV1, engine_forkchoiceUpdatedV2]")]
         public string[]? MethodsLoggingFiltering { get; set; }
 
         [ConfigItem(

src/Nethermind/Nethermind.JsonRpc/JsonRpcConfig.cs

@@ -44,6 +44,7 @@ public int WebSocketsPort
         {
             "engine_newPayloadV1",
             "engine_newPayloadV2",
+            "engine_newPayloadV3",
             "engine_forkchoiceUpdatedV1",
             "engine_forkchoiceUpdatedV2"
         };

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs

@@ -47,7 +47,7 @@ private void AssertExecutionStatusChanged(IBlockFinder blockFinder, Keccak headB
         }
 
         private Transaction[] BuildTransactions(MergeTestBlockchain chain, Keccak parentHash, PrivateKey from,
-            Address to, uint count, int value, out Account accountFrom, out BlockHeader parentHeader)
+            Address to, uint count, int value, out Account accountFrom, out BlockHeader parentHeader, int blobCountPerTx = 0)
         {
             Transaction BuildTransaction(uint index, Account senderAccount) =>
                 Build.A.Transaction.WithNonce(senderAccount.Nonce + index)
@@ -57,8 +57,9 @@ Transaction BuildTransaction(uint index, Account senderAccount) =>
                     .WithGasPrice(1.GWei())
                     .WithChainId(chain.SpecProvider.ChainId)
                     .WithSenderAddress(from.Address)
-                    .SignedAndResolved(from)
-                    .TestObject;
+                    .WithShardBlobTxTypeAndFields(blobCountPerTx)
+                    .WithMaxFeePerGasIfSupports1559(1.GWei())
+                    .SignedAndResolved(from).TestObject;
 
             parentHeader = chain.BlockTree.FindHeader(parentHash, BlockTreeLookupOptions.None)!;
             Account account = chain.StateReader.GetAccount(parentHeader.StateRoot!, from.Address)!;
@@ -78,11 +79,12 @@ private ExecutionPayload CreateParentBlockRequestOnHead(IBlockTree blockTree)
                 StateRoot = head.StateRoot!,
                 ReceiptsRoot = head.ReceiptsRoot!,
                 GasLimit = head.GasLimit,
-                Timestamp = head.Timestamp
+                Timestamp = head.Timestamp,
+                BaseFeePerGas = head.BaseFeePerGas,
             };
         }
 
-        private static ExecutionPayload CreateBlockRequest(ExecutionPayload parent, Address miner, IList<Withdrawal>? withdrawals = null)
+        private static ExecutionPayload CreateBlockRequest(ExecutionPayload parent, Address miner, IList<Withdrawal>? withdrawals = null, UInt256? excessDataGas = null)
         {
             ExecutionPayload blockRequest = new()
             {
@@ -95,7 +97,8 @@ private static ExecutionPayload CreateBlockRequest(ExecutionPayload parent, Addr
                 ReceiptsRoot = Keccak.EmptyTreeHash,
                 LogsBloom = Bloom.Empty,
                 Timestamp = parent.Timestamp + 1,
-                Withdrawals = withdrawals
+                Withdrawals = withdrawals,
+                ExcessDataGas = excessDataGas,
             };
 
             blockRequest.SetTransactions(Array.Empty<Transaction>());

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs

@@ -20,6 +20,7 @@
 using Nethermind.Core.Specs;
 using Nethermind.Core.Test.Blockchain;
 using Nethermind.Core.Timers;
+using Nethermind.Crypto;
 using Nethermind.Db;
 using Nethermind.Evm.Tracing;
 using Nethermind.Facade.Eth;
@@ -34,21 +35,34 @@
 using Nethermind.Specs.Forks;
 using Nethermind.State;
 using NSubstitute;
+using NUnit.Framework;
 
 namespace Nethermind.Merge.Plugin.Test;
 
 public partial class EngineModuleTests
 {
-    protected virtual MergeTestBlockchain CreateBaseBlockChain(IMergeConfig? mergeConfig = null, IPayloadPreparationService? mockedPayloadService = null, ILogManager? logManager = null) =>
+    [SetUp]
+    public Task Setup()
+    {
+        return KzgPolynomialCommitments.InitializeAsync();
+    }
+
+    protected virtual MergeTestBlockchain CreateBaseBlockChain(IMergeConfig? mergeConfig = null,
+        IPayloadPreparationService? mockedPayloadService = null, ILogManager? logManager = null) =>
         new(mergeConfig, mockedPayloadService, logManager);
 
-    protected async Task<MergeTestBlockchain> CreateShanghaiBlockChain(IMergeConfig? mergeConfig = null, IPayloadPreparationService? mockedPayloadService = null)
+    protected async Task<MergeTestBlockchain> CreateShanghaiBlockChain(IMergeConfig? mergeConfig = null,
+        IPayloadPreparationService? mockedPayloadService = null)
         => await CreateBlockChain(mergeConfig, mockedPayloadService, Shanghai.Instance);
 
-    protected async Task<MergeTestBlockchain> CreateBlockChain(IMergeConfig? mergeConfig = null, IPayloadPreparationService? mockedPayloadService = null, IReleaseSpec? releaseSpec = null)
-        => await CreateBaseBlockChain(mergeConfig, mockedPayloadService).Build(new TestSingleReleaseSpecProvider(releaseSpec ?? London.Instance));
 
-    protected async Task<MergeTestBlockchain> CreateBlockChain(ISpecProvider specProvider, ILogManager? logManager = null)
+    protected async Task<MergeTestBlockchain> CreateBlockChain(IMergeConfig? mergeConfig = null,
+        IPayloadPreparationService? mockedPayloadService = null, IReleaseSpec? releaseSpec = null)
+        => await CreateBaseBlockChain(mergeConfig, mockedPayloadService)
+            .Build(new TestSingleReleaseSpecProvider(releaseSpec ?? London.Instance));
+
+    protected async Task<MergeTestBlockchain> CreateBlockChain(ISpecProvider specProvider,
+        ILogManager? logManager = null)
         => await CreateBaseBlockChain(null, null, logManager).Build(specProvider);
 
     private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConfig? syncConfig = null, TimeSpan? newPayloadTimeout = null, int newPayloadCacheSize = 50)
@@ -72,6 +86,9 @@ private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConf
             new GetPayloadV2Handler(
                 chain.PayloadPreparationService!,
                 chain.LogManager),
+            new GetPayloadV3Handler(
+                chain.PayloadPreparationService!,
+                chain.LogManager),
             new NewPayloadHandler(
                 chain.BlockValidator,
                 chain.BlockTree,