Vectorize DataCost (#6953)

NethermindEth · Apr 26, 2024 · a0302b8 · a0302b8
1 parent 9ac4e45
commit a0302b8
Show file tree

Hide file tree

Showing 2 changed files with 70 additions and 11 deletions.
diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs
@@ -969,6 +969,63 @@ public static int CountLeadingZeros(this ReadOnlySpan<byte> bytes)
             return leadingZeros;
         }
 
+        public static int CountZeros(this Span<byte> data)
+            => CountZeros((ReadOnlySpan<byte>)data);
+
+        public static int CountZeros(this ReadOnlySpan<byte> data)
+        {
+            int totalZeros = 0;
+            if (Vector512.IsHardwareAccelerated && data.Length >= Vector512<byte>.Count)
+            {
+                ref byte bytes = ref MemoryMarshal.GetReference(data);
+                int i = 0;
+                for (; i < data.Length - Vector512<byte>.Count; i += Vector512<byte>.Count)
+                {
+                    Vector512<byte> dataVector = Unsafe.ReadUnaligned<Vector512<byte>>(ref Unsafe.Add(ref bytes, i));
+                    ulong flags = Vector512.Equals(dataVector, default).ExtractMostSignificantBits();
+                    totalZeros += BitOperations.PopCount(flags);
+                }
+
+                data = data[i..];
+            }
+            if (Vector256.IsHardwareAccelerated && data.Length >= Vector256<byte>.Count)
+            {
+                ref byte bytes = ref MemoryMarshal.GetReference(data);
+                int i = 0;
+                for (; i < data.Length - Vector256<byte>.Count; i += Vector256<byte>.Count)
+                {
+                    Vector256<byte> dataVector = Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.Add(ref bytes, i));
+                    uint flags = Vector256.Equals(dataVector, default).ExtractMostSignificantBits();
+                    totalZeros += BitOperations.PopCount(flags);
+                }
+
+                data = data[i..];
+            }
+            if (Vector128.IsHardwareAccelerated && data.Length >= Vector128<byte>.Count)
+            {
+                ref byte bytes = ref MemoryMarshal.GetReference(data);
+                int i = 0;
+                for (; i < data.Length - Vector128<byte>.Count; i += Vector128<byte>.Count)
+                {
+                    Vector128<byte> dataVector = Unsafe.ReadUnaligned<Vector128<byte>>(ref Unsafe.Add(ref MemoryMarshal.GetReference(data), i));
+                    uint flags = Vector128.Equals(dataVector, default).ExtractMostSignificantBits();
+                    totalZeros += BitOperations.PopCount(flags);
+                }
+
+                data = data[i..];
+            }
+
+            for (int i = 0; i < data.Length; i++)
+            {
+                if (data[i] == 0)
+                {
+                    totalZeros++;
+                }
+            }
+
+            return totalZeros;
+        }
+
         [DebuggerStepThrough]
         public static byte[] FromUtf8HexString(scoped ReadOnlySpan<byte> hexString)
         {

diff --git a/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs b/src/Nethermind/Nethermind.Evm/IntrinsicGasCalculator.cs
@@ -2,12 +2,16 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
-using System.Collections.Generic;
 using System.IO;
+using System.Numerics;
 using Nethermind.Core;
 using Nethermind.Core.Eip2930;
 using Nethermind.Core.Specs;
 using Nethermind.Int256;
+using System.Runtime.Intrinsics;
+using System.Runtime.InteropServices;
+using System.Runtime.CompilerServices;
+using Nethermind.Core.Extensions;
 
 namespace Nethermind.Evm;
 
@@ -37,19 +41,17 @@ private static long DataCost(Transaction transaction, IReleaseSpec releaseSpec)
     {
         long txDataNonZeroGasCost =
             releaseSpec.IsEip2028Enabled ? GasCostOf.TxDataNonZeroEip2028 : GasCostOf.TxDataNonZero;
-        long dataCost = 0;
         Span<byte> data = transaction.Data.GetValueOrDefault().Span;
-        for (int i = 0; i < data.Length; i++)
-        {
-            dataCost += data[i] == 0 ? GasCostOf.TxDataZero : txDataNonZeroGasCost;
-        }
 
-        if (transaction.IsContractCreation && releaseSpec.IsEip3860Enabled)
-        {
-            dataCost += EvmPooledMemory.Div32Ceiling((UInt256)data.Length) * GasCostOf.InitCodeWord;
-        }
+        int totalZeros = data.CountZeros();
+
+        var baseDataCost = (transaction.IsContractCreation && releaseSpec.IsEip3860Enabled
+            ? EvmPooledMemory.Div32Ceiling((UInt256)data.Length) * GasCostOf.InitCodeWord
+            : 0);
 
-        return dataCost;
+        return baseDataCost +
+            totalZeros * GasCostOf.TxDataZero +
+            (data.Length - totalZeros) * txDataNonZeroGasCost;
     }
 
     private static long AccessListCost(Transaction transaction, IReleaseSpec releaseSpec)