Skip to content

Commit

Permalink
Expose methods that accept a cached stream reader
Browse files Browse the repository at this point in the history
This allows to use call match the same filter against a different set of elements without requiring to re-read the filter data, which it really expensive.
  • Loading branch information
lontivero committed Sep 15, 2023
1 parent 7f02f91 commit 4dd32ae
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 15 deletions.
6 changes: 5 additions & 1 deletion NBitcoin.Tests/Bip158_tests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,13 @@ public void BuildFilterAndMatchValues()
.Build();

var testKey = key.ToBytes().SafeSubarray(0, 16);
var reader = filter.GetGRStreamReader();

// The filter should match all the values that were added.
foreach (var name in names)
{
Assert.True(filter.Match(name, testKey));
Assert.True(filter.Match(name, testKey, reader));
}

// The filter should NOT match any extra value.
Expand Down Expand Up @@ -445,9 +448,10 @@ public void RealScriptPubKeyFilterTest()
var filter = builder.Build();

var keyMatch = key.ToBytes().SafeSubarray(0, 16);
var reader = filter.GetGRStreamReader();
foreach (var script in scripts)
{
var match = filter.MatchAny(new[] { script.ToBytes() }, keyMatch);
var match = filter.Match(script.ToBytes(), keyMatch, reader);
Assert.True(match);
}
}
Expand Down
43 changes: 38 additions & 5 deletions NBitcoin/BIP158/BitStream.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;

namespace NBitcoin
{
Expand Down Expand Up @@ -170,7 +170,6 @@ private void EnsureCapacity()
}
}


internal class GRCodedStreamWriter
{
private readonly BitStream _stream;
Expand Down Expand Up @@ -204,22 +203,54 @@ public void Write(ulong value)
}
}

internal class GRCodedStreamReader
public class CachedGRCodedStreamReader : GRCodedStreamReader
{
private List<ulong> _cachedValues;
private int _position;

internal CachedGRCodedStreamReader(BitStream stream, byte p, ulong lastValue) : base(stream, p, lastValue)
{
_cachedValues = new List<ulong>(10_000);
_position = 0;
}

public override bool TryRead(out ulong value)
{
if (_position >= _cachedValues.Count)
{
if (!base.TryRead(out var readedValue))
{
value = 0;
return false;
}
_cachedValues.Add(readedValue);
}
value = _cachedValues[_position++];
return true;
}

internal override void ResetPosition()
{
_position = 0;
}
}

public class GRCodedStreamReader
{
private readonly BitStream _stream;
private readonly byte _p;
private readonly ulong _modP;
private ulong _lastValue;

public GRCodedStreamReader(BitStream stream, byte p, ulong lastValue)
internal GRCodedStreamReader(BitStream stream, byte p, ulong lastValue)
{
_stream = stream;
_p = p;
_modP = (1UL << p);
_lastValue = lastValue;
}

public bool TryRead(out ulong value)
public virtual bool TryRead(out ulong value)
{
if (TryReadUInt64(out var readedValue))
{
Expand All @@ -233,6 +264,8 @@ public bool TryRead(out ulong value)
return false;
}

internal virtual void ResetPosition () {}

private bool TryReadUInt64(out ulong value)
{
value = 0U;
Expand Down
63 changes: 54 additions & 9 deletions NBitcoin/BIP158/GolombRiceFilter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,23 @@ public uint256 GetHeader(uint256 previousHeader)
/// <param name="key">Key used for hashing the data elements.</param>
/// <returns>true if the element is in the filter, otherwise false.</returns>
public bool Match(byte[] data, byte[] key)
{
var reader = new GRCodedStreamReader(new BitStream(Data), P, 0);
return Match(data, key, reader);
}

/// <summary>
/// Checks if the value passed is in the filter.
/// </summary>
/// <param name="data">Data element to check in the filter.</param>
/// <param name="key">Key used for hashing the data elements.</param>
/// <param name="reader">Golomb-Rice stream reader.</param>
/// <returns>true if the element is in the filter, otherwise false.</returns>
public bool Match(byte[] data, byte[] key, GRCodedStreamReader reader)
{
if (data == null)
throw new ArgumentNullException(nameof(data));
return MatchAny(new[] { data }, 1, key);
return MatchAny(new[] { data }, 1, key, reader);
}

/// <summary>
Expand All @@ -170,7 +183,8 @@ public bool Match(byte[] data, byte[] key)
/// <returns>true if at least one of the elements is in the filter, otherwise false.</returns>
public bool MatchAny(byte[][] data, byte[] key)
{
return MatchAny(data, data.Length, key);
var reader = new GRCodedStreamReader(new BitStream(Data), P, 0);
return MatchAny(data, data.Length, key, reader);
}

/// <summary>
Expand All @@ -180,20 +194,33 @@ public bool MatchAny(byte[][] data, byte[] key)
/// <param name="key">Key used for hashing the data elements.</param>
/// <returns>true if at least one of the elements is in the filter, otherwise false.</returns>
public bool MatchAny(IEnumerable<byte[]> data, byte[] key)
{
var reader = new GRCodedStreamReader(new BitStream(Data), P, 0);
return MatchAny(data, key, reader);
}

/// <summary>
/// Checks if any of the provided elements is in the filter.
/// </summary>
/// <param name="data">Data elements to check in the filter.</param>
/// <param name="key">Key used for hashing the data elements.</param>
/// <param name="reader">Golomb-Rice stream reader.</param>
/// <returns>true if at least one of the elements is in the filter, otherwise false.</returns>
public bool MatchAny(IEnumerable<byte[]> data, byte[] key, GRCodedStreamReader reader)
{
if (data == null)
throw new ArgumentNullException(nameof(data));
if (data is byte[][] dataArray)
{
return MatchAny(dataArray, dataArray.Length, key);
return MatchAny(dataArray, dataArray.Length, key, reader);
}
else if (data is ICollection<byte[]> dataCollection)
{
return MatchAny(dataCollection, dataCollection.Count, key);
return MatchAny(dataCollection, dataCollection.Count, key, reader);
}
else
{
return MatchAny(data, data.Count(), key);
return MatchAny(data, data.Count(), key, reader);
}
}

Expand All @@ -203,7 +230,19 @@ public bool MatchAny(IEnumerable<byte[]> data, byte[] key)
/// <param name="data">Data elements to check in the filter.</param>
/// <param name="key">Key used for hashing the data elements.</param>
/// <returns>true if at least one of the elements is in the filter, otherwise false.</returns>
internal bool MatchAny(IEnumerable<byte[]> data, int dataCount, byte[] key)
internal bool MatchAny(IEnumerable<byte[]> data, int dataCount, byte[] key, GRCodedStreamReader reader)
{
try
{
return InternalMatchAny(data, dataCount, key, reader);
}
finally
{
reader.ResetPosition();
}
}

private bool InternalMatchAny(IEnumerable<byte[]> data, int dataCount, byte[] key, GRCodedStreamReader sr)
{
if (data == null || dataCount == 0)
throw new ArgumentException("data can not be null or empty array.", nameof(data));
Expand All @@ -212,9 +251,6 @@ internal bool MatchAny(IEnumerable<byte[]> data, int dataCount, byte[] key)

var hs = ConstructHashedSet(P, N, M, key, data, dataCount);

var bitStream = new BitStream(Data);
var sr = new GRCodedStreamReader(bitStream, P, 0);

while(sr.TryRead(out var val))
{
var dataIndex = 0;
Expand Down Expand Up @@ -255,6 +291,15 @@ public override string ToString()
return NBitcoin.DataEncoders.Encoders.Hex.EncodeData(ToBytes());
}

/// <summary>
/// Create a cached Golomb-Rice stream reader.
/// </summary>
/// <returns>A new cached Golomb-Rice stream reader instance</returns>
public CachedGRCodedStreamReader GetGRStreamReader()
{
return new CachedGRCodedStreamReader(new BitStream(Data), P, 0);
}

internal static ulong FastReduction(ulong value, ulong nhi, ulong nlo)
{
// First, we'll spit the item we need to reduce into its higher and lower bits.
Expand Down

0 comments on commit 4dd32ae

Please sign in to comment.