Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix stream decompression corner case failure #89

Merged
merged 1 commit into from
Jan 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions Snappier.Tests/SnappyStreamTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -151,5 +151,51 @@ public void CompressAndDecompressChunkStressTest(string filename)
Assert.Equal(originalBytes.Length, decompressed.Length);
Assert.Equal(originalBytes, decompressed.ToArray());
}

#if NET6_0_OR_GREATER

// Test case that we know was failing on decompress with the default 8192 byte chunk size
[Fact]
public void Known8192ByteChunkStressTest()
{
using Stream resource = typeof(SnappyStreamTests).Assembly.GetManifestResourceStream("Snappier.Tests.TestData.streamerrorsequence.txt")!;
byte[] originalBytes = ConvertFromHexStream(resource);

using var compressed = new MemoryStream();
using SnappyStream compressor = new(compressed, CompressionMode.Compress);

compressor.Write(originalBytes, 0, originalBytes.Length);
compressor.Flush();

compressed.Position = 0;

using SnappyStream decompressor = new(compressed, CompressionMode.Decompress);
using var decompressed = new MemoryStream();
decompressor.CopyTo(decompressed);

Assert.True(decompressed.GetBuffer().AsSpan(0, (int) decompressed.Length).SequenceEqual(originalBytes));
}

private static byte[] ConvertFromHexStream(Stream stream)
{
using var output = new MemoryStream();

using var textReader = new StreamReader(stream, Encoding.UTF8);

char[] buffer = new char[1024];

int charsRead = textReader.Read(buffer.AsSpan());
while (charsRead > 0)
{
byte[] bytes = Convert.FromHexString(buffer.AsSpan(0, charsRead));
output.Write(bytes.AsSpan());

charsRead = textReader.Read(buffer, 0, buffer.Length);
}

return output.ToArray();
}

#endif
}
}
1 change: 1 addition & 0 deletions Snappier.Tests/TestData/streamerrorsequence.txt

Large diffs are not rendered by default.

15 changes: 11 additions & 4 deletions Snappier/Internal/SnappyDecompressor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -228,15 +228,23 @@ internal void DecompressAllTags(ReadOnlySpan<byte> inputSpan)

(uint inputUsed, uint bytesWritten) =
DecompressTagFromScratch(ref input, ref inputEnd, ref op, ref buffer, ref bufferEnd, ref scratch);
op = ref Unsafe.Add(ref op, bytesWritten);

if (inputUsed == 0)
{
// There was insufficient data to read an entire tag. Some data was moved to scratch
// but short circuit for another pass when we have more data.
return;
}

if (_remainingLiteral > 0)
{
// The last tag was fully read by there is still literal content remaining that is
// not yet available. Make sure we update _lookbackPosition on exit.
goto exit;
}

input = ref Unsafe.Add(ref input, inputUsed);
op = ref Unsafe.Add(ref op, bytesWritten);
}

while (true)
Expand Down Expand Up @@ -348,6 +356,7 @@ internal void DecompressAllTags(ReadOnlySpan<byte> inputSpan)
}
}

exit:
// All input data is processed
_lookbackPosition = (int)Unsafe.ByteOffset(ref buffer, ref op);
}
Expand Down Expand Up @@ -391,10 +400,8 @@ internal void DecompressAllTags(ReadOnlySpan<byte> inputSpan)
{
Append(ref op, ref bufferEnd, in input, inputRemaining);
_remainingLiteral = (int) (literalLength - inputRemaining);
_lookbackPosition += (int)Unsafe.ByteOffset(ref buffer, ref op);

// Insufficient data in this case as well, trigger a short circuit
return (0, 0);
return (inputUsed + (uint) inputRemaining, (uint) inputRemaining);
}
else
{
Expand Down
20 changes: 13 additions & 7 deletions Snappier/Internal/SnappyStreamDecompressor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,19 @@ public int Decompress(Span<byte> buffer)
goto exit;
}

int availableChunkBytes = Math.Min(input.Length, _chunkSize - _chunkBytesProcessed);
Debug.Assert(availableChunkBytes > 0);

_decompressor.Decompress(input.Slice(0, availableChunkBytes));

_chunkBytesProcessed += availableChunkBytes;
input = input.Slice(availableChunkBytes);
int availableChunkBytes = _chunkSize - _chunkBytesProcessed;
if (availableChunkBytes > input.Length)
{
_decompressor.Decompress(input);
_chunkBytesProcessed += input.Length;
input = default;
}
else
{
_decompressor.Decompress(input.Slice(0, availableChunkBytes));
_chunkBytesProcessed += availableChunkBytes;
input = input.Slice(availableChunkBytes);
}
}

int decompressedBytes = _decompressor.Read(buffer);
Expand Down