Skip to content

Commit

Permalink
Merge pull request #15 from ied206/xz-abort
Browse files Browse the repository at this point in the history 
Implement Abort() in XZStream and relatives
  • Loading branch information
@ied206
ied206 authored Aug 31, 2023
2 parents 61f9750 + d36875b commit 31200ad
Show file tree
Hide file tree
Showing 3 changed files with 238 additions and 19 deletions.
175 changes: 175 additions & 0 deletions Joveler.Compression.XZ.Tests/XZStreamsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
Expand Down Expand Up @@ -457,5 +458,179 @@ private static void AutoDecompressTemplate(string lzmaFileName, string originFil
Assert.IsTrue(decompDigest.SequenceEqual(originDigest));
}
#endregion

#region Abort Compress
[TestMethod]
public void AbortCompress()
{
AbortCompressTemplate("A.pdf", -1);
AbortCompressTemplate("B.txt", -1);
AbortCompressTemplate("C.bin", -1);

AbortCompressTemplate("A.pdf", 1);
AbortCompressTemplate("B.txt", 2);
AbortCompressTemplate("C.bin", 2);
}

private static void AbortCompressTemplate(string sampleFileName, int threads)
{
XZCompressOptions compOpts = new XZCompressOptions
{
Level = LzmaCompLevel.Default,
};

string sampleFile = Path.Combine(TestSetup.SampleDir, sampleFileName);

foreach (bool doAbort in new bool[] { false, true })
{
using (FileStream sampleFs = new FileStream(sampleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
using (MemoryStream rms = new MemoryStream())
{
XZStream xzs = null;
try
{
if (threads == -1)
{ // Single-thread compression
xzs = new XZStream(rms, compOpts);
}
else if (0 < threads)
{ // Multi-thread compression
XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions
{
Threads = threads,
};
xzs = new XZStream(rms, compOpts, threadOpts);
}
else
{
Assert.Fail($"threads [{threads}] is not a valid test value.");
}

sampleFs.CopyTo(xzs);

DateTime before = DateTime.Now;
if (doAbort)
xzs.Abort();
else
xzs.Close();
DateTime after = DateTime.Now;
TimeSpan abortElapsed = after - before;
Console.WriteLine($"{sampleFileName}, {threads} = {(doAbort ? "Abort" : "Close")}() took {abortElapsed.TotalMilliseconds:0.000}ms");

if (doAbort == false)
continue;

// Internal xz resources are now freed. Every compress operation will fail.
sampleFs.Position = 0;
bool hadThrown = false;
try
{
sampleFs.CopyTo(xzs);
}
catch (XZException e)
{
Assert.AreEqual(LzmaRet.ProgError, e.ReturnCode);
hadThrown = true;
}
Assert.IsTrue(hadThrown);
}
finally
{
xzs?.Dispose();
xzs = null;
}
}
}
}
#endregion

#region Abort Decompress
[TestMethod]
public void AbortDecompress()
{
AbortDecompressTemplate("A.xz", -1);
AbortDecompressTemplate("B9.xz", -1);
AbortDecompressTemplate("C.xz", -1);

AbortDecompressTemplate("A_mt16.xz", 1);
AbortDecompressTemplate("B1_mt16.xz", 2);
AbortDecompressTemplate("C.xz", 2);
}

private static void AbortDecompressTemplate(string sampleFileName, int threads)
{
string xzFile = Path.Combine(TestSetup.SampleDir, sampleFileName);

XZDecompressOptions decompOpts = new XZDecompressOptions();

foreach (bool doAbort in new bool[] { false, true })
{
XZStream xzs = null;
try
{
using (FileStream compFs = new FileStream(xzFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
if (threads == -1)
{ // Single-thread compression
xzs = new XZStream(compFs, decompOpts);
}
else if (0 < threads)
{ // Multi-thread compression
XZThreadedDecompressOptions threadOpts = new XZThreadedDecompressOptions
{
Threads = threads,
};
xzs = new XZStream(compFs, decompOpts, threadOpts);
}
else
{
Assert.Fail($"threads [{threads}] is not a valid test value.");
}

long firstReadLen = compFs.Length / 2;
byte[] firstBuffer = new byte[firstReadLen];
int bytesRead = xzs.Read(firstBuffer, 0, firstBuffer.Length);

DateTime before = DateTime.Now;
if (doAbort)
xzs.Abort();
else
xzs.Close();
DateTime after = DateTime.Now;
TimeSpan abortElapsed = after - before;
Console.WriteLine($"{sampleFileName}, {threads} = {(doAbort ? "Abort" : "Close")}() took {abortElapsed.TotalMilliseconds:0.000}ms");

if (doAbort == false)
continue;

// Internal xz resources are now freed. Every decompress operation will fail.
bool hadThrown = false;
try
{
byte[] buffer = new byte[64 * 1024];
do
{
bytesRead = xzs.Read(buffer, 0, buffer.Length);
} while (0 < bytesRead);
}
catch (XZException e)
{
Assert.AreEqual(LzmaRet.ProgError, e.ReturnCode);
hadThrown = true;
}
Assert.IsTrue(hadThrown);
}
}
finally
{
xzs?.Dispose();
xzs = null;
}
Assert.IsNull(xzs);
}


}
#endregion
}
}
62 changes: 53 additions & 9 deletions Joveler.Compression.XZ/XZStreams.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,8 @@ public class XZThreadedDecompressOptions
#region XZStreamBase
/// <inheritdoc />
/// <summary>
/// The stream to handle .xz file format.
/// The stream to handle xz-related file/stream format.
/// <para>This symbol can be changed anytime, consider this as not a part of public ABI!</para>
/// </summary>
public abstract class XZStreamBase : Stream
{
Expand Down Expand Up @@ -333,6 +334,8 @@ protected enum CoderFormat
private int _workBufPos = 0;
private readonly byte[] _workBuf;

private bool _isAborted = false;

// Property
public Stream BaseStream { get; private set; }
public long TotalIn { get; private set; } = 0;
Expand Down Expand Up @@ -556,17 +559,21 @@ protected override void Dispose(bool disposing)
{
if (_lzmaStream != null)
{
if (_mode == Mode.Compress)
if (_isAborted == false)
{
Flush();
FinishWrite();
}
else
{
_workBufPos = ReadDone;
if (_mode == Mode.Compress)
{
Flush();
FinishWrite();
}
else
{
_workBufPos = ReadDone;
}

FreeLzmaStream();
}

XZInit.Lib.LzmaEnd(_lzmaStream);
_lzmaStreamPin.Free();
_lzmaStream = null;
}
Expand All @@ -583,6 +590,39 @@ protected override void Dispose(bool disposing)
}
#endregion

#region LzmaStream management and Abort
private void FreeLzmaStream()
{
// lzma_end frees memory allocated for coder data structures.
// It must be called to avoid memory leak.
if (_lzmaStream != null)
{
XZInit.Lib.LzmaEnd(_lzmaStream);
}
}

/// <summary>
/// Immediately aborts the current operation.
/// Internal XZ resources will be freed without flushing nor finalizing.
/// <para>The instance will not be able to perform any operations except disposing.</para>
/// <para>Data written to the BaseStream will become invalid, dispose it immediately.</para>
/// </summary>
public void Abort()
{
// In compress mode, Abort() is faster than Close().
// In threaded compress mode, Abort() is much faster than Close().
// In decompress mode, Abort() took similar time as Close().

// Invalidate LzmaStream instance.
// After running this code, liblzma will refuse any operations via this LzmaStream object.
if (_isAborted)
return;

FreeLzmaStream();
_isAborted = true;
}
#endregion

#region Stream Methods and Properties
/// <inheritdoc />
public override int Read(byte[] buffer, int offset, int count)
Expand Down Expand Up @@ -859,6 +899,8 @@ public double CompressionRatio
}
#endregion



#region GetProgress
/// <summary>
/// Get progress information of XZ stream.
Expand Down Expand Up @@ -888,6 +930,8 @@ public void GetProgress(out ulong progressIn, out ulong progressOut)
}
#endregion



#region Memory Usage (Decompression Only) - DISABLED
// lzma_memusage() only works on per-thread basis.
// It would not help users to perceive how many memory cap would needed on multi-threaded decompression.
Expand Down
20 changes: 10 additions & 10 deletions Joveler.Compression.ZLib/ZLibStreams.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,12 @@ public sealed class ZLibDecompressOptions
}
#endregion

#region DeflateBaseStream
#region DeflateStreamBase
/// <summary>
/// The stream which compress or decompress deflate stream format.
/// <para>This class can be changed anytime, does not rely on this as public stable ABI!</para>
/// The stream which compress or decompress zlib-related stream format.
/// <para>This symbol can be changed anytime, consider this as not a part of public ABI!</para>
/// </summary>
public abstract class DeflateBaseStream : Stream
public abstract class DeflateStreamBase : Stream
{
#region enum Mode, Format
internal enum Mode
Expand Down Expand Up @@ -145,7 +145,7 @@ AMD Ryzen 5 3600 / .NET Core 3.1.13 / Windows 10.0.19042 x64 / zlib 1.2.11
/// <summary>
/// Create compressing DeflateStream.
/// </summary>
protected DeflateBaseStream(Stream baseStream, ZLibCompressOptions compOpts, Format format)
protected DeflateStreamBase(Stream baseStream, ZLibCompressOptions compOpts, Format format)
{
ZLibInit.Manager.EnsureLoaded();

Expand All @@ -167,7 +167,7 @@ protected DeflateBaseStream(Stream baseStream, ZLibCompressOptions compOpts, For
ZLibException.CheckReturnValue(ret, _zs);
}

protected DeflateBaseStream(Stream baseStream, ZLibDecompressOptions decompOpts, Format format)
protected DeflateStreamBase(Stream baseStream, ZLibDecompressOptions decompOpts, Format format)
{
ZLibInit.Manager.EnsureLoaded();

Expand Down Expand Up @@ -205,7 +205,7 @@ protected DeflateBaseStream(Stream baseStream, ZLibDecompressOptions decompOpts,
#endregion

#region Disposable Pattern
~DeflateBaseStream()
~DeflateStreamBase()
{
Dispose(false);
}
Expand Down Expand Up @@ -506,7 +506,7 @@ private static void CheckMemLevel(ZLibMemLevel memLevel)
/// <summary>
/// The stream which compress or decompress deflate stream format.
/// </summary>
public sealed class DeflateStream : DeflateBaseStream
public sealed class DeflateStream : DeflateStreamBase
{
/// <summary>
/// Create compressing DeflateStream.
Expand All @@ -527,7 +527,7 @@ public DeflateStream(Stream baseStream, ZLibDecompressOptions decompOpts)
/// <summary>
/// The stream which compress or decompress zlib stream format.
/// </summary>
public sealed class ZLibStream : DeflateBaseStream
public sealed class ZLibStream : DeflateStreamBase
{
/// <summary>
/// Create compressing ZLibStream.
Expand All @@ -548,7 +548,7 @@ public ZLibStream(Stream baseStream, ZLibDecompressOptions decompOpts)
/// /// <summary>
/// The stream which compress or decompress gzip stream format.
/// </summary>
public sealed class GZipStream : DeflateBaseStream
public sealed class GZipStream : DeflateStreamBase
{
/// <summary>
/// Create compressing GZipStream.
Expand Down

0 comments on commit 31200ad

Please sign in to comment.