Skip to content

Commit

Permalink
fix: Remove dependency on System.IO.Compression for .NET Framework bu…
Browse files Browse the repository at this point in the history
…ild, since this package can be consumed by applications with the out-of-band System.IO.Compression NuGet package

Details:
- Vendor in SharpZipLib v1.3.3 (MIT License)
- Remove AssemblyReference to System.IO.Compression
- Update Datadog.Trace.Logging.TracerFlare.DebugLogReader to use SharpZipLib instead of System.IO.Compression.ZipArchive
- Update Datadog.Trace.Trimming.xml Root Descriptors File
- Update snapshot tests that assert Datadog.Trace assembly references
  • Loading branch information
zacharycmontoya committed Oct 24, 2024
1 parent c0e5e4d commit 89f01c0
Show file tree
Hide file tree
Showing 84 changed files with 32,129 additions and 12 deletions.
1 change: 1 addition & 0 deletions LICENSE-3rdparty.csv
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ Datadog.Trace,https://github.com/neuecc/MessagePack-CSharp,MIT,Copyright (c) 201
Datadog.Trace,https://github.com/serilog/serilog,Apache-2.0,Copyright (c) 2013-2018 Serilog Contributors
Datadog.Trace,https://github.com/serilog/serilog-sinks-file,Apache-2.0,Copyright (c) 2016 Serilog Contributors
Datadog.Trace,https://github.com/JamesNK/Newtonsoft.Json,MIT,Copyright (c) 2007 James Newton-King
Datadog.Trace,https://github.com/icsharpcode/SharpZipLib,MIT,Copyright (c) 2000-2018 SharpZipLib Contributors
Datadog.Tracer.Native,https://github.com/dotnet/runtime,MIT,Copyright (c) .NET Foundation and contributors. All rights reserved.
Datadog.Tracer.Native,https://github.com/Microsoft/clr-samples,MIT,Copyright (c) .NET Foundation and contributors. All rights reserved.
Datadog.Tracer.Native,https://github.com/MicrosoftArchive/clrprofiler,MIT,Copyright (c) Microsoft Corporation. All rights reserved.
Expand Down
15 changes: 15 additions & 0 deletions tracer/build/_build/UpdateVendors/VendoredDependency.cs
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,13 @@ static VendoredDependency()
downloadUrl: "https://github.com/DataDog/dotnet-vendored-code/archive/refs/tags/1.0.0.zip",
pathToSrc: new[] { "dotnet-vendored-code-1.0.0", "System.Reflection.Metadata", "System.Runtime.CompilerServices.Unsafe" },
transform: filePath => RewriteCsFileWithStandardTransform(filePath, originalNamespace: "System.Runtime", AddNullableDirectiveTransform, AddIgnoreNullabilityWarningDisablePragma));

Add(
libraryName: "ICSharpCode.SharpZipLib",
version: "1.3.3",
downloadUrl: "https://github.com/icsharpcode/SharpZipLib/archive/refs/tags/v1.3.3.zip",
pathToSrc: new[] { "SharpZipLib-1.3.3", "src", "ICSharpCode.SharpZipLib" },
transform: filePath => RewriteCsFileWithStandardTransform(filePath, originalNamespace: "ICSharpCode.SharpZipLib"));
}

public static List<VendoredDependency> All { get; set; } = new List<VendoredDependency>();
Expand Down Expand Up @@ -277,6 +284,13 @@ private bool TryGetValue(string key, [NotNullWhen(true)]out JsonProperty? item)
builder.Replace("#if NETSTANDARD || NETFRAMEWORK\n [System.Runtime.CompilerServices.MethodImpl", "#if NETSTANDARD || NETFRAMEWORK || NETCOREAPP\n [System.Runtime.CompilerServices.MethodImpl");
}
// Special SharpZipLib processing
if (originalNamespace.StartsWith("ICSharpCode"))
{
builder.Replace("#if NET45", "#if NETFRAMEWORK");
builder.Replace("using static ICSharpCode.SharpZipLib", "using static Datadog.Trace.Vendors.ICSharpCode.SharpZipLib");
}
// Debugger.Break() is a dangerous method that may crash the process. We don't
// want to take any risk of calling it, ever, so replace it with a noop.
builder.Replace("Debugger.Break();", "{}");
Expand Down Expand Up @@ -321,6 +335,7 @@ static string GenerateWarningDisablePragma() =>
"CS1573, " + // Parameter 'x' has no matching param tag in the XML comment for 'y' (but other parameters do)
"CS8018, " + // Within cref attributes, nested types of generic types should be qualified
"SYSLIB0011, " + // BinaryFormatter serialization is obsolete and should not be used.
"SYSLIB0023, " + // RNGCryptoServiceProvider is obsolete. To generate a random number, use one of the RandomNumberGenerator static methods instead.
"SYSLIB0032"; // Recovery from corrupted process state exceptions is not supported; HandleProcessCorruptedStateExceptionsAttribute is ignored."

static string AddIgnoreNullabilityWarningDisablePragma(string filePath, string content) =>
Expand Down
5 changes: 4 additions & 1 deletion tracer/dependabot/Datadog.Dependabot.Vendors.csproj
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk.Web">

<PropertyGroup>
<TargetFrameworks>netcoreapp3.1</TargetFrameworks>
<TargetFrameworks>net8.0</TargetFrameworks>
<RootNamespace>Datadog.Dependabot.Honeypot</RootNamespace>
</PropertyGroup>

Expand Down Expand Up @@ -39,6 +39,9 @@
<!-- https://www.nuget.org/packages/System.Reflection.Metadata/1.0.0 -->
<PackageReference Include="System.Reflection.Metadata" Version="7.0.2" />

<!-- https://www.nuget.org/packages/ICSharpCode.SharpZipLib/1.3.3 -->
<PackageReference Include="ICSharpCode.SharpZipLib" Version="1.3.3" />

</ItemGroup>

</Project>
24 changes: 21 additions & 3 deletions tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml
Original file line number Diff line number Diff line change
Expand Up @@ -269,9 +269,6 @@
<type fullname="System.IO.Compression.CompressionMode" />
<type fullname="System.IO.Compression.DeflateStream" />
<type fullname="System.IO.Compression.GZipStream" />
<type fullname="System.IO.Compression.ZipArchive" />
<type fullname="System.IO.Compression.ZipArchiveEntry" />
<type fullname="System.IO.Compression.ZipArchiveMode" />
</assembly>
<assembly fullname="System.IO.FileSystem">
<type fullname="System.IO.Directory" />
Expand Down Expand Up @@ -417,6 +414,9 @@
<assembly fullname="System.Net.WebProxy">
<type fullname="System.Net.WebProxy" />
</assembly>
<assembly fullname="System.Numerics.Vectors">
<type fullname="System.Numerics.Vector`1" />
</assembly>
<assembly fullname="System.ObjectModel">
<type fullname="System.Collections.ObjectModel.KeyedCollection`2" />
<type fullname="System.Collections.ObjectModel.ReadOnlyDictionary`2" />
Expand Down Expand Up @@ -613,6 +613,7 @@
<type fullname="System.IO.EndOfStreamException" />
<type fullname="System.IO.File" />
<type fullname="System.IO.FileAccess" />
<type fullname="System.IO.FileAttributes" />
<type fullname="System.IO.FileInfo" />
<type fullname="System.IO.FileMode" />
<type fullname="System.IO.FileNotFoundException" />
Expand All @@ -623,6 +624,7 @@
<type fullname="System.IO.IOException" />
<type fullname="System.IO.MemoryStream" />
<type fullname="System.IO.Path" />
<type fullname="System.IO.PathTooLongException" />
<type fullname="System.IO.SearchOption" />
<type fullname="System.IO.SeekOrigin" />
<type fullname="System.IO.Stream" />
Expand Down Expand Up @@ -785,6 +787,7 @@
<type fullname="System.RuntimeMethodHandle" />
<type fullname="System.RuntimeTypeHandle" />
<type fullname="System.SByte" />
<type fullname="System.Security.Cryptography.CryptographicException" />
<type fullname="System.Security.SecurityCriticalAttribute" />
<type fullname="System.Security.SecurityException" />
<type fullname="System.Security.SecurityRuleSet" />
Expand Down Expand Up @@ -958,8 +961,13 @@
</assembly>
<assembly fullname="System.Security.Cryptography" />
<assembly fullname="System.Security.Cryptography.Algorithms">
<type fullname="System.Security.Cryptography.Aes" />
<type fullname="System.Security.Cryptography.AsymmetricSignatureFormatter" />
<type fullname="System.Security.Cryptography.DeriveBytes" />
<type fullname="System.Security.Cryptography.IncrementalHash" />
<type fullname="System.Security.Cryptography.MD5" />
<type fullname="System.Security.Cryptography.RandomNumberGenerator" />
<type fullname="System.Security.Cryptography.Rfc2898DeriveBytes" />
<type fullname="System.Security.Cryptography.RSA" />
<type fullname="System.Security.Cryptography.RSAParameters" />
<type fullname="System.Security.Cryptography.RSAPKCS1SignatureFormatter" />
Expand All @@ -968,9 +976,19 @@
<type fullname="System.Security.Cryptography.SHA384" />
<type fullname="System.Security.Cryptography.SHA512" />
</assembly>
<assembly fullname="System.Security.Cryptography.Csp">
<type fullname="System.Security.Cryptography.RNGCryptoServiceProvider" />
</assembly>
<assembly fullname="System.Security.Cryptography.Primitives">
<type fullname="System.Security.Cryptography.AsymmetricAlgorithm" />
<type fullname="System.Security.Cryptography.CipherMode" />
<type fullname="System.Security.Cryptography.CryptoStream" />
<type fullname="System.Security.Cryptography.CryptoStreamMode" />
<type fullname="System.Security.Cryptography.HashAlgorithm" />
<type fullname="System.Security.Cryptography.HashAlgorithmName" />
<type fullname="System.Security.Cryptography.ICryptoTransform" />
<type fullname="System.Security.Cryptography.KeySizes" />
<type fullname="System.Security.Cryptography.SymmetricAlgorithm" />
</assembly>
<assembly fullname="System.Text.Encoding.Extensions">
<type fullname="System.Text.UnicodeEncoding" />
Expand Down
1 change: 0 additions & 1 deletion tracer/src/Datadog.Trace/Datadog.Trace.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
<ItemGroup Condition=" $(TargetFramework.StartsWith('net4')) ">
<Reference Include="System.Configuration" />
<Reference Include="System.Web" />
<Reference Include="System.IO.Compression" />
</ItemGroup>

<ItemGroup Condition=" !$(TargetFramework.StartsWith('net4')) ">
Expand Down
29 changes: 23 additions & 6 deletions tracer/src/Datadog.Trace/Logging/TracerFlare/DebugLogReader.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// <copyright file="DebugLogReader.cs" company="Datadog">
// <copyright file="DebugLogReader.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>
Expand All @@ -12,6 +12,8 @@
using System.Threading;
using System.Threading.Tasks;
using Datadog.Trace.Util.Streams;
using Datadog.Trace.Vendors.ICSharpCode.SharpZipLib;
using Datadog.Trace.Vendors.ICSharpCode.SharpZipLib.Core;

namespace Datadog.Trace.Logging.TracerFlare;

Expand Down Expand Up @@ -65,7 +67,12 @@ public static async Task WriteDebugLogArchiveToStream(Stream writeTo, string log
try
{
using var monitoringStream = new WriteCountingStream(writeTo);
using var archive = new ZipArchive(monitoringStream, ZipArchiveMode.Create, true);
using var archive = new Vendors.ICSharpCode.SharpZipLib.Zip.ZipOutputStream(monitoringStream);
archive.IsStreamOwner = false; // Do not close the underlying stream when closing the archive
archive.SetLevel(9); // Set to optimal compression

// Create a fixed-size buffer to bound memory operations
byte[] buffer = new byte[4096];

// Only sending .log files to avoid the risk of sending files we don't want.
// Also not recurrsing, in-case they have a weird log setup
Expand Down Expand Up @@ -121,13 +128,23 @@ public static async Task WriteDebugLogArchiveToStream(Stream writeTo, string log
{
// There's a SmallestSize for .NET 5+ but it doesn't give much benefit
// and we would rather not add the extra memory pressure for little gain
var entry = archive.CreateEntry(fileDetails.Name, CompressionLevel.Optimal);
using var entryStream = entry.Open();
var entry = new Vendors.ICSharpCode.SharpZipLib.Zip.ZipEntry(Path.GetFileName(fileDetails.FullName));
// archive.PutNextEntry(entry);
archive.PutNextEntry(entry);

// Have to allow FileShare.ReadWrite as the logger will already have it open for writing
using var file = File.Open(fileDetails.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
// StreamUtils.Copy(file, archive, buffer);

await file.CopyToAsync(entryStream).ConfigureAwait(false);
await entryStream.FlushAsync().ConfigureAwait(false);
// Using a fixed size buffer here makes no noticeable difference for output
// but keeps a lid on memory usage.
int sourceBytes;
do
{
sourceBytes = await file.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false);
await archive.WriteAsync(buffer, 0, sourceBytes).ConfigureAwait(false);
}
while (sourceBytes > 0);
}
catch (Exception ex)
{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
//------------------------------------------------------------------------------
// <auto-generated />
// This file was automatically generated by the UpdateVendors tool.
//------------------------------------------------------------------------------
#pragma warning disable CS0618, CS0649, CS1574, CS1580, CS1581, CS1584, CS1591, CS1573, CS8018, SYSLIB0011, SYSLIB0023, SYSLIB0032
using System;
using System.IO;

namespace Datadog.Trace.Vendors.ICSharpCode.SharpZipLib.BZip2
{
/// <summary>
/// An example class to demonstrate compression and decompression of BZip2 streams.
/// </summary>
internal static class BZip2
{
/// <summary>
/// Decompress the <paramref name="inStream">input</paramref> writing
/// uncompressed data to the <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The readable stream containing data to decompress.</param>
/// <param name="outStream">The output stream to receive the decompressed data.</param>
/// <param name="isStreamOwner">Both streams are closed on completion if true.</param>
public static void Decompress(Stream inStream, Stream outStream, bool isStreamOwner)
{
if (inStream == null)
throw new ArgumentNullException(nameof(inStream));

if (outStream == null)
throw new ArgumentNullException(nameof(outStream));

try
{
using (BZip2InputStream bzipInput = new BZip2InputStream(inStream))
{
bzipInput.IsStreamOwner = isStreamOwner;
Core.StreamUtils.Copy(bzipInput, outStream, new byte[4096]);
}
}
finally
{
if (isStreamOwner)
{
// inStream is closed by the BZip2InputStream if stream owner
outStream.Dispose();
}
}
}

/// <summary>
/// Compress the <paramref name="inStream">input stream</paramref> sending
/// result data to <paramref name="outStream">output stream</paramref>
/// </summary>
/// <param name="inStream">The readable stream to compress.</param>
/// <param name="outStream">The output stream to receive the compressed data.</param>
/// <param name="isStreamOwner">Both streams are closed on completion if true.</param>
/// <param name="level">Block size acts as compression level (1 to 9) with 1 giving
/// the lowest compression and 9 the highest.</param>
public static void Compress(Stream inStream, Stream outStream, bool isStreamOwner, int level)
{
if (inStream == null)
throw new ArgumentNullException(nameof(inStream));

if (outStream == null)
throw new ArgumentNullException(nameof(outStream));

try
{
using (BZip2OutputStream bzipOutput = new BZip2OutputStream(outStream, level))
{
bzipOutput.IsStreamOwner = isStreamOwner;
Core.StreamUtils.Copy(inStream, bzipOutput, new byte[4096]);
}
}
finally
{
if (isStreamOwner)
{
// outStream is closed by the BZip2OutputStream if stream owner
inStream.Dispose();
}
}
}
}
}
Loading

0 comments on commit 89f01c0

Please sign in to comment.