From dea7eca2a946fb0e487107be3992445e30937562 Mon Sep 17 00:00:00 2001 From: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com> Date: Fri, 23 Dec 2022 14:11:54 +0100 Subject: [PATCH 1/3] chore: Replace SharpZipLib TarArchive helper class with TarOutputStream --- .github/workflows/cicd.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .../Clients/DockerImageOperations.cs | 4 +-- .../Clients/TestcontainersClient.cs | 16 +++++++----- .../Images/DockerfileArchive.cs | 26 +++++++++++-------- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 61c7d5952..51e6ea14b 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -13,7 +13,7 @@ on: type: boolean concurrency: - group: ${{ github.head_ref || github.sha }} + group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} cancel-in-progress: true env: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6afeaa8cf..4e79d162e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,7 +7,7 @@ on: branches: [ develop, master ] concurrency: - group: ${{ github.head_ref || github.sha }} + group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} cancel-in-progress: true jobs: diff --git a/src/Testcontainers/Clients/DockerImageOperations.cs b/src/Testcontainers/Clients/DockerImageOperations.cs index 4b8b9d230..6fb92d6aa 100644 --- a/src/Testcontainers/Clients/DockerImageOperations.cs +++ b/src/Testcontainers/Clients/DockerImageOperations.cs @@ -115,9 +115,9 @@ await this.DeleteAsync(image, ct) try { - using (var dockerfileStream = new FileStream(dockerfileArchiveFilePath, FileMode.Open, FileAccess.Read)) + using (var dockerfileArchiveStream = new FileStream(dockerfileArchiveFilePath, FileMode.Open, FileAccess.Read)) { - await this.Docker.Images.BuildImageFromDockerfileAsync(buildParameters, dockerfileStream, Array.Empty(), new Dictionary(), this.traceProgress, ct) + await this.Docker.Images.BuildImageFromDockerfileAsync(buildParameters, dockerfileArchiveStream, Array.Empty(), new Dictionary(), this.traceProgress, ct) .ConfigureAwait(false); var imageHasBeenCreated = await this.ExistsWithNameAsync(image.FullName, ct) diff --git a/src/Testcontainers/Clients/TestcontainersClient.cs b/src/Testcontainers/Clients/TestcontainersClient.cs index 28cad8a76..a6969da8d 100644 --- a/src/Testcontainers/Clients/TestcontainersClient.cs +++ b/src/Testcontainers/Clients/TestcontainersClient.cs @@ -186,9 +186,9 @@ public async Task CopyFileAsync(string id, string filePath, byte[] fileContent, IOperatingSystem os = new Unix(dockerEndpointAuthConfig: null); var containerPath = os.NormalizePath(filePath); - using (var memStream = new MemoryStream()) + using (var tarOutputMemStream = new MemoryStream()) { - using (var tarOutputStream = new TarOutputStream(memStream, Encoding.Default)) + using (var tarOutputStream = new TarOutputStream(tarOutputMemStream, Encoding.Default)) { tarOutputStream.IsStreamOwner = false; @@ -201,22 +201,24 @@ public async Task CopyFileAsync(string id, string filePath, byte[] fileContent, var entry = new TarEntry(header); - tarOutputStream.PutNextEntry(entry); + await tarOutputStream.PutNextEntryAsync(entry, ct) + .ConfigureAwait(false); #if NETSTANDARD2_1_OR_GREATER - await tarOutputStream.WriteAsync(fileContent.AsMemory(0, fileContent.Length), ct) + await tarOutputStream.WriteAsync(fileContent, ct) .ConfigureAwait(false); #else await tarOutputStream.WriteAsync(fileContent, 0, fileContent.Length, ct) .ConfigureAwait(false); #endif - tarOutputStream.CloseEntry(); + await tarOutputStream.CloseEntryAsync(ct) + .ConfigureAwait(false); } - memStream.Seek(0, SeekOrigin.Begin); + tarOutputMemStream.Seek(0, SeekOrigin.Begin); - await this.containers.ExtractArchiveToContainerAsync(id, Path.AltDirectorySeparatorChar.ToString(), memStream, ct) + await this.containers.ExtractArchiveToContainerAsync(id, Path.AltDirectorySeparatorChar.ToString(), tarOutputMemStream, ct) .ConfigureAwait(false); } } diff --git a/src/Testcontainers/Images/DockerfileArchive.cs b/src/Testcontainers/Images/DockerfileArchive.cs index f46d1e067..9a3ac2b1a 100644 --- a/src/Testcontainers/Images/DockerfileArchive.cs +++ b/src/Testcontainers/Images/DockerfileArchive.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; + using System.Text; using System.Text.RegularExpressions; using DotNet.Testcontainers.Configurations; using ICSharpCode.SharpZipLib.Tar; @@ -76,11 +77,11 @@ public string Tar() var dockerIgnoreFile = new DockerIgnoreFile(dockerfileDirectoryPath, ".dockerignore", dockerfileFilePath, this.logger); - using (var stream = new FileStream(dockerfileArchiveFilePath, FileMode.Create)) + using (var tarOutputFileStream = new FileStream(dockerfileArchiveFilePath, FileMode.Create, FileAccess.Write)) { - using (var tarArchive = TarArchive.CreateOutputTarArchive(stream)) + using (var tarOutputStream = new TarOutputStream(tarOutputFileStream, Encoding.Default)) { - tarArchive.RootPath = dockerfileDirectoryPath; + tarOutputStream.IsStreamOwner = false; foreach (var absoluteFilePath in GetFiles(dockerfileDirectoryPath)) { @@ -92,21 +93,24 @@ public string Tar() continue; } - // SharpZipLib's WriteEntry(TarEntry, bool) writes the entry, but without bytes if the file is used by another process. - // This results in a TarException on TarArchive.Dispose(): Entry closed at '0' before the 'x' bytes specified in the header were written: https://github.com/icsharpcode/SharpZipLib/issues/819. try { - var fileStream = File.Open(absoluteFilePath, FileMode.Open, FileAccess.Read, FileShare.Read); - fileStream.Dispose(); + using (var inputStream = new FileStream(absoluteFilePath, FileMode.Open, FileAccess.Read)) + { + var entry = TarEntry.CreateTarEntry(relativeFilePath); + entry.Size = inputStream.Length; + + tarOutputStream.PutNextEntry(entry); + + inputStream.CopyTo(tarOutputStream); + + tarOutputStream.CloseEntry(); + } } catch (IOException e) { throw new IOException("Cannot create Docker image tar archive.", e); } - - var tarEntry = TarEntry.CreateEntryFromFile(absoluteFilePath); - tarEntry.Name = relativeFilePath; - tarArchive.WriteEntry(tarEntry, false); } } } From e68c0cff377578a591467a226d8c640580db2905 Mon Sep 17 00:00:00 2001 From: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com> Date: Tue, 27 Dec 2022 08:46:32 +0100 Subject: [PATCH 2/3] refactor: Change ITarArchive.Tar() to async method --- src/Testcontainers/Clients/DockerImageOperations.cs | 3 ++- src/Testcontainers/Images/DockerfileArchive.cs | 13 +++++++++---- src/Testcontainers/Images/ITarArchive.cs | 8 ++++++-- .../Unit/Images/ImageFromDockerfileTest.cs | 11 +++++++---- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/Testcontainers/Clients/DockerImageOperations.cs b/src/Testcontainers/Clients/DockerImageOperations.cs index 6fb92d6aa..4efd381f1 100644 --- a/src/Testcontainers/Clients/DockerImageOperations.cs +++ b/src/Testcontainers/Clients/DockerImageOperations.cs @@ -111,7 +111,8 @@ await this.DeleteAsync(image, ct) Labels = configuration.Labels.ToDictionary(item => item.Key, item => item.Value), }; - var dockerfileArchiveFilePath = dockerfileArchive.Tar(); + var dockerfileArchiveFilePath = await dockerfileArchive.Tar(ct) + .ConfigureAwait(false); try { diff --git a/src/Testcontainers/Images/DockerfileArchive.cs b/src/Testcontainers/Images/DockerfileArchive.cs index 9a3ac2b1a..6f6bd2fd6 100644 --- a/src/Testcontainers/Images/DockerfileArchive.cs +++ b/src/Testcontainers/Images/DockerfileArchive.cs @@ -6,6 +6,8 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; using DotNet.Testcontainers.Configurations; using ICSharpCode.SharpZipLib.Tar; using Microsoft.Extensions.Logging; @@ -65,7 +67,7 @@ public DockerfileArchive(DirectoryInfo dockerfileDirectory, FileInfo dockerfile, } /// - public string Tar() + public async Task Tar(CancellationToken ct = default) { var dockerfileDirectoryPath = OS.NormalizePath(this.dockerfileDirectory.FullName); @@ -100,11 +102,14 @@ public string Tar() var entry = TarEntry.CreateTarEntry(relativeFilePath); entry.Size = inputStream.Length; - tarOutputStream.PutNextEntry(entry); + await tarOutputStream.PutNextEntryAsync(entry, ct) + .ConfigureAwait(false); - inputStream.CopyTo(tarOutputStream); + await inputStream.CopyToAsync(tarOutputStream, 4096, ct) + .ConfigureAwait(false); - tarOutputStream.CloseEntry(); + await tarOutputStream.CloseEntryAsync(ct) + .ConfigureAwait(false); } } catch (IOException e) diff --git a/src/Testcontainers/Images/ITarArchive.cs b/src/Testcontainers/Images/ITarArchive.cs index f069740b7..48bee6b87 100644 --- a/src/Testcontainers/Images/ITarArchive.cs +++ b/src/Testcontainers/Images/ITarArchive.cs @@ -1,5 +1,8 @@ namespace DotNet.Testcontainers.Images { + using System.Threading; + using System.Threading.Tasks; + /// /// Collects files into one tar archive file. /// @@ -8,7 +11,8 @@ internal interface ITarArchive /// /// Creates a tar archive file. /// - /// Path to the created archive file. - string Tar(); + /// Cancellation token. + /// Task that completes when the tar archive file has been created. + Task Tar(CancellationToken ct = default); } } diff --git a/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs b/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs index 863df8c63..1913f6c78 100644 --- a/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs +++ b/tests/Testcontainers.Tests/Unit/Images/ImageFromDockerfileTest.cs @@ -14,7 +14,7 @@ namespace DotNet.Testcontainers.Tests.Unit public sealed class ImageFromDockerfileTest { [Fact] - public void DockerfileArchiveTar() + public async Task DockerfileArchiveTar() { // Given var image = new DockerImage("testcontainers", "test", "0.1.0"); @@ -23,12 +23,15 @@ public void DockerfileArchiveTar() var actual = new SortedSet(); - var dockerFileArchive = new DockerfileArchive("Assets", "Dockerfile", image, NullLogger.Instance); + var dockerfileArchive = new DockerfileArchive("Assets", "Dockerfile", image, NullLogger.Instance); + + var dockerfileArchiveFilePath = await dockerfileArchive.Tar() + .ConfigureAwait(false); // When - using (var tarOut = new FileInfo(dockerFileArchive.Tar()).OpenRead()) + using (var tarOut = new FileStream(dockerfileArchiveFilePath, FileMode.Open, FileAccess.Read)) { - using (var tarIn = TarArchive.CreateInputTarArchive(tarOut, Encoding.UTF8)) + using (var tarIn = TarArchive.CreateInputTarArchive(tarOut, Encoding.Default)) { tarIn.ProgressMessageEvent += (_, entry, _) => actual.Add(entry.Name); tarIn.ListContents(); From c36980fc743320a72775fe666cd28b3a8afd987f Mon Sep 17 00:00:00 2001 From: Andre Hofmeister <9199345+HofmeisterAn@users.noreply.github.com> Date: Tue, 27 Dec 2022 10:21:56 +0100 Subject: [PATCH 3/3] refactor: Ignore IDE files --- src/Testcontainers/Images/DockerIgnoreFile.cs | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/Testcontainers/Images/DockerIgnoreFile.cs b/src/Testcontainers/Images/DockerIgnoreFile.cs index 5d3c2e87d..6f06d2db1 100644 --- a/src/Testcontainers/Images/DockerIgnoreFile.cs +++ b/src/Testcontainers/Images/DockerIgnoreFile.cs @@ -1,5 +1,6 @@ namespace DotNet.Testcontainers.Images { + using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -13,35 +14,43 @@ internal sealed class DockerIgnoreFile : IgnoreFile /// /// Initializes a new instance of the class. /// - /// Directory that contains all docker configuration files. - /// .dockerignore file name. + /// Directory that contains all docker configuration files. + /// .dockerignore file name. /// Dockerfile file name. /// The logger. - public DockerIgnoreFile(string dockerIgnoreFileDirectory, string dockerIgnoreFile, string dockerfileFile, ILogger logger) - : this(new DirectoryInfo(dockerIgnoreFileDirectory), dockerIgnoreFile, dockerfileFile, logger) + public DockerIgnoreFile(string dockerignoreFileDirectory, string dockerignoreFile, string dockerfileFile, ILogger logger) + : this(new DirectoryInfo(dockerignoreFileDirectory), dockerignoreFile, dockerfileFile, logger) { } /// /// Initializes a new instance of the class. /// - /// Directory that contains all docker configuration files. - /// .dockerignore file name. + /// Directory that contains all docker configuration files. + /// .dockerignore file name. /// Dockerfile file name. /// The logger. - public DockerIgnoreFile(DirectoryInfo dockerIgnoreFileDirectory, string dockerIgnoreFile, string dockerfileFile, ILogger logger) - : base(GetPatterns(dockerIgnoreFileDirectory, dockerIgnoreFile, dockerfileFile), logger) + public DockerIgnoreFile(DirectoryInfo dockerignoreFileDirectory, string dockerignoreFile, string dockerfileFile, ILogger logger) + : base(GetPatterns(dockerignoreFileDirectory, dockerignoreFile, dockerfileFile), logger) { } - private static IEnumerable GetPatterns(DirectoryInfo dockerIgnoreFileDirectory, string dockerIgnoreFile, string dockerfileFile) + private static IEnumerable GetPatterns(DirectoryInfo dockerignoreFileDirectory, string dockerignoreFile, string dockerfileFile) { + var dockerignoreFilePath = Path.Combine(dockerignoreFileDirectory.FullName, dockerignoreFile); + // These files are necessary and sent to the Docker daemon. The ADD and COPY instructions do not copy them to the image: // https://docs.docker.com/engine/reference/builder/#dockerignore-file. - var negateNecessaryFiles = new[] { dockerIgnoreFile, dockerfileFile }.Select(file => "!" + file); - return dockerIgnoreFileDirectory.GetFiles(dockerIgnoreFile, SearchOption.TopDirectoryOnly).Any() - ? File.ReadLines(Path.Combine(dockerIgnoreFileDirectory.FullName, dockerIgnoreFile)).Concat(negateNecessaryFiles) - : negateNecessaryFiles; + var negateNecessaryFiles = new[] { dockerignoreFile, dockerfileFile } + .Select(file => "!" + file); + + var dockerignorePatterns = File.Exists(dockerignoreFilePath) + ? File.ReadLines(dockerignoreFilePath) + : Array.Empty(); + + return new[] { "**/.idea", "**/.vs" } + .Concat(dockerignorePatterns) + .Concat(negateNecessaryFiles); } } }