Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce shared store size on roll forward #2223

Merged
merged 7 commits into from
Feb 21, 2023
Merged

Reduce shared store size on roll forward #2223

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e10857f
Reduce shared store size on roll forward
pjanotti Feb 18, 2023
195850a
Fix BuildStructure.Distribution_Structure*.verified.txt
pjanotti Feb 18, 2023
bb6024f
Merge branch 'main' into optiomize-shared-store-roll-forward
pellared Feb 20, 2023
3508ec7
Merge branch 'main' into optiomize-shared-store-roll-forward
RassK Feb 20, 2023
d701bd4
Merge branch 'main' into optiomize-shared-store-roll-forward
pellared Feb 20, 2023
92fc8c1
Merge branch 'main' into optiomize-shared-store-roll-forward
RassK Feb 21, 2023
c2ed7cc
Merge branch 'main' into optiomize-shared-store-roll-forward
pjanotti Feb 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 27 additions & 38 deletions build/Build.Steps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,20 +454,21 @@ partial class Build

CopyNativeDependenciesToStore(file, jsonDocument, architectureStores);

RemoveDuplicatedLibraries(depsJsonContent, architectureStores);
depsJsonContent = RemoveOpenTelemetryAutoInstrumentationAdditionalDepsFromDepsContent(depsJsonContent);

RemoveOpenTelemetryAutoInstrumentationAdditionalDepsFromDepsFile(depsJsonContent, file);

// To allow roll forward for applications, like Roslyn, that target one tfm
// but have a later runtime make additional copies under the original tfm folder.
if (folderRuntimeName == TargetFramework.NET6_0)
{
AddFrameworkRollForwardCopy(TargetFramework.NET6_0, TargetFramework.NET7_0, architectureStores);
// To allow roll forward for applications, like Roslyn, that target one tfm
// but have a later runtime move the libraries under the original tfm folder
// to the latest one.
depsJsonContent = RollFrameworkForward(TargetFramework.NET6_0, TargetFramework.NET7_0, architectureStores, depsJsonContent);
}

// Write the updated deps.json file.
File.WriteAllText(file, depsJsonContent);
});
RemoveFilesFromAdditionalDepsDirectory();


void CopyNativeDependenciesToStore(AbsolutePath file, JsonDocument jsonDocument, IReadOnlyList<string> architectureStores)
{
var depsDirectory = file.Parent;
Expand Down Expand Up @@ -499,41 +500,17 @@ void CopyNativeDependenciesToStore(AbsolutePath file, JsonDocument jsonDocument,
}
}

void RemoveDuplicatedLibraries(string depsJsonContent, IReadOnlyList<string> architectureStores)
{
var duplicatedLibraries = new List<(string Name, string Version)> { };
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed this function because due to this line, it was never deleting anything.


foreach (var duplicatedLibrary in duplicatedLibraries)
{
if (depsJsonContent.Contains(duplicatedLibrary.Name.ToLower() + "/" + duplicatedLibrary.Version))
{
throw new NotSupportedException($"Cannot remove {duplicatedLibrary.Name.ToLower()}/{duplicatedLibrary.Version} folder. It is referenced in json file");
}

foreach (var architectureStore in architectureStores)
{
var directoryToBeRemoved = Path.Combine(architectureStore, duplicatedLibrary.Name.ToLower(), duplicatedLibrary.Version);

if (!Directory.Exists(directoryToBeRemoved))
{
throw new NotSupportedException($"Directory {directoryToBeRemoved} does not exists. Verify it.");
}

Directory.Delete(directoryToBeRemoved, true);
}
}
}

void RemoveOpenTelemetryAutoInstrumentationAdditionalDepsFromDepsFile(string depsJsonContent, AbsolutePath file)
string RemoveOpenTelemetryAutoInstrumentationAdditionalDepsFromDepsContent(string depsJsonContent)
{
// Remove OpenTelemetry.Instrumentation.AutoInstrumentationAdditionalDeps entry from target section.
depsJsonContent = Regex.Replace(depsJsonContent,
var updatedDepsJsonContent = Regex.Replace(depsJsonContent,
"\"OpenTelemetry(.+)AutoInstrumentation.AdditionalDeps.dll(.+?)}," + Environment.NewLine + "(.+?)\"", "\"",
RegexOptions.IgnoreCase | RegexOptions.Singleline);
// Remove OpenTelemetry.Instrumentation.AutoInstrumentationAdditionalDeps entry from library section and write to file.
depsJsonContent = Regex.Replace(depsJsonContent, "\"OpenTelemetry(.+?)}," + Environment.NewLine + "(.+?)\"", "\"",
updatedDepsJsonContent = Regex.Replace(updatedDepsJsonContent, "\"OpenTelemetry(.+?)}," + Environment.NewLine + "(.+?)\"", "\"",
RegexOptions.IgnoreCase | RegexOptions.Singleline);
File.WriteAllText(file, depsJsonContent);

return updatedDepsJsonContent;
}

void RemoveFilesFromAdditionalDepsDirectory()
Expand All @@ -542,8 +519,14 @@ void RemoveFilesFromAdditionalDepsDirectory()
AdditionalDepsDirectory.GlobDirectories("**/runtimes").ForEach(DeleteDirectory);
}

void AddFrameworkRollForwardCopy(string runtime, string rollForwardRuntime, IReadOnlyList<string> architectureStores)
string RollFrameworkForward(string runtime, string rollForwardRuntime, IReadOnlyList<string> architectureStores, string depsJsonContent)
{
// Update the contents of the json file.
var updatedDepsJsonContent = Regex.Replace(depsJsonContent,
$"\"lib/{runtime}/", $"\"lib/{rollForwardRuntime}/",
RegexOptions.IgnoreCase | RegexOptions.Singleline);

// Roll forward each architecture by renaming the tfm folder holding the assemblies.
foreach (var architectureStore in architectureStores)
{
var assemblyDirectories = Directory.GetDirectories(architectureStore);
Expand All @@ -561,11 +544,17 @@ void AddFrameworkRollForwardCopy(string runtime, string rollForwardRuntime, IRea
if (Directory.Exists(sourceDir))
{
var destDir = Path.Combine(assemblyVersionDirectory, "lib", rollForwardRuntime);
// Directory.CreateDirectory(destDir);

CopyDirectoryRecursively(sourceDir, destDir);

// Since the json was also rolled forward the original tfm folder can be deleted.
DeleteDirectory(sourceDir);
}
}
}

// Return the updated json contents.
return updatedDepsJsonContent;
}
});

Expand Down
24 changes: 0 additions & 24 deletions test/IntegrationTests/BuildTests.DistributionStructure_alpine-linux.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,27 +45,16 @@
/net/System.ServiceModel.Primitives.dll,
/net/System.ServiceModel.dll,
/store/x64/net6.0/dnsclient/1.4.0/lib/netstandard2.1/DnsClient.dll,
/store/x64/net6.0/microsoft.extensions.configuration.abstractions/7.0.0/lib/net6.0/Microsoft.Extensions.Configuration.Abstractions.dll,
/store/x64/net6.0/microsoft.extensions.configuration.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.Abstractions.dll,
/store/x64/net6.0/microsoft.extensions.configuration.binder/7.0.0/lib/net6.0/Microsoft.Extensions.Configuration.Binder.dll,
/store/x64/net6.0/microsoft.extensions.configuration.binder/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.Binder.dll,
/store/x64/net6.0/microsoft.extensions.configuration/7.0.0/lib/net6.0/Microsoft.Extensions.Configuration.dll,
/store/x64/net6.0/microsoft.extensions.configuration/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.dll,
/store/x64/net6.0/microsoft.extensions.dependencyinjection.abstractions/7.0.0/lib/net6.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll,
/store/x64/net6.0/microsoft.extensions.dependencyinjection.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll,
/store/x64/net6.0/microsoft.extensions.dependencyinjection/7.0.0/lib/net6.0/Microsoft.Extensions.DependencyInjection.dll,
/store/x64/net6.0/microsoft.extensions.dependencyinjection/7.0.0/lib/net7.0/Microsoft.Extensions.DependencyInjection.dll,
/store/x64/net6.0/microsoft.extensions.logging.abstractions/7.0.0/lib/net6.0/Microsoft.Extensions.Logging.Abstractions.dll,
/store/x64/net6.0/microsoft.extensions.logging.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.Logging.Abstractions.dll,
/store/x64/net6.0/microsoft.extensions.logging.configuration/7.0.0/lib/net6.0/Microsoft.Extensions.Logging.Configuration.dll,
/store/x64/net6.0/microsoft.extensions.logging.configuration/7.0.0/lib/net7.0/Microsoft.Extensions.Logging.Configuration.dll,
/store/x64/net6.0/microsoft.extensions.logging/7.0.0/lib/net6.0/Microsoft.Extensions.Logging.dll,
/store/x64/net6.0/microsoft.extensions.logging/7.0.0/lib/net7.0/Microsoft.Extensions.Logging.dll,
/store/x64/net6.0/microsoft.extensions.options.configurationextensions/7.0.0/lib/net6.0/Microsoft.Extensions.Options.ConfigurationExtensions.dll,
/store/x64/net6.0/microsoft.extensions.options.configurationextensions/7.0.0/lib/net7.0/Microsoft.Extensions.Options.ConfigurationExtensions.dll,
/store/x64/net6.0/microsoft.extensions.options/7.0.0/lib/net6.0/Microsoft.Extensions.Options.dll,
/store/x64/net6.0/microsoft.extensions.options/7.0.0/lib/net7.0/Microsoft.Extensions.Options.dll,
/store/x64/net6.0/microsoft.extensions.primitives/7.0.0/lib/net6.0/Microsoft.Extensions.Primitives.dll,
/store/x64/net6.0/microsoft.extensions.primitives/7.0.0/lib/net7.0/Microsoft.Extensions.Primitives.dll,
/store/x64/net6.0/mongodb.bson/2.13.3/lib/netstandard2.1/MongoDB.Bson.dll,
/store/x64/net6.0/mongodb.driver.core.extensions.diagnosticsources/1.3.0/lib/netstandard2.0/MongoDB.Driver.Core.Extensions.DiagnosticSources.dll,
Expand All @@ -78,7 +67,6 @@
/store/x64/net6.0/mongodb.libmongocrypt/1.2.2/runtimes/osx/native/libmongocrypt.dylib,
/store/x64/net6.0/mongodb.libmongocrypt/1.2.2/runtimes/win/native/mongocrypt.dll,
/store/x64/net6.0/sharpcompress/0.23.0/lib/netstandard2.0/SharpCompress.dll,
/store/x64/net6.0/system.diagnostics.diagnosticsource/7.0.0/lib/net6.0/System.Diagnostics.DiagnosticSource.dll,
/store/x64/net6.0/system.diagnostics.diagnosticsource/7.0.0/lib/net7.0/System.Diagnostics.DiagnosticSource.dll,
/store/x64/net7.0/dnsclient/1.4.0/lib/netstandard2.1/DnsClient.dll,
/store/x64/net7.0/microsoft.extensions.configuration.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.Abstractions.dll,
Expand All @@ -104,27 +92,16 @@
/store/x64/net7.0/mongodb.libmongocrypt/1.2.2/runtimes/win/native/mongocrypt.dll,
/store/x64/net7.0/sharpcompress/0.23.0/lib/netstandard2.0/SharpCompress.dll,
/store/x86/net6.0/dnsclient/1.4.0/lib/netstandard2.1/DnsClient.dll,
/store/x86/net6.0/microsoft.extensions.configuration.abstractions/7.0.0/lib/net6.0/Microsoft.Extensions.Configuration.Abstractions.dll,
/store/x86/net6.0/microsoft.extensions.configuration.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.Abstractions.dll,
/store/x86/net6.0/microsoft.extensions.configuration.binder/7.0.0/lib/net6.0/Microsoft.Extensions.Configuration.Binder.dll,
/store/x86/net6.0/microsoft.extensions.configuration.binder/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.Binder.dll,
/store/x86/net6.0/microsoft.extensions.configuration/7.0.0/lib/net6.0/Microsoft.Extensions.Configuration.dll,
/store/x86/net6.0/microsoft.extensions.configuration/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.dll,
/store/x86/net6.0/microsoft.extensions.dependencyinjection.abstractions/7.0.0/lib/net6.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll,
/store/x86/net6.0/microsoft.extensions.dependencyinjection.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll,
/store/x86/net6.0/microsoft.extensions.dependencyinjection/7.0.0/lib/net6.0/Microsoft.Extensions.DependencyInjection.dll,
/store/x86/net6.0/microsoft.extensions.dependencyinjection/7.0.0/lib/net7.0/Microsoft.Extensions.DependencyInjection.dll,
/store/x86/net6.0/microsoft.extensions.logging.abstractions/7.0.0/lib/net6.0/Microsoft.Extensions.Logging.Abstractions.dll,
/store/x86/net6.0/microsoft.extensions.logging.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.Logging.Abstractions.dll,
/store/x86/net6.0/microsoft.extensions.logging.configuration/7.0.0/lib/net6.0/Microsoft.Extensions.Logging.Configuration.dll,
/store/x86/net6.0/microsoft.extensions.logging.configuration/7.0.0/lib/net7.0/Microsoft.Extensions.Logging.Configuration.dll,
/store/x86/net6.0/microsoft.extensions.logging/7.0.0/lib/net6.0/Microsoft.Extensions.Logging.dll,
/store/x86/net6.0/microsoft.extensions.logging/7.0.0/lib/net7.0/Microsoft.Extensions.Logging.dll,
/store/x86/net6.0/microsoft.extensions.options.configurationextensions/7.0.0/lib/net6.0/Microsoft.Extensions.Options.ConfigurationExtensions.dll,
/store/x86/net6.0/microsoft.extensions.options.configurationextensions/7.0.0/lib/net7.0/Microsoft.Extensions.Options.ConfigurationExtensions.dll,
/store/x86/net6.0/microsoft.extensions.options/7.0.0/lib/net6.0/Microsoft.Extensions.Options.dll,
/store/x86/net6.0/microsoft.extensions.options/7.0.0/lib/net7.0/Microsoft.Extensions.Options.dll,
/store/x86/net6.0/microsoft.extensions.primitives/7.0.0/lib/net6.0/Microsoft.Extensions.Primitives.dll,
/store/x86/net6.0/microsoft.extensions.primitives/7.0.0/lib/net7.0/Microsoft.Extensions.Primitives.dll,
/store/x86/net6.0/mongodb.bson/2.13.3/lib/netstandard2.1/MongoDB.Bson.dll,
/store/x86/net6.0/mongodb.driver.core.extensions.diagnosticsources/1.3.0/lib/netstandard2.0/MongoDB.Driver.Core.Extensions.DiagnosticSources.dll,
Expand All @@ -137,7 +114,6 @@
/store/x86/net6.0/mongodb.libmongocrypt/1.2.2/runtimes/osx/native/libmongocrypt.dylib,
/store/x86/net6.0/mongodb.libmongocrypt/1.2.2/runtimes/win/native/mongocrypt.dll,
/store/x86/net6.0/sharpcompress/0.23.0/lib/netstandard2.0/SharpCompress.dll,
/store/x86/net6.0/system.diagnostics.diagnosticsource/7.0.0/lib/net6.0/System.Diagnostics.DiagnosticSource.dll,
/store/x86/net6.0/system.diagnostics.diagnosticsource/7.0.0/lib/net7.0/System.Diagnostics.DiagnosticSource.dll,
/store/x86/net7.0/dnsclient/1.4.0/lib/netstandard2.1/DnsClient.dll,
/store/x86/net7.0/microsoft.extensions.configuration.abstractions/7.0.0/lib/net7.0/Microsoft.Extensions.Configuration.Abstractions.dll,
Expand Down
Loading