From 7a65e2b71dcf521b87d80cc74bbb3106b27fc9aa Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Nov 2023 15:59:03 -0500 Subject: [PATCH 1/7] [wasi] EmitBuildBase: Output an item for the bundle file also. This is correct as it represents all the outputs --- .../EmitBundleTask/EmitBundleBase.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 4378dc4046140..7db895010bb2b 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -188,8 +188,6 @@ public override bool Execute() bundledResource.SetMetadata("DataLenSymbolValue", symbolDataLen[resourceDataSymbol].ToString()); } - BundledResources = bundledResources.ToArray(); - if (!string.IsNullOrEmpty(BundleFile)) { string resourceSymbols = GatherUniqueExportedResourceDataSymbols(bundledResources); @@ -211,14 +209,22 @@ public override bool Execute() Log.LogMessage(MessageImportance.Low, $"Bundling {files.Count} files for {BundleRegistrationFunctionName}"); + string bundleFilePath = Path.Combine(OutputDirectory, BundleFile); + // Generate source file to preallocate resources and register bundled resources - EmitBundleFile(Path.Combine(OutputDirectory, BundleFile), (outputStream) => + EmitBundleFile(bundleFilePath, (outputStream) => { using var outputUtf8Writer = new StreamWriter(outputStream, Utf8NoBom); GenerateBundledResourcePreallocationAndRegistration(resourceSymbols, BundleRegistrationFunctionName, files, outputUtf8Writer); }); + + TaskItem itemForBundleFile = new TaskItem(Path.Combine(OutputDirectory, BundleFile)); + itemForBundleFile.SetMetadata("DestinationFile", bundleFilePath); + bundledResources.Add(itemForBundleFile); } + BundledResources = bundledResources.ToArray(); + return !Log.HasLoggedErrors; } From a61bb372d113f7d657dc39d2bcf48ffbbffb5e63 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Nov 2023 16:00:02 -0500 Subject: [PATCH 2/7] [wasi] WasiApp.Native.targets: Simplify target for bundling resources in single file bundle, and remove special cased way of handling bundlefiles --- src/mono/wasi/build/WasiApp.Native.targets | 89 +++++++--------------- 1 file changed, 29 insertions(+), 60 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index c4b19a32d579d..c45b8e00dc8f9 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -15,7 +15,7 @@ _GetNativeFilesForLinking; _GenerateManagedToNative; _WasmCompileNativeFiles; - _GenerateAssemblyObjectFiles; + _GenerateObjectFilesForSingleFileBundle; _WasiLinkDotNet; @@ -359,74 +359,43 @@ - - - - <_WasmAssembliesBundleObjectFile>wasi_bundled_assemblies.o - <_WasmIcuBundleObjectFile>wasi_bundled_icu.o - <_WasmRuntimeConfigBundleFile>wasi_runtime_config_bin.o - - - - - - - - <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmAssembliesBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="%(_WasmBundledAssemblies.DestinationFile)" /> - - - + - - - - - - - - - - - <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmIcuBundleObjectFile)" /> - <_WasiObjectFilesForBundle Include="%(BundledWasmIcu.DestinationFile)" /> - - - + <_EmitBundleItemName + Include="_WasmAssembliesInternal" + BundleRegistrationFunctionName="mono_register_assemblies_bundle" + BundleFile="wasi_bundled_assemblies.o" /> + + <_IcuFilesToBundle Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)icudt.dat" /> + <_EmitBundleItemName + Include="_IcuFilesToBundle" + BundleRegistrationFunctionName="mono_register_icu_bundle" + BundleFile="wasi_bundled_icu.o" /> + + <_RuntimeConfigBinToBundle Include="$(_ParsedRuntimeConfigFilePath)" /> + <_EmitBundleItemName + Include="_RuntimeConfigBinToBundle" + BundleRegistrationFunctionName="mono_register_runtimeconfig_bin" + BundleFile="wasi_bundled_runtimeconfig_bin.o" /> - + - + FilesToBundle="@(%(_EmitBundleItemName.Identity))" + ClangExecutable="$(WasiClang)" + BundleRegistrationFunctionName="@(_EmitBundleItemName->'%(BundleRegistrationFunctionName)')" + BundleFile="@(_EmitBundleItemName->'%(BundleFile)')" + OutputDirectory="$(_WasmIntermediateOutputPath)"> + - <_WasiObjectFilesForBundle Include="$(_WasmIntermediateOutputPath)$(_WasmRuntimeConfigBundleFile)" /> - <_WasiObjectFilesForBundle Include="%(_BundledRuntimeConfigBin.DestinationFile)" /> + <_EmitBundleOutputFile Remove="@(_EmitBundleOutputFile)" /> + <_EmitBundleOutputFile Include="%(_EmitBundleOutputItem.DestinationFile)" /> - - + <_WasiObjectFilesForBundle Include="@(_EmitBundleOutputFile)" /> + - From c52ea55f28718b92d58aafe7b19123c64858bcf7 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Nov 2023 16:00:58 -0500 Subject: [PATCH 3/7] [wasi] EmitBundleBase: ReleaseCores once the execution is done, as msbuild does not seem to release them between batched task invocations --- .../EmitBundleTask/EmitBundleBase.cs | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 7db895010bb2b..40d8d2d61d23e 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -150,34 +150,43 @@ public override bool Execute() // Generate source file(s) containing each resource's byte data and size int allowedParallelism = Math.Max(Math.Min(bundledResources.Count, Environment.ProcessorCount), 1); - if (BuildEngine is IBuildEngine9 be9) + IBuildEngine9? be9 = BuildEngine as IBuildEngine9; + if (be9 is not null) allowedParallelism = be9.RequestCores(allowedParallelism); - Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => + try { - var group = remainingDestinationFilesToBundle[i]; + Parallel.For(0, remainingDestinationFilesToBundle.Length, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism, CancellationToken = BuildTaskCancelled.Token }, (i, state) => + { + var group = remainingDestinationFilesToBundle[i]; - var contentSourceFile = group.First(); + var contentSourceFile = group.First(); - var inputFile = contentSourceFile.ItemSpec; - var destinationFile = contentSourceFile.GetMetadata("DestinationFile"); - var registeredName = contentSourceFile.GetMetadata(RegisteredName); + var inputFile = contentSourceFile.ItemSpec; + var destinationFile = contentSourceFile.GetMetadata("DestinationFile"); + var registeredName = contentSourceFile.GetMetadata(RegisteredName); - var count = Interlocked.Increment(ref verboseCount); - Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); + var count = Interlocked.Increment(ref verboseCount); + Log.LogMessage(MessageImportance.Low, "{0}/{1} Bundling {2} ...", count, remainingDestinationFilesToBundle.Length, registeredName); - Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, destinationFile); - var symbolName = _resourceDataSymbolDictionary[registeredName]; - if (!EmitBundleFile(destinationFile, (codeStream) => - { - using var inputStream = File.OpenRead(inputFile); - using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); - BundleFileToCSource(symbolName, inputStream, outputUtf8Writer); - })) - { - state.Stop(); - } - }); + Log.LogMessage(MessageImportance.Low, "Bundling {0} into {1}", inputFile, destinationFile); + var symbolName = _resourceDataSymbolDictionary[registeredName]; + if (!EmitBundleFile(destinationFile, (codeStream) => + { + using var inputStream = File.OpenRead(inputFile); + using var outputUtf8Writer = new StreamWriter(codeStream, Utf8NoBom); + BundleFileToCSource(symbolName, inputStream, outputUtf8Writer); + })) + { + state.Stop(); + } + }); + } + finally + { + // msbuild does not release cores between invocations if the task is batched + be9?.ReleaseCores(allowedParallelism); + } foreach (ITaskItem bundledResource in bundledResources) { From 2a8434ece2a533e8a7685aa7e7657738ff8402fd Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Nov 2023 16:01:42 -0500 Subject: [PATCH 4/7] [wasi] Makefile: add build-packages target --- src/mono/wasi/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mono/wasi/Makefile b/src/mono/wasi/Makefile index f80efedad528d..7fc6904bc0702 100644 --- a/src/mono/wasi/Makefile +++ b/src/mono/wasi/Makefile @@ -54,6 +54,10 @@ app-builder: build-tasks: $(DOTNET) build $(TOP)/src/tasks/tasks.proj /p:Configuration=$(CONFIG) $(MSBUILD_ARGS) +build-packages: + rm -f $(TOP)/artifacts/packages/$(CONFIG)/Shipping/*.nupkg + WASI_SDK_PATH=$(WASI_SDK_PATH) $(TOP)/build.sh mono.packages+mono.manifests+packs.product -os wasi -c $(CONFIG) --binaryLog /p:ContinueOnError=false /p:StopOnFirstFailure=true $(MSBUILD_ARGS) + clean: $(RM) -rf $(BUILDS_OBJ_DIR) From f1c503e14ad141ec74ac6526ce8a8efa0fd94a41 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Nov 2023 18:48:35 -0500 Subject: [PATCH 5/7] Address review feedback from @ mdh1418 --- src/mono/wasi/build/WasiApp.Native.targets | 2 ++ .../MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index c45b8e00dc8f9..060082bf156db 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -387,11 +387,13 @@ BundleFile="@(_EmitBundleItemName->'%(BundleFile)')" OutputDirectory="$(_WasmIntermediateOutputPath)"> + <_EmitBundleOutputFile Remove="@(_EmitBundleOutputFile)" /> <_EmitBundleOutputFile Include="%(_EmitBundleOutputItem.DestinationFile)" /> + <_EmitBundleOutputFile Include="$(_EmitBundleRegistrationFile)" /> <_WasiObjectFilesForBundle Include="@(_EmitBundleOutputFile)" /> diff --git a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs index 40d8d2d61d23e..a872a0af55127 100644 --- a/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs +++ b/src/tasks/MonoTargetsTasks/EmitBundleTask/EmitBundleBase.cs @@ -63,6 +63,10 @@ public abstract class EmitBundleBase : Microsoft.Build.Utilities.Task, ICancelab [Output] public ITaskItem[] BundledResources { get; set; } = default!; + // Set only if @BundleFile was set + [Output] + public string? BundleRegistrationFile { get; set; } + public override bool Execute() { if (!Directory.Exists(OutputDirectory)) @@ -184,7 +188,6 @@ public override bool Execute() } finally { - // msbuild does not release cores between invocations if the task is batched be9?.ReleaseCores(allowedParallelism); } @@ -227,9 +230,7 @@ public override bool Execute() GenerateBundledResourcePreallocationAndRegistration(resourceSymbols, BundleRegistrationFunctionName, files, outputUtf8Writer); }); - TaskItem itemForBundleFile = new TaskItem(Path.Combine(OutputDirectory, BundleFile)); - itemForBundleFile.SetMetadata("DestinationFile", bundleFilePath); - bundledResources.Add(itemForBundleFile); + BundleRegistrationFile = bundleFilePath; } BundledResources = bundledResources.ToArray(); From b1b8f8fb7bdde6f2cb5ff1324f0969fd33d3a90d Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Nov 2023 19:08:55 -0500 Subject: [PATCH 6/7] simplify - prompted by review feedback from @ mdh1418 --- src/mono/wasi/build/WasiApp.Native.targets | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 060082bf156db..0bea4b86450bf 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -362,38 +362,36 @@ <_EmitBundleItemName - Include="_WasmAssembliesInternal" + Include="@(_WasmAssembliesInternal)" BundleRegistrationFunctionName="mono_register_assemblies_bundle" BundleFile="wasi_bundled_assemblies.o" /> - <_IcuFilesToBundle Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)icudt.dat" /> <_EmitBundleItemName - Include="_IcuFilesToBundle" + Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)icudt.dat" BundleRegistrationFunctionName="mono_register_icu_bundle" BundleFile="wasi_bundled_icu.o" /> - <_RuntimeConfigBinToBundle Include="$(_ParsedRuntimeConfigFilePath)" /> <_EmitBundleItemName - Include="_RuntimeConfigBinToBundle" + Include="$(_ParsedRuntimeConfigFilePath)" BundleRegistrationFunctionName="mono_register_runtimeconfig_bin" BundleFile="wasi_bundled_runtimeconfig_bin.o" /> - + - + <_EmitBundleOutputFile Remove="@(_EmitBundleOutputFile)" /> <_EmitBundleOutputFile Include="%(_EmitBundleOutputItem.DestinationFile)" /> - <_EmitBundleOutputFile Include="$(_EmitBundleRegistrationFile)" /> + <_EmitBundleOutputFile Include="@(_EmitBundleRegistrationFile)" /> <_WasiObjectFilesForBundle Include="@(_EmitBundleOutputFile)" /> From bfbf6cceaed5416fff51ae0e0fd5c2c7e81149e1 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Nov 2023 19:09:02 -0500 Subject: [PATCH 7/7] cleanup --- src/mono/wasi/build/WasiApp.Native.targets | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index 0bea4b86450bf..53e9840e09d7e 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -326,7 +326,6 @@ - <_WasmEHLib Condition="'$(WasmEnableExceptionHandling)' == 'true'">libmono-wasm-eh-wasm.a