From 4042acb1b7a46f834746cd854a8f7837b6c4b6bd Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Mon, 11 Jul 2022 13:50:01 -0400 Subject: [PATCH 01/13] First pass at switching the android build around --- eng/Subsets.props | 8 +- eng/testing/tests.android.targets | 84 ++ eng/testing/tests.mobile.targets | 112 +- .../android/build/AndroidApp.InTree.props | 13 + .../android/build/AndroidApp.InTree.targets | 20 + .../msbuild/android/build/AndroidApp.props | 17 + .../msbuild/android/build/AndroidApp.targets | 171 +++ .../NetTraceToMibcConverter.cs | 1294 +++++++++++++++++ .../NetTraceToMibcConverter.csproj | 45 + 9 files changed, 1657 insertions(+), 107 deletions(-) create mode 100644 eng/testing/tests.android.targets create mode 100644 src/mono/msbuild/android/build/AndroidApp.InTree.props create mode 100644 src/mono/msbuild/android/build/AndroidApp.InTree.targets create mode 100644 src/mono/msbuild/android/build/AndroidApp.props create mode 100644 src/mono/msbuild/android/build/AndroidApp.targets create mode 100644 src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs create mode 100644 src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj diff --git a/eng/Subsets.props b/eng/Subsets.props index 39ecf822395bc8..66d0c6498665b2 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -61,8 +61,7 @@ mono.llvm+ $(DefaultMonoSubsets)mono.wasmruntime+ $(DefaultMonoSubsets)mono.aotcross+ - $(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages+ - $(DefaultMonoSubsets)host+ + $(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages+mono.tools+ + @@ -358,6 +358,10 @@ + + + + diff --git a/eng/testing/tests.android.targets b/eng/testing/tests.android.targets new file mode 100644 index 00000000000000..ca5251ed16da13 --- /dev/null +++ b/eng/testing/tests.android.targets @@ -0,0 +1,84 @@ + + + $(BundleTestAppTargets);BundleTestAndroidApp + + + + + + + PrepareForAndroidBuildApp;$(AndroidBuildAppDependsOn);_CopyTestArchive + + AndroidBuildApp + + + + + + + + + + + + true + AndroidTestRunner.dll + + $(PublishDir) + $(BundleDir) + + + + arm64-v8a + armeabi-v7a + x86_64 + x86 + + AndroidTestRunner.dll + + + + + 1 + + + 1883302047 + + + 1 + + + + + + <_InternalForceInterpret>true + <_IsNative>true + + + <_PublishAssemblies Include="$(PublishDir)\**\*.dll" Exclude="$(PublishDir)\**\*.resources.dll" /> + <_SatelliteAssemblies Include="$(PublishDir)\**\*.resources.dll" /> + + + <_InternalForceInterpret Condition="'$(UseMonoJustInterp)' == 'true' and '%(FileName)%(Extension)' != 'System.Private.CoreLib.dll'">true + <_IsNative>false + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index edeed52b946f1d..96366a7e965e6e 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -16,6 +16,11 @@ true + + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)', 'dotnet-pgo')) + $([MSBuild]::NormalizePath('$(DotnetPgoToolDir)', 'dotnet-pgo')) + + true @@ -55,9 +60,8 @@ - - - + - - - $(PublishDir)$(AssemblyName).runtimeconfig.json - $(PublishDir)runtimeconfig.bin - - - - - - - - - - - - - - - - - - - - arm64-v8a - armeabi-v7a - x86_64 - x86 - - AndroidTestRunner.dll - - - - <_AndroidEnv Condition="'$(XUnitSingleThreadedMode)' == 'true'" Include="XUNIT_SINGLE_THREADED"> - 1 - - <_AndroidEnv Condition="'$(XUnitUseRandomizedTestOrderer)' == 'true'" Include="XUNIT_RANDOM_ORDER_SEED"> - 1883302047 - - <_AndroidEnv Condition="'$(XUnitSingleThreadedMode)' == 'true'" Include="XUNIT_VERBOSE"> - 1 - - - - - - @(MonoAOTCompilerDefaultAotArguments, ';') - @(MonoAOTCompilerDefaultProcessArguments, ';') - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/mono/msbuild/android/build/AndroidApp.InTree.props b/src/mono/msbuild/android/build/AndroidApp.InTree.props new file mode 100644 index 00000000000000..ff68f1085a0ac5 --- /dev/null +++ b/src/mono/msbuild/android/build/AndroidApp.InTree.props @@ -0,0 +1,13 @@ + + + + + + $(NetCoreAppCurrent) + false + false + true + link + false + + diff --git a/src/mono/msbuild/android/build/AndroidApp.InTree.targets b/src/mono/msbuild/android/build/AndroidApp.InTree.targets new file mode 100644 index 00000000000000..3b1f7c60a2626f --- /dev/null +++ b/src/mono/msbuild/android/build/AndroidApp.InTree.targets @@ -0,0 +1,20 @@ + + + + + + + + + + + <_LocalMicrosoftNetCoreAppRuntimePackDir>$(MicrosoftNetCoreAppRuntimePackDir) + + + + + + + diff --git a/src/mono/msbuild/android/build/AndroidApp.props b/src/mono/msbuild/android/build/AndroidApp.props new file mode 100644 index 00000000000000..4e2b2a5720f0f5 --- /dev/null +++ b/src/mono/msbuild/android/build/AndroidApp.props @@ -0,0 +1,17 @@ + + + $(TargetOS.ToLowerInvariant())-$(TargetArchitecture.ToLowerInvariant()) + true + true + + Publish + + _InitializeCommonProperties; + _BeforeAndroidBuildApp; + _AndroidResolveReferences; + _AndroidAotCompileApp; + _AndroidGenerateAppBundle; + _AfterAndroidBuildApp + + + \ No newline at end of file diff --git a/src/mono/msbuild/android/build/AndroidApp.targets b/src/mono/msbuild/android/build/AndroidApp.targets new file mode 100644 index 00000000000000..3ce50e882ceb61 --- /dev/null +++ b/src/mono/msbuild/android/build/AndroidApp.targets @@ -0,0 +1,171 @@ + + + + + + + + + + + + <_MobileIntermediateOutputPath>$([MSBuild]::NormalizeDirectory($(IntermediateOutputPath), 'mobile')) + + + + + + <_AndroidRuntimeConfigFilePath Condition="'$(_AndroidRuntimeConfigFilePath)' == ''">$([MSBuild]::NormalizePath($(AndroidAppDir), '$(AssemblyName).runtimeconfig.json')) + <_ParsedRuntimeConfigFilePath Condition="'$(_ParsedRuntimeConfigFilePath)' == ''">$([MSBuild]::NormalizePath($(AndroidAppDir), 'runtimeconfig.bin')) + + + + + + + + <_AndroidAssembliesInternal Remove="@(_AndroidAssembliesInternal)" /> + <_AndroidAssembliesInternal Include="@(AndroidAssembliesToBundle)"> + <_InternalForceInterpret>%(AndroidAssembliesToBundle._InternalForceInterpret) + <_IsNative>%(AndroidAssembliesToBundle._IsNative) + + + + + + + <_AOTMode Condition="'$(UseMonoJustInterp)' != 'true'">Full + <_AOTMode Condition="'$(UseMonoJustInterp)' == 'true'">JustInterp + + + + + + + + + + + + + + + + + + + + + + + + @(MonoAOTCompilerDefaultAotArguments, ';') + @(MonoAOTCompilerDefaultProcessArguments, ';') + + + + <_AotExcludeAssemblies Include="*System.Runtime.WindowsRuntime.dll" /> + + + <_InternalForceInterpret>%(_AndroidAssembliesInternal._InternalForceInterpret) + <_IsNative>%(_AndroidAssembliesInternal._IsNative) + + + <_AotInputAssemblies Include="@(_AndroidAssembliesInternal)" + Condition="'%(_AndroidAssembliesInternal._InternalForceInterpret)' != 'true'"> + $(AotArguments) + $(ProcessArguments) + + + <_AOT_InternalForceInterpretAssemblies Include="@(_AndroidAssembliesInternal->WithMetadataValue('_InternalForceInterpret', 'true'))" /> + <_AndroidAssembliesInternal Remove="@(_AndroidAssembliesInternal)" /> + + + + + + + + + + + + + <_AndroidAssembliesInternal Include="@(_AOT_InternalForceInterpretAssemblies)" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_RuntimeConfigReservedProperties Include="RUNTIME_IDENTIFIER"/> + <_RuntimeConfigReservedProperties Include="APP_CONTEXT_BASE_DIRECTORY"/> + + + + + + \ No newline at end of file diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs new file mode 100644 index 00000000000000..6d639794d3289f --- /dev/null +++ b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs @@ -0,0 +1,1294 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Reflection.Metadata; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.Reflection.PortableExecutable; +using System.Text.Json.Serialization; + +public class MonoAOTCompiler : Microsoft.Build.Utilities.Task +{ + /// + /// Path to AOT cross-compiler binary (mono-aot-cross) + /// + [Required] + public string CompilerBinaryPath { get; set; } = ""!; + + /// + /// Assemblies to be AOTd. They need to be in a self-contained directory. + /// + /// Metadata: + /// - AotArguments: semicolon-separated list of options that will be passed to --aot= + /// - ProcessArguments: semicolon-separated list of options that will be passed to the AOT compiler itself + /// + [Required] + public ITaskItem[] Assemblies { get; set; } = Array.Empty(); + + /// + /// Paths to be passed as MONO_PATH environment variable, when running mono-cross-aot. + /// These are in addition to the directory containing the assembly being precompiled. + /// + /// MONO_PATH=${dir_containing_assembly}:${AdditionalAssemblySearchPaths} + /// + /// + public string[]? AdditionalAssemblySearchPaths { get; set; } + + /// + /// Directory where the AOT'ed files will be emitted + /// + [NotNull] + [Required] + public string? OutputDir { get; set; } + + /// + /// Target triple passed to the AOT compiler. + /// + public string? Triple { get; set; } + + /// + /// Assemblies which were AOT compiled. + /// + /// Successful AOT compilation will set the following metadata on the items: + /// - AssemblerFile (when using OutputType=AsmOnly) + /// - ObjectFile (when using OutputType=Normal) + /// - LibraryFile (when using OutputType=Library) + /// - AotDataFile (when using UseAotDataFile=true) + /// - LlvmObjectFile (if using LLVM) + /// - LlvmBitcodeFile (if using LLVM-only) + /// + [Output] + public ITaskItem[]? CompiledAssemblies { get; set; } + + /// + /// Disable parallel AOT compilation + /// + public bool DisableParallelAot { get; set; } + + /// + /// Use LLVM for AOT compilation. + /// The cross-compiler must be built with LLVM support + /// + public bool UseLLVM { get; set; } + + /// + /// This instructs the AOT code generator to output certain data constructs into a separate file. This can reduce the executable images some five to twenty percent. + /// Developers need to then ship the resulting aotdata as a resource and register a hook to load the data on demand by using the mono_install_load_aot_data_hook() method. + /// Defaults to true. + /// + public bool UseAotDataFile { get; set; } = true; + + /// + /// Create an ELF object file (.o) or .s file which can be statically linked into an executable when embedding the mono runtime. + /// Only valid if OutputType is ObjectFile or AsmOnly. + /// + public bool UseStaticLinking { get; set; } + + /// + /// When this option is specified, icalls (internal calls made from the standard library into the mono runtime code) are invoked directly instead of going through the operating system symbol lookup operation. + /// This requires UseStaticLinking=true. + /// + public bool UseDirectIcalls { get; set; } + + /// + /// When this option is specified, P/Invoke methods are invoked directly instead of going through the operating system symbol lookup operation + /// This requires UseStaticLinking=true. + /// + public bool UseDirectPInvoke { get; set; } + + /// + /// Instructs the AOT compiler to emit DWARF debugging information. + /// + public bool UseDwarfDebug { get; set; } + + /// + /// Path to Dotnet PGO binary (dotnet-pgo) + /// + public string? PgoBinaryPath { get; set; } + + /// + /// NetTrace file to use when invoking dotnet-pgo for + /// + public string? NetTracePath { get; set; } + + /// + /// File to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. + /// + public string[]? AotProfilePath { get; set; } + + /// + /// Mibc file to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. + /// + public string[] MibcProfilePath { get; set; } = Array.Empty(); + + /// + /// List of profilers to use. + /// + public string[]? Profilers { get; set; } + + /// + /// Generate a file containing mono_aot_register_module() calls for each AOT module which can be compiled into the app embedding mono. + /// If set, this implies UseStaticLinking=true. + /// + public string? AotModulesTablePath { get; set; } + + /// + /// Source code language of the AOT modules table. Supports "C" or "ObjC". + /// Defaults to "C". + /// + public string? AotModulesTableLanguage { get; set; } = nameof(MonoAotModulesTableLanguage.C); + + /// + /// Choose between 'Normal', 'JustInterp', 'Full', 'FullInterp', 'Hybrid', 'LLVMOnly', 'LLVMOnlyInterp'. + /// LLVMOnly means to use only LLVM for FullAOT, AOT result will be a LLVM Bitcode file (the cross-compiler must be built with LLVM support) + /// The "interp" options ('LLVMOnlyInterp' and 'FullInterp') mean generate necessary support to fall back to interpreter if AOT code is not possible for some methods. + /// The difference between 'JustInterp' and 'FullInterp' is that 'FullInterp' will AOT all the methods in the given assemblies, while 'JustInterp' will only AOT the wrappers and trampolines necessary for the runtime to execute the managed methods using the interpreter and to interoperate with P/Invokes and unmanaged callbacks. + /// + public string Mode { get; set; } = nameof(MonoAotMode.Normal); + + /// + /// Choose between 'ObjectFile', 'AsmOnly', 'Library' + /// ObjectFile means the AOT compiler will produce an .o object file, AsmOnly will produce .s assembly code and Library will produce a .so/.dylib/.dll shared library. + /// + public string OutputType { get; set; } = nameof(MonoAotOutputType.ObjectFile); + + /// + /// Choose between 'Dll', 'Dylib', 'So'. Only valid if OutputType is Library. + /// Dll means the AOT compiler will produce a Windows PE .dll file, Dylib means an Apple Mach-O .dylib and So means a Linux/Android ELF .so file. + /// + public string? LibraryFormat { get; set; } + + /// + /// Prefix that will be added to the library file name, e.g. to add 'lib' prefix required by some platforms. Only valid if OutputType is Library. + /// + public string LibraryFilePrefix { get; set; } = ""; + + /// + /// Path to the directory where LLVM binaries (opt and llc) are found. + /// It's required if UseLLVM is set + /// + public string? LLVMPath { get; set; } + + /// + /// Prepends a prefix to the name of tools ran by the AOT compiler, i.e. 'as'/'ld'. + /// + public string? ToolPrefix { get; set; } + + /// + /// Path to the directory where msym artifacts are stored. + /// + public string? MsymPath { get; set; } + + /// + /// The assembly whose AOT image will contained dedup-ed generic instances + /// + public string? DedupAssembly { get; set; } + + /// + /// Debug option in llvm aot mode + /// defaults to "nodebug" since some targes can't generate debug info + /// + public string? LLVMDebug { get; set; } = "nodebug"; + + /// + /// File used to track hashes of assemblies, to act as a cache + /// Output files don't get written, if they haven't changed + /// + public string? CacheFilePath { get; set; } + + /// + /// Passes additional, custom arguments to --aot + /// + public string? AotArguments { get; set; } + + /// + /// Passes temp-path to the AOT compiler + /// + public string? TempPath { get; set; } + + /// + /// Passes ld-name to the AOT compiler, for use with UseLLVM=true + /// + public string? LdName { get; set; } + + /// + /// Passes ld-flags to the AOT compiler, for use with UseLLVM=true + /// + public string? LdFlags { get; set; } + + /// + /// Specify WorkingDirectory for the AOT compiler + /// + public string? WorkingDirectory { get; set; } + + [Required] + public string IntermediateOutputPath { get; set; } = string.Empty; + + [Output] + public string[]? FileWrites { get; private set; } + + private static readonly Encoding s_utf8Encoding = new UTF8Encoding(false); + + private List _fileWrites = new(); + + private IList? _assembliesToCompile; + private ConcurrentDictionary compiledAssemblies = new(); + + private MonoAotMode parsedAotMode; + private MonoAotOutputType parsedOutputType; + private MonoAotLibraryFormat parsedLibraryFormat; + private MonoAotModulesTableLanguage parsedAotModulesTableLanguage; + + private FileCache? _cache; + private int _numCompiled; + private int _totalNumAssemblies; + + private bool ProcessAndValidateArguments() + { + if (!File.Exists(CompilerBinaryPath)) + { + Log.LogError($"{nameof(CompilerBinaryPath)}='{CompilerBinaryPath}' doesn't exist."); + return false; + } + + if (Assemblies.Length == 0) + { + Log.LogError($"'{nameof(Assemblies)}' is required."); + return false; + } + + // A relative path might be used along with WorkingDirectory, + // only call Path.GetFullPath() if WorkingDirectory is blank. + if (string.IsNullOrEmpty(WorkingDirectory) && !Path.IsPathRooted(OutputDir)) + OutputDir = Path.GetFullPath(OutputDir); + + if (!Directory.Exists(OutputDir)) + { + Log.LogError($"OutputDir={OutputDir} doesn't exist"); + return false; + } + + if (!Directory.Exists(IntermediateOutputPath)) + Directory.CreateDirectory(IntermediateOutputPath); + + if (!string.IsNullOrEmpty(NetTracePath)) + { + if (!File.Exists(NetTracePath)) + { + Log.LogError($"NetTracePath={nameof(NetTracePath)} doesn't exist"); + return false; + } + if (!File.Exists(PgoBinaryPath)) + { + Log.LogError($"NetTracePath was provided, but {nameof(PgoBinaryPath)}='{PgoBinaryPath}' doesn't exist"); + return false; + } + } + + if (AotProfilePath != null) + { + foreach (var path in AotProfilePath) + { + if (!File.Exists(path)) + { + Log.LogError($"AotProfilePath '{path}' doesn't exist."); + return false; + } + } + } + + foreach (var path in MibcProfilePath) + { + if (!File.Exists(path)) + { + Log.LogError($"MibcProfilePath '{path}' doesn't exist."); + return false; + } + } + + if (UseLLVM) + { + if (string.IsNullOrEmpty(LLVMPath)) + // prevent using some random llc/opt from PATH (installed with clang) + throw new LogAsErrorException($"'{nameof(LLVMPath)}' is required when '{nameof(UseLLVM)}' is true."); + + if (!Directory.Exists(LLVMPath)) + { + Log.LogError($"Could not find LLVMPath=${LLVMPath}"); + return false; + } + } + + if (!Enum.TryParse(Mode, true, out parsedAotMode)) + { + Log.LogError($"Unknown Mode value: {Mode}. '{nameof(Mode)}' must be one of: {string.Join(",", Enum.GetNames(typeof(MonoAotMode)))}"); + return false; + } + + switch (OutputType) + { + case "ObjectFile": parsedOutputType = MonoAotOutputType.ObjectFile; break; + case "AsmOnly": parsedOutputType = MonoAotOutputType.AsmOnly; break; + case "Library": parsedOutputType = MonoAotOutputType.Library; break; + case "Normal": + Log.LogWarning($"'{nameof(OutputType)}=Normal' is deprecated, use 'ObjectFile' instead."); + parsedOutputType = MonoAotOutputType.ObjectFile; break; + default: + throw new LogAsErrorException($"'{nameof(OutputType)}' must be one of: '{nameof(MonoAotOutputType.ObjectFile)}', '{nameof(MonoAotOutputType.AsmOnly)}', '{nameof(MonoAotOutputType.Library)}'. Received: '{OutputType}'."); + } + + switch (LibraryFormat) + { + case "Dll": parsedLibraryFormat = MonoAotLibraryFormat.Dll; break; + case "Dylib": parsedLibraryFormat = MonoAotLibraryFormat.Dylib; break; + case "So": parsedLibraryFormat = MonoAotLibraryFormat.So; break; + default: + if (parsedOutputType == MonoAotOutputType.Library) + throw new LogAsErrorException($"'{nameof(LibraryFormat)}' must be one of: '{nameof(MonoAotLibraryFormat.Dll)}', '{nameof(MonoAotLibraryFormat.Dylib)}', '{nameof(MonoAotLibraryFormat.So)}'. Received: '{LibraryFormat}'."); + break; + } + + if (parsedAotMode == MonoAotMode.LLVMOnly && !UseLLVM) + { + throw new LogAsErrorException($"'{nameof(UseLLVM)}' must be true when '{nameof(Mode)}' is {nameof(MonoAotMode.LLVMOnly)}."); + } + + switch (AotModulesTableLanguage) + { + case "C": parsedAotModulesTableLanguage = MonoAotModulesTableLanguage.C; break; + case "ObjC": parsedAotModulesTableLanguage = MonoAotModulesTableLanguage.ObjC; break; + default: + throw new LogAsErrorException($"'{nameof(AotModulesTableLanguage)}' must be one of: '{nameof(MonoAotModulesTableLanguage.C)}', '{nameof(MonoAotModulesTableLanguage.ObjC)}'. Received: '{AotModulesTableLanguage}'."); + } + + if (!string.IsNullOrEmpty(AotModulesTablePath)) + { + // AOT modules for static linking, needs the aot modules table + UseStaticLinking = true; + } + + if (UseDirectIcalls && !UseStaticLinking) + { + throw new LogAsErrorException($"'{nameof(UseDirectIcalls)}' can only be used with '{nameof(UseStaticLinking)}=true'."); + } + + if (UseDirectPInvoke && !UseStaticLinking) + { + throw new LogAsErrorException($"'{nameof(UseDirectPInvoke)}' can only be used with '{nameof(UseStaticLinking)}=true'."); + } + + if (UseStaticLinking && (parsedOutputType == MonoAotOutputType.Library)) + { + throw new LogAsErrorException($"'{nameof(OutputType)}=Library' can not be used with '{nameof(UseStaticLinking)}=true'."); + } + + foreach (var asmItem in Assemblies) + { + string? fullPath = asmItem.GetMetadata("FullPath"); + if (!File.Exists(fullPath)) + throw new LogAsErrorException($"Could not find {fullPath} to AOT"); + } + + return !Log.HasLoggedErrors; + } + + public override bool Execute() + { + try + { + return ExecuteInternal(); + } + catch (LogAsErrorException laee) + { + Log.LogError(laee.Message); + return false; + } + finally + { + if (_cache != null && _cache.Save(CacheFilePath!)) + _fileWrites.Add(CacheFilePath!); + FileWrites = _fileWrites.ToArray(); + } + } + + private bool ProcessNettrace(string netTraceFile) + { + var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(netTraceFile), ".mibc")); + + if (_cache!.Enabled) + { + string hash = Utils.ComputeHash(netTraceFile); + if (!_cache!.UpdateAndCheckHasFileChanged($"-mibc-source-file-{Path.GetFileName(netTraceFile)}", hash)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping generating {outputMibcPath} from {netTraceFile} because source file hasn't changed"); + return true; + } + else + { + Log.LogMessage(MessageImportance.Low, $"Generating {outputMibcPath} from {netTraceFile} because the source file's hash has changed."); + } + } + + (int exitCode, string output) = Utils.TryRunProcess(Log, + PgoBinaryPath!, + $"create-mibc --trace {netTraceFile} --output {outputMibcPath}"); + + if (exitCode != 0) + { + Log.LogError($"dotnet-pgo({PgoBinaryPath}) failed for {netTraceFile}:{output}"); + return false; + } + + MibcProfilePath = MibcProfilePath.Append(outputMibcPath).ToArray(); + Log.LogMessage(MessageImportance.Low, $"Generated {outputMibcPath} from {PgoBinaryPath}"); + return true; + } + + private bool ExecuteInternal() + { + if (!ProcessAndValidateArguments()) + return false; + + _assembliesToCompile = EnsureAndGetAssembliesInTheSameDir(Assemblies); + _assembliesToCompile = FilterAssemblies(_assembliesToCompile); + + if (!string.IsNullOrEmpty(AotModulesTablePath) && !GenerateAotModulesTable(_assembliesToCompile, Profilers, AotModulesTablePath)) + return false; + + string? monoPaths = null; + if (AdditionalAssemblySearchPaths != null) + monoPaths = string.Join(Path.PathSeparator.ToString(), AdditionalAssemblySearchPaths); + + _cache = new FileCache(CacheFilePath, Log); + + if (!string.IsNullOrEmpty(NetTracePath) && !ProcessNettrace(NetTracePath)) + return false; + + List argsList = new(); + foreach (var assemblyItem in _assembliesToCompile) + argsList.Add(GetPrecompileArgumentsFor(assemblyItem, monoPaths)); + + _totalNumAssemblies = _assembliesToCompile.Count; + if (CheckAllUpToDate(argsList)) + { + Log.LogMessage(MessageImportance.High, "Everything is up-to-date, nothing to precompile"); + + _fileWrites.AddRange(argsList.SelectMany(args => args.ProxyFiles).Select(pf => pf.TargetFile)); + foreach (var args in argsList) + compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly); + } + else + { + int allowedParallelism = DisableParallelAot ? 1 : Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); + + /* + From: https://github.com/dotnet/runtime/issues/46146#issuecomment-754021690 + + Stephen Toub: + "As such, by default ForEach works on a scheme whereby each + thread takes one item each time it goes back to the enumerator, + and then after a few times of this upgrades to taking two items + each time it goes back to the enumerator, and then four, and + then eight, and so on. This ammortizes the cost of taking and + releasing the lock across multiple items, while still enabling + parallelization for enumerables containing just a few items. It + does, however, mean that if you've got a case where the body + takes a really long time and the work for every item is + heterogeneous, you can end up with an imbalance." + + The time taken by individual compile jobs here can vary a + lot, depending on various factors like file size. This can + create an imbalance, like mentioned above, and we can end up + in a situation where one of the partitions has a job that + takes very long to execute, by which time other partitions + have completed, so some cores are idle. But the idle + ones won't get any of the remaining jobs, because they are + all assigned to that one partition. + + Instead, we want to use work-stealing so jobs can be run by any partition. + */ + ParallelLoopResult result = Parallel.ForEach( + Partitioner.Create(argsList, EnumerablePartitionerOptions.NoBuffering), + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (args, state) => PrecompileLibraryParallel(args, state)); + + if (result.IsCompleted) + { + int numUnchanged = _totalNumAssemblies - _numCompiled; + if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) + Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); + } + else if (!Log.HasLoggedErrors) + { + Log.LogError($"Precompiling failed due to unknown reasons. Check log for more info"); + } + } + + CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray(); + return !Log.HasLoggedErrors; + } + + private static bool CheckAllUpToDate(IList argsList) + { + foreach (var args in argsList) + { + // compare original assembly vs it's outputs.. all it's outputs! + string assemblyPath = args.AOTAssembly.GetMetadata("FullPath"); + if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile))) + return false; + } + + return true; + + static bool IsNewerThanOutput(string inFile, string outFile) + => !File.Exists(inFile) || !File.Exists(outFile) || + (File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile)); + } + + private IList FilterAssemblies(IEnumerable assemblies) + { + List filteredAssemblies = new(); + foreach (var asmItem in assemblies) + { + if (ShouldSkip(asmItem)) + { + if (parsedAotMode == MonoAotMode.LLVMOnly) + throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {asmItem.ItemSpec}"); + + Log.LogMessage(MessageImportance.Low, $"Skipping {asmItem.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true"); + continue; + } + + string assemblyPath = asmItem.GetMetadata("FullPath"); + using var assemblyFile = File.OpenRead(assemblyPath); + using PEReader reader = new(assemblyFile, PEStreamOptions.Default); + if (!reader.HasMetadata) + { + Log.LogWarning($"Skipping unmanaged {assemblyPath} for AOT"); + continue; + } + + filteredAssemblies.Add(asmItem); + } + + return filteredAssemblies; + + static bool ShouldSkip(ITaskItem asmItem) + => bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip; + } + + private IList EnsureAndGetAssembliesInTheSameDir(IList assemblies) + { + string firstAsmDir = Path.GetDirectoryName(assemblies.First().GetMetadata("FullPath")) ?? string.Empty; + bool allInSameDir = assemblies.All(asm => Path.GetDirectoryName(asm.GetMetadata("FullPath")) == firstAsmDir); + if (allInSameDir) + return assemblies; + + // Copy to aot-in + + string aotInPath = Path.Combine(IntermediateOutputPath, "aot-in"); + Directory.CreateDirectory(aotInPath); + + List newAssemblies = new(); + foreach (var asmItem in assemblies) + { + string asmPath = asmItem.GetMetadata("FullPath"); + string newPath = Path.Combine(aotInPath, Path.GetFileName(asmPath)); + + // FIXME: delete files not in originalAssemblies though + // FIXME: or .. just delete the whole dir? + if (Utils.CopyIfDifferent(asmPath, newPath, useHash: true)) + Log.LogMessage(MessageImportance.Low, $"Copying {asmPath} to {newPath}"); + _fileWrites.Add(newPath); + + ITaskItem newAsm = new TaskItem(newPath); + asmItem.CopyMetadataTo(newAsm); + newAssemblies.Add(newAsm); + } + + return newAssemblies; + } + + private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, string? monoPaths) + { + string assembly = assemblyItem.GetMetadata("FullPath"); + string assemblyDir = Path.GetDirectoryName(assembly)!; + var aotAssembly = new TaskItem(assembly); + var aotArgs = new List(); + var processArgs = new List(); + bool isDedup = assembly == DedupAssembly; + List proxyFiles = new(capacity: 5); + + var a = assemblyItem.GetMetadata("AotArguments"); + if (a != null) + { + aotArgs.AddRange(a.Split(new char[]{ ';' }, StringSplitOptions.RemoveEmptyEntries)); + } + + var p = assemblyItem.GetMetadata("ProcessArguments"); + if (p != null) + { + processArgs.AddRange(p.Split(new char[]{ ';' }, StringSplitOptions.RemoveEmptyEntries)); + } + + processArgs.Add("--debug"); + + // add LLVM options + if (UseLLVM) + { + processArgs.Add("--llvm"); + + if (!string.IsNullOrEmpty(LLVMDebug)) + aotArgs.Add(LLVMDebug); + + aotArgs.Add($"llvm-path={LLVMPath}"); + } + else + { + processArgs.Add("--nollvm"); + } + + if (UseStaticLinking) + { + aotArgs.Add($"static"); + } + + if (UseDwarfDebug) + { + aotArgs.Add($"dwarfdebug"); + } + + if (!string.IsNullOrEmpty(Triple)) + { + aotArgs.Add($"mtriple={Triple}"); + } + + if (!string.IsNullOrEmpty(ToolPrefix)) + { + aotArgs.Add($"tool-prefix={ToolPrefix}"); + } + + string assemblyFilename = Path.GetFileName(assembly); + + if (isDedup) + { + aotArgs.Add($"dedup-include={assemblyFilename}"); + } + else if (!string.IsNullOrEmpty (DedupAssembly)) + { + aotArgs.Add("dedup-skip"); + } + + // compute output mode and file names + if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.LLVMOnlyInterp) + { + aotArgs.Add("llvmonly"); + + string llvmBitcodeFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.bc")); + ProxyFile proxyFile = _cache!.NewFile(llvmBitcodeFile); + proxyFiles.Add(proxyFile); + aotAssembly.SetMetadata("LlvmBitcodeFile", proxyFile.TargetFile); + + if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) + { + aotArgs.Add("interp"); + } + + if (parsedOutputType == MonoAotOutputType.AsmOnly) + { + aotArgs.Add("asmonly"); + aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); + } + else + { + aotArgs.Add($"outfile={proxyFile.TempFile}"); + } + } + else + { + if (parsedAotMode == MonoAotMode.Full || parsedAotMode == MonoAotMode.FullInterp) + { + aotArgs.Add("full"); + } + + if (parsedAotMode == MonoAotMode.Hybrid) + { + aotArgs.Add("hybrid"); + } + + if (parsedAotMode == MonoAotMode.FullInterp || parsedAotMode == MonoAotMode.JustInterp) + { + aotArgs.Add("interp"); + } + + switch (parsedOutputType) + { + case MonoAotOutputType.ObjectFile: + { + string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); + ProxyFile proxyFile = _cache!.NewFile(objectFile); + proxyFiles.Add((proxyFile)); + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("ObjectFile", proxyFile.TargetFile); + } + break; + + case MonoAotOutputType.AsmOnly: + { + aotArgs.Add("asmonly"); + + string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); + ProxyFile proxyFile = _cache!.NewFile(assemblerFile); + proxyFiles.Add(proxyFile); + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("AssemblerFile", proxyFile.TargetFile); + } + break; + + case MonoAotOutputType.Library: + { + string extension = parsedLibraryFormat switch { + MonoAotLibraryFormat.Dll => ".dll", + MonoAotLibraryFormat.Dylib => ".dylib", + MonoAotLibraryFormat.So => ".so", + _ => throw new ArgumentOutOfRangeException() + }; + string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; + string libraryFilePath = Path.Combine(OutputDir, libraryFileName); + ProxyFile proxyFile = _cache!.NewFile(libraryFilePath); + proxyFiles.Add(proxyFile); + + aotArgs.Add($"outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("LibraryFile", proxyFile.TargetFile); + } + break; + + default: + throw new Exception($"Bug: Unhandled MonoAotOutputType: {parsedAotMode}"); + } + + if (UseLLVM) + { + string llvmObjectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll-llvm.o")); + ProxyFile proxyFile = _cache.NewFile(llvmObjectFile); + proxyFiles.Add(proxyFile); + aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("LlvmObjectFile", proxyFile.TargetFile); + } + } + + // pass msym-dir if specified + if (MsymPath != null) + { + aotArgs.Add($"msym-dir={MsymPath}"); + } + + if (UseAotDataFile) + { + string aotDataFile = Path.ChangeExtension(assembly, ".aotdata"); + ProxyFile proxyFile = _cache.NewFile(aotDataFile); + proxyFiles.Add(proxyFile); + aotArgs.Add($"data-outfile={proxyFile.TempFile}"); + aotAssembly.SetMetadata("AotDataFile", proxyFile.TargetFile); + } + + if (AotProfilePath?.Length > 0) + { + aotArgs.Add("profile-only"); + foreach (var path in AotProfilePath) + { + aotArgs.Add($"profile={path}"); + } + } + + if (MibcProfilePath.Length > 0) + { + aotArgs.Add("profile-only"); + foreach (var path in MibcProfilePath) + { + aotArgs.Add($"mibc-profile={path}"); + } + } + + if (!string.IsNullOrEmpty(AotArguments)) + { + aotArgs.Add(AotArguments); + } + + if (!string.IsNullOrEmpty(TempPath)) + { + aotArgs.Add($"temp-path={TempPath}"); + } + + if (!string.IsNullOrEmpty(LdName)) + { + aotArgs.Add($"ld-name={LdName}"); + } + + if (!string.IsNullOrEmpty(LdFlags)) + { + aotArgs.Add($"ld-flags={LdFlags}"); + } + + // we need to quote the entire --aot arguments here to make sure it is parsed + // on Windows as one argument. Otherwise it will be split up into multiple + // values, which wont work. + processArgs.Add($"\"--aot={string.Join(",", aotArgs)}\""); + + if (isDedup) + { + foreach (var aItem in _assembliesToCompile!) + processArgs.Add(aItem.ItemSpec); + } + else + { + if (string.IsNullOrEmpty(WorkingDirectory)) + { + processArgs.Add('"' + assemblyFilename + '"'); + } + else + { + // If WorkingDirectory is supplied, the caller could be passing in a relative path + // Use the original ItemSpec that was passed in. + processArgs.Add('"' + assemblyItem.ItemSpec + '"'); + } + } + + monoPaths = $"{assemblyDir}{Path.PathSeparator}{monoPaths}"; + var envVariables = new Dictionary + { + {"MONO_PATH", monoPaths }, + {"MONO_ENV_OPTIONS", string.Empty} // we do not want options to be provided out of band to the cross compilers + }; + + var responseFileContent = string.Join(" ", processArgs); + var responseFilePath = Path.GetTempFileName(); + using (var sw = new StreamWriter(responseFilePath, append: false, encoding: s_utf8Encoding)) + { + sw.WriteLine(responseFileContent); + } + + return new PrecompileArguments(ResponseFilePath: responseFilePath, + EnvironmentVariables: envVariables, + WorkingDir: string.IsNullOrEmpty(WorkingDirectory) ? assemblyDir : WorkingDirectory, + AOTAssembly: aotAssembly, + ProxyFiles: proxyFiles); + } + + private bool PrecompileLibrary(PrecompileArguments args) + { + string assembly = args.AOTAssembly.GetMetadata("FullPath"); + string output; + try + { + string msgPrefix = $"[{Path.GetFileName(assembly)}] "; + + // run the AOT compiler + (int exitCode, output) = Utils.TryRunProcess(Log, + CompilerBinaryPath, + $"--response=\"{args.ResponseFilePath}\"", + args.EnvironmentVariables, + args.WorkingDir, + silent: true, + debugMessageImportance: MessageImportance.Low, + label: Path.GetFileName(assembly)); + + var importance = exitCode == 0 ? MessageImportance.Low : MessageImportance.High; + // Log the command in a compact format which can be copy pasted + { + StringBuilder envStr = new StringBuilder(string.Empty); + foreach (KeyValuePair kvp in args.EnvironmentVariables) + envStr.Append($"{kvp.Key}={kvp.Value} "); + Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {args.WorkingDir}: {envStr}{CompilerBinaryPath} {File.ReadAllText(args.ResponseFilePath, s_utf8Encoding)}"); + } + + if (exitCode != 0) + { + Log.LogError($"Precompiling failed for {assembly}.{Environment.NewLine}{output}"); + return false; + } + + Log.LogMessage(importance, output); + } + catch (Exception ex) + { + Log.LogMessage(MessageImportance.Low, ex.ToString()); + Log.LogError($"Precompiling failed for {assembly}: {ex.Message}"); + return false; + } + finally + { + File.Delete(args.ResponseFilePath); + } + + bool copied = false; + foreach (var proxyFile in args.ProxyFiles) + { + if (!File.Exists(proxyFile.TempFile)) + { + Log.LogError($"Precompile command succeeded, but can't find the expected temporary output file - {proxyFile.TempFile} for {assembly}.{Environment.NewLine}{output}"); + return false; + } + + copied |= proxyFile.CopyOutputFileIfChanged(); + _fileWrites.Add(proxyFile.TargetFile); + } + + if (copied) + { + string copiedFiles = string.Join(", ", args.ProxyFiles.Select(tf => Path.GetFileName(tf.TargetFile))); + int count = Interlocked.Increment(ref _numCompiled); + Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assembly)} -> {copiedFiles}"); + } + + compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly); + return true; + } + + private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopState state) + { + try + { + if (PrecompileLibrary(args)) + return; + } + catch (LogAsErrorException laee) + { + Log.LogError($"Precompiling failed for {args.AOTAssembly}: {laee.Message}"); + } + catch (Exception ex) + { + if (Log.HasLoggedErrors) + Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}"); + else + Log.LogError($"Precompiling failed for {args.AOTAssembly}: {ex}"); + } + + state.Break(); + } + + private bool GenerateAotModulesTable(IEnumerable assemblies, string[]? profilers, string outputFile) + { + var symbols = new List(); + foreach (var asm in assemblies) + { + string asmPath = asm.ItemSpec; + if (!File.Exists(asmPath)) + { + Log.LogError($"Could not find assembly {asmPath}"); + return false; + } + + if (!TryGetAssemblyName(asmPath, out string? assemblyName)) + return false; + + string symbolName = assemblyName.Replace ('.', '_').Replace ('-', '_').Replace(' ', '_'); + symbols.Add($"mono_aot_module_{symbolName}_info"); + } + + Directory.CreateDirectory(Path.GetDirectoryName(outputFile)!); + + string tmpAotModulesTablePath = Path.GetTempFileName(); + using (var writer = File.CreateText(tmpAotModulesTablePath)) + { + if (parsedAotModulesTableLanguage == MonoAotModulesTableLanguage.C) + { + writer.WriteLine("#include "); + + foreach (var symbol in symbols) + { + writer.WriteLine($"extern void *{symbol};"); + } + writer.WriteLine("void register_aot_modules ()"); + writer.WriteLine("{"); + foreach (var symbol in symbols) + { + writer.WriteLine($"\tmono_aot_register_module ({symbol});"); + } + writer.WriteLine("}"); + + foreach (var profiler in profilers ?? Enumerable.Empty()) + { + writer.WriteLine($"void mono_profiler_init_{profiler} (const char *desc);"); + writer.WriteLine("EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_" + profiler + " (const char *desc) { mono_profiler_init_" + profiler + " (desc); }"); + } + + if (parsedAotMode == MonoAotMode.LLVMOnly) + { + writer.WriteLine("#define EE_MODE_LLVMONLY 1"); + } + + if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) + { + writer.WriteLine("#define EE_MODE_LLVMONLY_INTERP 1"); + } + } + else if (parsedAotModulesTableLanguage == MonoAotModulesTableLanguage.ObjC) + { + writer.WriteLine("#include "); + writer.WriteLine("#include "); + writer.WriteLine(""); + writer.WriteLine("#if TARGET_OS_IPHONE && (!TARGET_IPHONE_SIMULATOR || FORCE_AOT)"); + + foreach (var symbol in symbols) + { + writer.WriteLine($"extern void *{symbol};"); + } + + writer.WriteLine("void register_aot_modules (void)"); + writer.WriteLine("{"); + foreach (var symbol in symbols) + { + writer.WriteLine($"\tmono_aot_register_module ({symbol});"); + } + writer.WriteLine("}"); + writer.WriteLine("#endif"); + } + else + { + throw new NotSupportedException(); + } + } + + if (Utils.CopyIfDifferent(tmpAotModulesTablePath, outputFile, useHash: false)) + { + _fileWrites.Add(outputFile); + Log.LogMessage(MessageImportance.Low, $"Generated {outputFile}"); + } + + return true; + } + + private bool TryGetAssemblyName(string asmPath, [NotNullWhen(true)] out string? assemblyName) + { + assemblyName = null; + + try + { + using var fs = new FileStream(asmPath, FileMode.Open, FileAccess.Read); + using var peReader = new PEReader(fs); + MetadataReader mr = peReader.GetMetadataReader(); + assemblyName = mr.GetAssemblyDefinition().GetAssemblyName().Name; + + if (string.IsNullOrEmpty(assemblyName)) + { + Log.LogError($"Could not get assembly name for {asmPath}"); + return false; + } + + return true; + } + catch (Exception ex) + { + Log.LogError($"Failed to get assembly name for {asmPath}: {ex.Message}"); + return false; + } + } + + private static IList ConvertAssembliesDictToOrderedList(ConcurrentDictionary dict, IList originalAssemblies) + { + List outItems = new(originalAssemblies.Count); + foreach (ITaskItem item in originalAssemblies) + { + if (dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem)) + outItems.Add(dictItem); + } + return outItems; + } + + internal sealed class PrecompileArguments + { + public PrecompileArguments(string ResponseFilePath, IDictionary EnvironmentVariables, string WorkingDir, ITaskItem AOTAssembly, IList ProxyFiles) + { + this.ResponseFilePath = ResponseFilePath; + this.EnvironmentVariables = EnvironmentVariables; + this.WorkingDir = WorkingDir; + this.AOTAssembly = AOTAssembly; + this.ProxyFiles = ProxyFiles; + } + + public string ResponseFilePath { get; private set; } + public IDictionary EnvironmentVariables { get; private set; } + public string WorkingDir { get; private set; } + public ITaskItem AOTAssembly { get; private set; } + public IList ProxyFiles { get; private set; } + } +} + +internal sealed class FileCache +{ + private CompilerCache? _newCache; + private CompilerCache? _oldCache; + + public bool Enabled { get; } + public TaskLoggingHelper Log { get; } + + public FileCache(string? cacheFilePath, TaskLoggingHelper log) + { + Log = log; + if (string.IsNullOrEmpty(cacheFilePath)) + { + Log.LogMessage(MessageImportance.Low, $"Disabling cache, because CacheFilePath is not set"); + return; + } + + Enabled = true; + if (File.Exists(cacheFilePath)) + { + _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), + typeof(CompilerCache), + new JsonSerializerOptions()); + } + + _oldCache ??= new(); + _newCache = new(_oldCache.FileHashes); + } + + public bool UpdateAndCheckHasFileChanged(string filePath, string newHash) + { + if (!Enabled) + throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); + + _newCache!.FileHashes[filePath] = newHash; + return !_oldCache!.FileHashes.TryGetValue(filePath, out string? oldHash) || oldHash != newHash; + } + + public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause) + { + if (!Enabled) + throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); + + cause = null; + + string newHash = Utils.ComputeHash(proxyFile.TempFile); + _newCache!.FileHashes[proxyFile.TargetFile] = newHash; + + if (!File.Exists(proxyFile.TargetFile)) + { + cause = $"the output file didn't exist"; + return true; + } + + string? oldHash; + if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) + oldHash = Utils.ComputeHash(proxyFile.TargetFile); + + if (oldHash != newHash) + { + cause = $"hash for the file changed"; + return true; + } + + return false; + } + + public bool Save(string? cacheFilePath) + { + if (!Enabled || string.IsNullOrEmpty(cacheFilePath)) + return false; + + var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(cacheFilePath!, json); + return true; + } + + public ProxyFile NewFile(string targetFile) => new ProxyFile(targetFile, this); +} + +internal sealed class ProxyFile +{ + public string TargetFile { get; } + public string TempFile { get; } + private FileCache _cache; + + public ProxyFile(string targetFile, FileCache cache) + { + _cache = cache; + this.TargetFile = targetFile; + this.TempFile = _cache.Enabled ? targetFile + ".tmp" : targetFile; + } + + public bool CopyOutputFileIfChanged() + { + if (!_cache.Enabled) + return true; + + try + { + if (!File.Exists(TempFile)) + throw new LogAsErrorException($"Could not find the temporary file {TempFile} for target file {TargetFile}. Look for any errors/warnings generated earlier in the build."); + + if (!_cache.ShouldCopy(this, out string? cause)) + { + _cache.Log.LogMessage(MessageImportance.Low, $"Skipping copying over {TargetFile} as the contents are unchanged"); + return false; + } + + if (File.Exists(TargetFile)) + File.Delete(TargetFile); + + File.Copy(TempFile, TargetFile); + + _cache.Log.LogMessage(MessageImportance.Low, $"Copying {TempFile} to {TargetFile} because {cause}"); + return true; + } + finally + { + File.Delete(TempFile); + } + } + +} + +public enum MonoAotMode +{ + Normal, + JustInterp, + Full, + FullInterp, + Hybrid, + LLVMOnly, + LLVMOnlyInterp +} + +public enum MonoAotOutputType +{ + ObjectFile, + AsmOnly, + Library, +} + +public enum MonoAotLibraryFormat +{ + Dll, + Dylib, + So, +} + +public enum MonoAotModulesTableLanguage +{ + C, + ObjC +} + +internal sealed class CompilerCache +{ + public CompilerCache() => FileHashes = new(); + public CompilerCache(IDictionary oldHashes) + => FileHashes = new(oldHashes); + + [JsonPropertyName("file_hashes")] + public ConcurrentDictionary FileHashes { get; set; } +} diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj new file mode 100644 index 00000000000000..87a6349afea5e9 --- /dev/null +++ b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj @@ -0,0 +1,45 @@ + + + $(TargetFrameworkForNETCoreTasks);$(TargetFrameworkForNETFrameworkTasks) + Library + true + false + enable + $(NoWarn),CA1050 + + + $(NoWarn),CS8604,CS8602 + true + + + + + + + + + + + + + + + PreserveNewest + + + + + + + <_PublishFramework Remove="@(_PublishFramework)" /> + <_PublishFramework Include="$(TargetFrameworks)" /> + + + + + + + + + From 582144351d2088d76a9f50d3d4bbfb53b18c7d8b Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Tue, 12 Jul 2022 15:42:03 -0400 Subject: [PATCH 02/13] Add NetTraceToMibcConverter task and adjust build --- Directory.Build.props | 2 + eng/testing/tests.mobile.targets | 4 + .../msbuild/android/build/AndroidApp.props | 1 + .../msbuild/android/build/AndroidApp.targets | 26 +- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 136 -- .../AotCompilerTask/MonoAOTCompiler.csproj | 3 + src/tasks/Common/CompilerCache.cs | 19 + src/tasks/Common/FileCache.cs | 91 ++ src/tasks/Common/ProxyFile.cs | 54 + .../NetTraceToMibcConverter.cs | 1225 +---------------- .../NetTraceToMibcConverter.csproj | 25 +- 11 files changed, 243 insertions(+), 1343 deletions(-) create mode 100644 src/tasks/Common/CompilerCache.cs create mode 100644 src/tasks/Common/FileCache.cs create mode 100644 src/tasks/Common/ProxyFile.cs diff --git a/Directory.Build.props b/Directory.Build.props index f3b2b5a217d83c..c70619a9cb8ecb 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -100,6 +100,7 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'NetTraceToMibcConverter', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll')) @@ -113,6 +114,7 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmAppHost', 'wasm', '$(Configuration)')) $([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) + $([MSBuild]::NormalizePath('$(NetTraceToMibcConverterDir)', 'NetTraceToMibcConverter.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) $([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)')) diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index 96366a7e965e6e..aade6dc5fa2553 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -44,6 +44,10 @@ $(AdditionalXHarnessArguments) --expected-exit-code $(ExpectedExitCode) + + $(DiagnosticPorts),$(DiagnosticStartup) + + $(AdditionalXHarnessArguments) --arg=-m=$(XUnitMethodName) diff --git a/src/mono/msbuild/android/build/AndroidApp.props b/src/mono/msbuild/android/build/AndroidApp.props index 4e2b2a5720f0f5..3e8f62b3e0a4bc 100644 --- a/src/mono/msbuild/android/build/AndroidApp.props +++ b/src/mono/msbuild/android/build/AndroidApp.props @@ -9,6 +9,7 @@ _InitializeCommonProperties; _BeforeAndroidBuildApp; _AndroidResolveReferences; + _AndroidPrepareProfiledAot; _AndroidAotCompileApp; _AndroidGenerateAppBundle; _AfterAndroidBuildApp diff --git a/src/mono/msbuild/android/build/AndroidApp.targets b/src/mono/msbuild/android/build/AndroidApp.targets index 3ce50e882ceb61..a58da8bd2a7c76 100644 --- a/src/mono/msbuild/android/build/AndroidApp.targets +++ b/src/mono/msbuild/android/build/AndroidApp.targets @@ -1,6 +1,8 @@ + @@ -37,6 +39,7 @@ <_AOTMode Condition="'$(UseMonoJustInterp)' != 'true'">Full <_AOTMode Condition="'$(UseMonoJustInterp)' == 'true'">JustInterp + <_AOTMode Condition="'$(RunProfileAOT)' == 'true'">Normal @@ -97,6 +100,7 @@ Assemblies="@(_AotInputAssemblies)" AotModulesTablePath="$(AndroidAppBundleDir)\modules.c" IntermediateOutputPath="$(_MobileIntermediateOutputPath)" + MibcProfilePath="$(_MibcProfilePath)" UseLLVM="$(MonoEnableLLVM)" LLVMPath="$(MonoAotCrossDir)"> @@ -107,23 +111,19 @@ - - - - + CacheFilePath="$(IntermediateOutputPath)netrace_to_mibc_converter_cache.json" + OutputDir="$(_MobileIntermediateOutputPath)"> + + + @@ -138,6 +138,8 @@ ForceAOT="$(RunAOTCompilation)" ForceInterpreter="$(MonoForceInterpreter)" StripDebugSymbols="False" + RuntimeComponents="$(RuntimeComponents)" + DiagnosticPorts="$(DiagnosticPorts)" OutputDir="$(AndroidAppBundleDir)" AppDir="$(AndroidAppDir)"> diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index ab7b43be88c278..35e1f34181a2cb 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -1146,132 +1146,6 @@ public PrecompileArguments(string ResponseFilePath, IDictionary } } -internal sealed class FileCache -{ - private CompilerCache? _newCache; - private CompilerCache? _oldCache; - - public bool Enabled { get; } - public TaskLoggingHelper Log { get; } - - public FileCache(string? cacheFilePath, TaskLoggingHelper log) - { - Log = log; - if (string.IsNullOrEmpty(cacheFilePath)) - { - Log.LogMessage(MessageImportance.Low, $"Disabling cache, because CacheFilePath is not set"); - return; - } - - Enabled = true; - if (File.Exists(cacheFilePath)) - { - _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), - typeof(CompilerCache), - new JsonSerializerOptions()); - } - - _oldCache ??= new(); - _newCache = new(_oldCache.FileHashes); - } - - public bool UpdateAndCheckHasFileChanged(string filePath, string newHash) - { - if (!Enabled) - throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); - - _newCache!.FileHashes[filePath] = newHash; - return !_oldCache!.FileHashes.TryGetValue(filePath, out string? oldHash) || oldHash != newHash; - } - - public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause) - { - if (!Enabled) - throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); - - cause = null; - - string newHash = Utils.ComputeHash(proxyFile.TempFile); - _newCache!.FileHashes[proxyFile.TargetFile] = newHash; - - if (!File.Exists(proxyFile.TargetFile)) - { - cause = $"the output file didn't exist"; - return true; - } - - string? oldHash; - if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) - oldHash = Utils.ComputeHash(proxyFile.TargetFile); - - if (oldHash != newHash) - { - cause = $"hash for the file changed"; - return true; - } - - return false; - } - - public bool Save(string? cacheFilePath) - { - if (!Enabled || string.IsNullOrEmpty(cacheFilePath)) - return false; - - var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); - File.WriteAllText(cacheFilePath!, json); - return true; - } - - public ProxyFile NewFile(string targetFile) => new ProxyFile(targetFile, this); -} - -internal sealed class ProxyFile -{ - public string TargetFile { get; } - public string TempFile { get; } - private FileCache _cache; - - public ProxyFile(string targetFile, FileCache cache) - { - _cache = cache; - this.TargetFile = targetFile; - this.TempFile = _cache.Enabled ? targetFile + ".tmp" : targetFile; - } - - public bool CopyOutputFileIfChanged() - { - if (!_cache.Enabled) - return true; - - if (!File.Exists(TempFile)) - throw new LogAsErrorException($"Could not find the temporary file {TempFile} for target file {TargetFile}. Look for any errors/warnings generated earlier in the build."); - - try - { - if (!_cache.ShouldCopy(this, out string? cause)) - { - _cache.Log.LogMessage(MessageImportance.Low, $"Skipping copying over {TargetFile} as the contents are unchanged"); - return false; - } - - if (File.Exists(TargetFile)) - File.Delete(TargetFile); - - File.Copy(TempFile, TargetFile); - - _cache.Log.LogMessage(MessageImportance.Low, $"Copying {TempFile} to {TargetFile} because {cause}"); - return true; - } - finally - { - _cache.Log.LogMessage(MessageImportance.Low, $"Deleting temp file {TempFile}"); - File.Delete(TempFile); - } - } - -} - public enum MonoAotMode { Normal, @@ -1302,13 +1176,3 @@ public enum MonoAotModulesTableLanguage C, ObjC } - -internal sealed class CompilerCache -{ - public CompilerCache() => FileHashes = new(); - public CompilerCache(IDictionary oldHashes) - => FileHashes = new(oldHashes); - - [JsonPropertyName("file_hashes")] - public ConcurrentDictionary FileHashes { get; set; } -} diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj b/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj index 87a6349afea5e9..28e5cc265c80aa 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.csproj @@ -18,6 +18,9 @@ + + + diff --git a/src/tasks/Common/CompilerCache.cs b/src/tasks/Common/CompilerCache.cs new file mode 100644 index 00000000000000..22e7e6c87122cd --- /dev/null +++ b/src/tasks/Common/CompilerCache.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Build.Utilities; + +internal sealed class CompilerCache +{ + public CompilerCache() => FileHashes = new(); + public CompilerCache(IDictionary oldHashes) + => FileHashes = new(oldHashes); + + [JsonPropertyName("file_hashes")] + public ConcurrentDictionary FileHashes { get; set; } +} diff --git a/src/tasks/Common/FileCache.cs b/src/tasks/Common/FileCache.cs new file mode 100644 index 00000000000000..098a4b7c5528b3 --- /dev/null +++ b/src/tasks/Common/FileCache.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text.Json; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +internal sealed class FileCache +{ + private CompilerCache? _newCache; + private CompilerCache? _oldCache; + + public bool Enabled { get; } + public TaskLoggingHelper Log { get; } + + public FileCache(string? cacheFilePath, TaskLoggingHelper log) + { + Log = log; + if (string.IsNullOrEmpty(cacheFilePath)) + { + Log.LogMessage(MessageImportance.Low, $"Disabling cache, because CacheFilePath is not set"); + return; + } + + //Enabled = true; + if (File.Exists(cacheFilePath)) + { + _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), + typeof(CompilerCache), + new JsonSerializerOptions()); + } + + _oldCache ??= new(); + _newCache = new(_oldCache.FileHashes); + } + + public bool UpdateAndCheckHasFileChanged(string filePath, string newHash) + { + if (!Enabled) + throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); + + _newCache!.FileHashes[filePath] = newHash; + return !_oldCache!.FileHashes.TryGetValue(filePath, out string? oldHash) || oldHash != newHash; + } + + public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause) + { + if (!Enabled) + throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); + + cause = null; + + string newHash = Utils.ComputeHash(proxyFile.TempFile); + _newCache!.FileHashes[proxyFile.TargetFile] = newHash; + + if (!File.Exists(proxyFile.TargetFile)) + { + cause = $"the output file didn't exist"; + return true; + } + + string? oldHash; + if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) + oldHash = Utils.ComputeHash(proxyFile.TargetFile); + + if (oldHash != newHash) + { + cause = $"hash for the file changed"; + return true; + } + + return false; + } + + public bool Save(string? cacheFilePath) + { + if (!Enabled || string.IsNullOrEmpty(cacheFilePath)) + return false; + + var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(cacheFilePath!, json); + return true; + } + + public ProxyFile NewFile(string targetFile) => new ProxyFile(targetFile, this); +} diff --git a/src/tasks/Common/ProxyFile.cs b/src/tasks/Common/ProxyFile.cs new file mode 100644 index 00000000000000..79d96c83dada6a --- /dev/null +++ b/src/tasks/Common/ProxyFile.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.IO; +using System.Text.Json; +using Microsoft.Build.Framework; + +internal sealed class ProxyFile +{ + public string TargetFile { get; } + public string TempFile { get; } + private FileCache _cache; + + public ProxyFile(string targetFile, FileCache cache) + { + _cache = cache; + this.TargetFile = targetFile; + this.TempFile = _cache.Enabled ? targetFile + ".tmp" : targetFile; + } + + public bool CopyOutputFileIfChanged() + { + if (!_cache.Enabled) + return true; + + if (!File.Exists(TempFile)) + throw new LogAsErrorException($"Could not find the temporary file {TempFile} for target file {TargetFile}. Look for any errors/warnings generated earlier in the build."); + + try + { + if (!_cache.ShouldCopy(this, out string? cause)) + { + _cache.Log.LogMessage(MessageImportance.Low, $"Skipping copying over {TargetFile} as the contents are unchanged"); + return false; + } + + if (File.Exists(TargetFile)) + File.Delete(TargetFile); + + File.Copy(TempFile, TargetFile); + + _cache.Log.LogMessage(MessageImportance.Low, $"Copying {TempFile} to {TargetFile} because {cause}"); + return true; + } + finally + { + _cache.Log.LogMessage(MessageImportance.Low, $"Deleting temp file {TempFile}"); + File.Delete(TempFile); + } + } +} diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs index 6d639794d3289f..0c3e1229dca658 100644 --- a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs +++ b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs @@ -17,188 +17,25 @@ using System.Reflection.PortableExecutable; using System.Text.Json.Serialization; -public class MonoAOTCompiler : Microsoft.Build.Utilities.Task +public class NetTraceToMibcConverter : Microsoft.Build.Utilities.Task { - /// - /// Path to AOT cross-compiler binary (mono-aot-cross) - /// - [Required] - public string CompilerBinaryPath { get; set; } = ""!; - - /// - /// Assemblies to be AOTd. They need to be in a self-contained directory. - /// - /// Metadata: - /// - AotArguments: semicolon-separated list of options that will be passed to --aot= - /// - ProcessArguments: semicolon-separated list of options that will be passed to the AOT compiler itself - /// - [Required] - public ITaskItem[] Assemblies { get; set; } = Array.Empty(); /// - /// Paths to be passed as MONO_PATH environment variable, when running mono-cross-aot. - /// These are in addition to the directory containing the assembly being precompiled. - /// - /// MONO_PATH=${dir_containing_assembly}:${AdditionalAssemblySearchPaths} - /// + /// Path to the mibc converter tool /// - public string[]? AdditionalAssemblySearchPaths { get; set; } + public string? MibcConverterBinaryPath { get; set; } /// - /// Directory where the AOT'ed files will be emitted + /// Entries for assemblies referenced in a .nettrace file. Important when you run traces against an executable on a different machine / device /// - [NotNull] [Required] - public string? OutputDir { get; set; } - - /// - /// Target triple passed to the AOT compiler. - /// - public string? Triple { get; set; } - - /// - /// Assemblies which were AOT compiled. - /// - /// Successful AOT compilation will set the following metadata on the items: - /// - AssemblerFile (when using OutputType=AsmOnly) - /// - ObjectFile (when using OutputType=Normal) - /// - LibraryFile (when using OutputType=Library) - /// - AotDataFile (when using UseAotDataFile=true) - /// - LlvmObjectFile (if using LLVM) - /// - LlvmBitcodeFile (if using LLVM-only) - /// - [Output] - public ITaskItem[]? CompiledAssemblies { get; set; } - - /// - /// Disable parallel AOT compilation - /// - public bool DisableParallelAot { get; set; } - - /// - /// Use LLVM for AOT compilation. - /// The cross-compiler must be built with LLVM support - /// - public bool UseLLVM { get; set; } - - /// - /// This instructs the AOT code generator to output certain data constructs into a separate file. This can reduce the executable images some five to twenty percent. - /// Developers need to then ship the resulting aotdata as a resource and register a hook to load the data on demand by using the mono_install_load_aot_data_hook() method. - /// Defaults to true. - /// - public bool UseAotDataFile { get; set; } = true; - - /// - /// Create an ELF object file (.o) or .s file which can be statically linked into an executable when embedding the mono runtime. - /// Only valid if OutputType is ObjectFile or AsmOnly. - /// - public bool UseStaticLinking { get; set; } - - /// - /// When this option is specified, icalls (internal calls made from the standard library into the mono runtime code) are invoked directly instead of going through the operating system symbol lookup operation. - /// This requires UseStaticLinking=true. - /// - public bool UseDirectIcalls { get; set; } - - /// - /// When this option is specified, P/Invoke methods are invoked directly instead of going through the operating system symbol lookup operation - /// This requires UseStaticLinking=true. - /// - public bool UseDirectPInvoke { get; set; } - - /// - /// Instructs the AOT compiler to emit DWARF debugging information. - /// - public bool UseDwarfDebug { get; set; } - - /// - /// Path to Dotnet PGO binary (dotnet-pgo) - /// - public string? PgoBinaryPath { get; set; } + public ITaskItem[] Assemblies { get; set; } = Array.Empty(); /// /// NetTrace file to use when invoking dotnet-pgo for /// - public string? NetTracePath { get; set; } - - /// - /// File to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. - /// - public string[]? AotProfilePath { get; set; } - - /// - /// Mibc file to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. - /// - public string[] MibcProfilePath { get; set; } = Array.Empty(); - - /// - /// List of profilers to use. - /// - public string[]? Profilers { get; set; } - - /// - /// Generate a file containing mono_aot_register_module() calls for each AOT module which can be compiled into the app embedding mono. - /// If set, this implies UseStaticLinking=true. - /// - public string? AotModulesTablePath { get; set; } - - /// - /// Source code language of the AOT modules table. Supports "C" or "ObjC". - /// Defaults to "C". - /// - public string? AotModulesTableLanguage { get; set; } = nameof(MonoAotModulesTableLanguage.C); - - /// - /// Choose between 'Normal', 'JustInterp', 'Full', 'FullInterp', 'Hybrid', 'LLVMOnly', 'LLVMOnlyInterp'. - /// LLVMOnly means to use only LLVM for FullAOT, AOT result will be a LLVM Bitcode file (the cross-compiler must be built with LLVM support) - /// The "interp" options ('LLVMOnlyInterp' and 'FullInterp') mean generate necessary support to fall back to interpreter if AOT code is not possible for some methods. - /// The difference between 'JustInterp' and 'FullInterp' is that 'FullInterp' will AOT all the methods in the given assemblies, while 'JustInterp' will only AOT the wrappers and trampolines necessary for the runtime to execute the managed methods using the interpreter and to interoperate with P/Invokes and unmanaged callbacks. - /// - public string Mode { get; set; } = nameof(MonoAotMode.Normal); - - /// - /// Choose between 'ObjectFile', 'AsmOnly', 'Library' - /// ObjectFile means the AOT compiler will produce an .o object file, AsmOnly will produce .s assembly code and Library will produce a .so/.dylib/.dll shared library. - /// - public string OutputType { get; set; } = nameof(MonoAotOutputType.ObjectFile); - - /// - /// Choose between 'Dll', 'Dylib', 'So'. Only valid if OutputType is Library. - /// Dll means the AOT compiler will produce a Windows PE .dll file, Dylib means an Apple Mach-O .dylib and So means a Linux/Android ELF .so file. - /// - public string? LibraryFormat { get; set; } - - /// - /// Prefix that will be added to the library file name, e.g. to add 'lib' prefix required by some platforms. Only valid if OutputType is Library. - /// - public string LibraryFilePrefix { get; set; } = ""; - - /// - /// Path to the directory where LLVM binaries (opt and llc) are found. - /// It's required if UseLLVM is set - /// - public string? LLVMPath { get; set; } - - /// - /// Prepends a prefix to the name of tools ran by the AOT compiler, i.e. 'as'/'ld'. - /// - public string? ToolPrefix { get; set; } - - /// - /// Path to the directory where msym artifacts are stored. - /// - public string? MsymPath { get; set; } - - /// - /// The assembly whose AOT image will contained dedup-ed generic instances - /// - public string? DedupAssembly { get; set; } - - /// - /// Debug option in llvm aot mode - /// defaults to "nodebug" since some targes can't generate debug info - /// - public string? LLVMDebug { get; set; } = "nodebug"; + [Required] + public string NetTracePath { get; set; } = ""; /// /// File used to track hashes of assemblies, to act as a cache @@ -207,57 +44,25 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task public string? CacheFilePath { get; set; } /// - /// Passes additional, custom arguments to --aot - /// - public string? AotArguments { get; set; } - - /// - /// Passes temp-path to the AOT compiler + /// Directory where the mibc file will be placed /// - public string? TempPath { get; set; } - - /// - /// Passes ld-name to the AOT compiler, for use with UseLLVM=true - /// - public string? LdName { get; set; } - - /// - /// Passes ld-flags to the AOT compiler, for use with UseLLVM=true - /// - public string? LdFlags { get; set; } + [NotNull] + [Required] + public string? OutputDir { get; set; } /// - /// Specify WorkingDirectory for the AOT compiler + /// The path to the mibc file generated from the converter. /// - public string? WorkingDirectory { get; set; } - - [Required] - public string IntermediateOutputPath { get; set; } = string.Empty; - [Output] - public string[]? FileWrites { get; private set; } - - private static readonly Encoding s_utf8Encoding = new UTF8Encoding(false); - - private List _fileWrites = new(); - - private IList? _assembliesToCompile; - private ConcurrentDictionary compiledAssemblies = new(); - - private MonoAotMode parsedAotMode; - private MonoAotOutputType parsedOutputType; - private MonoAotLibraryFormat parsedLibraryFormat; - private MonoAotModulesTableLanguage parsedAotModulesTableLanguage; + public string? MibcProfilePath { get; set; } private FileCache? _cache; - private int _numCompiled; - private int _totalNumAssemblies; private bool ProcessAndValidateArguments() { - if (!File.Exists(CompilerBinaryPath)) + if (!File.Exists(MibcConverterBinaryPath)) { - Log.LogError($"{nameof(CompilerBinaryPath)}='{CompilerBinaryPath}' doesn't exist."); + Log.LogError($"{nameof(MibcConverterBinaryPath)}='{MibcConverterBinaryPath}' doesn't exist."); return false; } @@ -267,131 +72,12 @@ private bool ProcessAndValidateArguments() return false; } - // A relative path might be used along with WorkingDirectory, - // only call Path.GetFullPath() if WorkingDirectory is blank. - if (string.IsNullOrEmpty(WorkingDirectory) && !Path.IsPathRooted(OutputDir)) - OutputDir = Path.GetFullPath(OutputDir); - - if (!Directory.Exists(OutputDir)) - { - Log.LogError($"OutputDir={OutputDir} doesn't exist"); - return false; - } - - if (!Directory.Exists(IntermediateOutputPath)) - Directory.CreateDirectory(IntermediateOutputPath); - - if (!string.IsNullOrEmpty(NetTracePath)) - { - if (!File.Exists(NetTracePath)) - { - Log.LogError($"NetTracePath={nameof(NetTracePath)} doesn't exist"); - return false; - } - if (!File.Exists(PgoBinaryPath)) - { - Log.LogError($"NetTracePath was provided, but {nameof(PgoBinaryPath)}='{PgoBinaryPath}' doesn't exist"); - return false; - } - } - - if (AotProfilePath != null) - { - foreach (var path in AotProfilePath) - { - if (!File.Exists(path)) - { - Log.LogError($"AotProfilePath '{path}' doesn't exist."); - return false; - } - } - } - - foreach (var path in MibcProfilePath) - { - if (!File.Exists(path)) - { - Log.LogError($"MibcProfilePath '{path}' doesn't exist."); - return false; - } - } - - if (UseLLVM) - { - if (string.IsNullOrEmpty(LLVMPath)) - // prevent using some random llc/opt from PATH (installed with clang) - throw new LogAsErrorException($"'{nameof(LLVMPath)}' is required when '{nameof(UseLLVM)}' is true."); - - if (!Directory.Exists(LLVMPath)) - { - Log.LogError($"Could not find LLVMPath=${LLVMPath}"); - return false; - } - } - - if (!Enum.TryParse(Mode, true, out parsedAotMode)) + if (!File.Exists(NetTracePath)) { - Log.LogError($"Unknown Mode value: {Mode}. '{nameof(Mode)}' must be one of: {string.Join(",", Enum.GetNames(typeof(MonoAotMode)))}"); + Log.LogError($"{nameof(NetTracePath)}='{NetTracePath}' doesn't exist"); return false; } - switch (OutputType) - { - case "ObjectFile": parsedOutputType = MonoAotOutputType.ObjectFile; break; - case "AsmOnly": parsedOutputType = MonoAotOutputType.AsmOnly; break; - case "Library": parsedOutputType = MonoAotOutputType.Library; break; - case "Normal": - Log.LogWarning($"'{nameof(OutputType)}=Normal' is deprecated, use 'ObjectFile' instead."); - parsedOutputType = MonoAotOutputType.ObjectFile; break; - default: - throw new LogAsErrorException($"'{nameof(OutputType)}' must be one of: '{nameof(MonoAotOutputType.ObjectFile)}', '{nameof(MonoAotOutputType.AsmOnly)}', '{nameof(MonoAotOutputType.Library)}'. Received: '{OutputType}'."); - } - - switch (LibraryFormat) - { - case "Dll": parsedLibraryFormat = MonoAotLibraryFormat.Dll; break; - case "Dylib": parsedLibraryFormat = MonoAotLibraryFormat.Dylib; break; - case "So": parsedLibraryFormat = MonoAotLibraryFormat.So; break; - default: - if (parsedOutputType == MonoAotOutputType.Library) - throw new LogAsErrorException($"'{nameof(LibraryFormat)}' must be one of: '{nameof(MonoAotLibraryFormat.Dll)}', '{nameof(MonoAotLibraryFormat.Dylib)}', '{nameof(MonoAotLibraryFormat.So)}'. Received: '{LibraryFormat}'."); - break; - } - - if (parsedAotMode == MonoAotMode.LLVMOnly && !UseLLVM) - { - throw new LogAsErrorException($"'{nameof(UseLLVM)}' must be true when '{nameof(Mode)}' is {nameof(MonoAotMode.LLVMOnly)}."); - } - - switch (AotModulesTableLanguage) - { - case "C": parsedAotModulesTableLanguage = MonoAotModulesTableLanguage.C; break; - case "ObjC": parsedAotModulesTableLanguage = MonoAotModulesTableLanguage.ObjC; break; - default: - throw new LogAsErrorException($"'{nameof(AotModulesTableLanguage)}' must be one of: '{nameof(MonoAotModulesTableLanguage.C)}', '{nameof(MonoAotModulesTableLanguage.ObjC)}'. Received: '{AotModulesTableLanguage}'."); - } - - if (!string.IsNullOrEmpty(AotModulesTablePath)) - { - // AOT modules for static linking, needs the aot modules table - UseStaticLinking = true; - } - - if (UseDirectIcalls && !UseStaticLinking) - { - throw new LogAsErrorException($"'{nameof(UseDirectIcalls)}' can only be used with '{nameof(UseStaticLinking)}=true'."); - } - - if (UseDirectPInvoke && !UseStaticLinking) - { - throw new LogAsErrorException($"'{nameof(UseDirectPInvoke)}' can only be used with '{nameof(UseStaticLinking)}=true'."); - } - - if (UseStaticLinking && (parsedOutputType == MonoAotOutputType.Library)) - { - throw new LogAsErrorException($"'{nameof(OutputType)}=Library' can not be used with '{nameof(UseStaticLinking)}=true'."); - } - foreach (var asmItem in Assemblies) { string? fullPath = asmItem.GetMetadata("FullPath"); @@ -413,45 +99,13 @@ public override bool Execute() Log.LogError(laee.Message); return false; } - finally - { - if (_cache != null && _cache.Save(CacheFilePath!)) - _fileWrites.Add(CacheFilePath!); - FileWrites = _fileWrites.ToArray(); - } - } - - private bool ProcessNettrace(string netTraceFile) - { - var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(netTraceFile), ".mibc")); - - if (_cache!.Enabled) - { - string hash = Utils.ComputeHash(netTraceFile); - if (!_cache!.UpdateAndCheckHasFileChanged($"-mibc-source-file-{Path.GetFileName(netTraceFile)}", hash)) - { - Log.LogMessage(MessageImportance.Low, $"Skipping generating {outputMibcPath} from {netTraceFile} because source file hasn't changed"); - return true; - } - else - { - Log.LogMessage(MessageImportance.Low, $"Generating {outputMibcPath} from {netTraceFile} because the source file's hash has changed."); - } - } - - (int exitCode, string output) = Utils.TryRunProcess(Log, - PgoBinaryPath!, - $"create-mibc --trace {netTraceFile} --output {outputMibcPath}"); - - if (exitCode != 0) + /*finally { - Log.LogError($"dotnet-pgo({PgoBinaryPath}) failed for {netTraceFile}:{output}"); - return false; - } - - MibcProfilePath = MibcProfilePath.Append(outputMibcPath).ToArray(); - Log.LogMessage(MessageImportance.Low, $"Generated {outputMibcPath} from {PgoBinaryPath}"); - return true; + //if (_cache != null) + //{ + // _cache.Save(CacheFilePath!); + //} + }*/ } private bool ExecuteInternal() @@ -459,836 +113,53 @@ private bool ExecuteInternal() if (!ProcessAndValidateArguments()) return false; - _assembliesToCompile = EnsureAndGetAssembliesInTheSameDir(Assemblies); - _assembliesToCompile = FilterAssemblies(_assembliesToCompile); - - if (!string.IsNullOrEmpty(AotModulesTablePath) && !GenerateAotModulesTable(_assembliesToCompile, Profilers, AotModulesTablePath)) - return false; - - string? monoPaths = null; - if (AdditionalAssemblySearchPaths != null) - monoPaths = string.Join(Path.PathSeparator.ToString(), AdditionalAssemblySearchPaths); - _cache = new FileCache(CacheFilePath, Log); - if (!string.IsNullOrEmpty(NetTracePath) && !ProcessNettrace(NetTracePath)) + if (!ProcessNettrace(NetTracePath)) return false; - List argsList = new(); - foreach (var assemblyItem in _assembliesToCompile) - argsList.Add(GetPrecompileArgumentsFor(assemblyItem, monoPaths)); - - _totalNumAssemblies = _assembliesToCompile.Count; - if (CheckAllUpToDate(argsList)) - { - Log.LogMessage(MessageImportance.High, "Everything is up-to-date, nothing to precompile"); - - _fileWrites.AddRange(argsList.SelectMany(args => args.ProxyFiles).Select(pf => pf.TargetFile)); - foreach (var args in argsList) - compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly); - } - else - { - int allowedParallelism = DisableParallelAot ? 1 : Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - - /* - From: https://github.com/dotnet/runtime/issues/46146#issuecomment-754021690 - - Stephen Toub: - "As such, by default ForEach works on a scheme whereby each - thread takes one item each time it goes back to the enumerator, - and then after a few times of this upgrades to taking two items - each time it goes back to the enumerator, and then four, and - then eight, and so on. This ammortizes the cost of taking and - releasing the lock across multiple items, while still enabling - parallelization for enumerables containing just a few items. It - does, however, mean that if you've got a case where the body - takes a really long time and the work for every item is - heterogeneous, you can end up with an imbalance." - - The time taken by individual compile jobs here can vary a - lot, depending on various factors like file size. This can - create an imbalance, like mentioned above, and we can end up - in a situation where one of the partitions has a job that - takes very long to execute, by which time other partitions - have completed, so some cores are idle. But the idle - ones won't get any of the remaining jobs, because they are - all assigned to that one partition. - - Instead, we want to use work-stealing so jobs can be run by any partition. - */ - ParallelLoopResult result = Parallel.ForEach( - Partitioner.Create(argsList, EnumerablePartitionerOptions.NoBuffering), - new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (args, state) => PrecompileLibraryParallel(args, state)); - - if (result.IsCompleted) - { - int numUnchanged = _totalNumAssemblies - _numCompiled; - if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) - Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); - } - else if (!Log.HasLoggedErrors) - { - Log.LogError($"Precompiling failed due to unknown reasons. Check log for more info"); - } - } - - CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray(); return !Log.HasLoggedErrors; } - private static bool CheckAllUpToDate(IList argsList) + private bool ProcessNettrace(string netTraceFile) { - foreach (var args in argsList) - { - // compare original assembly vs it's outputs.. all it's outputs! - string assemblyPath = args.AOTAssembly.GetMetadata("FullPath"); - if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile))) - return false; - } - - return true; - - static bool IsNewerThanOutput(string inFile, string outFile) - => !File.Exists(inFile) || !File.Exists(outFile) || - (File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile)); - } + var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(netTraceFile), ".mibc")); - private IList FilterAssemblies(IEnumerable assemblies) - { - List filteredAssemblies = new(); - foreach (var asmItem in assemblies) + if (_cache!.Enabled) { - if (ShouldSkip(asmItem)) + string hash = Utils.ComputeHash(netTraceFile); + if (!_cache!.UpdateAndCheckHasFileChanged($"-mibc-source-file-{Path.GetFileName(netTraceFile)}", hash)) { - if (parsedAotMode == MonoAotMode.LLVMOnly) - throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {asmItem.ItemSpec}"); - - Log.LogMessage(MessageImportance.Low, $"Skipping {asmItem.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true"); - continue; + Log.LogMessage(MessageImportance.Low, $"Skipping generating {outputMibcPath} from {netTraceFile} because source file hasn't changed"); + return true; } - - string assemblyPath = asmItem.GetMetadata("FullPath"); - using var assemblyFile = File.OpenRead(assemblyPath); - using PEReader reader = new(assemblyFile, PEStreamOptions.Default); - if (!reader.HasMetadata) + else { - Log.LogWarning($"Skipping unmanaged {assemblyPath} for AOT"); - continue; + Log.LogMessage(MessageImportance.Low, $"Generating {outputMibcPath} from {netTraceFile} because the source file's hash has changed."); } - - filteredAssemblies.Add(asmItem); - } - - return filteredAssemblies; - - static bool ShouldSkip(ITaskItem asmItem) - => bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip; - } - - private IList EnsureAndGetAssembliesInTheSameDir(IList assemblies) - { - string firstAsmDir = Path.GetDirectoryName(assemblies.First().GetMetadata("FullPath")) ?? string.Empty; - bool allInSameDir = assemblies.All(asm => Path.GetDirectoryName(asm.GetMetadata("FullPath")) == firstAsmDir); - if (allInSameDir) - return assemblies; - - // Copy to aot-in - - string aotInPath = Path.Combine(IntermediateOutputPath, "aot-in"); - Directory.CreateDirectory(aotInPath); - - List newAssemblies = new(); - foreach (var asmItem in assemblies) - { - string asmPath = asmItem.GetMetadata("FullPath"); - string newPath = Path.Combine(aotInPath, Path.GetFileName(asmPath)); - - // FIXME: delete files not in originalAssemblies though - // FIXME: or .. just delete the whole dir? - if (Utils.CopyIfDifferent(asmPath, newPath, useHash: true)) - Log.LogMessage(MessageImportance.Low, $"Copying {asmPath} to {newPath}"); - _fileWrites.Add(newPath); - - ITaskItem newAsm = new TaskItem(newPath); - asmItem.CopyMetadataTo(newAsm); - newAssemblies.Add(newAsm); - } - - return newAssemblies; - } - - private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, string? monoPaths) - { - string assembly = assemblyItem.GetMetadata("FullPath"); - string assemblyDir = Path.GetDirectoryName(assembly)!; - var aotAssembly = new TaskItem(assembly); - var aotArgs = new List(); - var processArgs = new List(); - bool isDedup = assembly == DedupAssembly; - List proxyFiles = new(capacity: 5); - - var a = assemblyItem.GetMetadata("AotArguments"); - if (a != null) - { - aotArgs.AddRange(a.Split(new char[]{ ';' }, StringSplitOptions.RemoveEmptyEntries)); } - var p = assemblyItem.GetMetadata("ProcessArguments"); - if (p != null) - { - processArgs.AddRange(p.Split(new char[]{ ';' }, StringSplitOptions.RemoveEmptyEntries)); - } - - processArgs.Add("--debug"); - - // add LLVM options - if (UseLLVM) + StringBuilder pgoArgsStr = new StringBuilder(string.Empty); + pgoArgsStr.Append($"create-mibc"); + pgoArgsStr.Append($" --trace {netTraceFile} "); + foreach (var refAsmItem in Assemblies) { - processArgs.Add("--llvm"); - - if (!string.IsNullOrEmpty(LLVMDebug)) - aotArgs.Add(LLVMDebug); - - aotArgs.Add($"llvm-path={LLVMPath}"); - } - else - { - processArgs.Add("--nollvm"); + string? fullPath = refAsmItem.GetMetadata("FullPath"); + pgoArgsStr.Append($" --reference \"{fullPath}\" "); } + pgoArgsStr.Append($" --output {outputMibcPath} "); + (int exitCode, string output) = Utils.TryRunProcess(Log, + MibcConverterBinaryPath!, + pgoArgsStr.ToString()); - if (UseStaticLinking) + if (exitCode != 0) { - aotArgs.Add($"static"); + Log.LogError($"dotnet-pgo({MibcConverterBinaryPath}) failed for {netTraceFile}:{output}"); + return false; } - if (UseDwarfDebug) - { - aotArgs.Add($"dwarfdebug"); - } - - if (!string.IsNullOrEmpty(Triple)) - { - aotArgs.Add($"mtriple={Triple}"); - } - - if (!string.IsNullOrEmpty(ToolPrefix)) - { - aotArgs.Add($"tool-prefix={ToolPrefix}"); - } - - string assemblyFilename = Path.GetFileName(assembly); - - if (isDedup) - { - aotArgs.Add($"dedup-include={assemblyFilename}"); - } - else if (!string.IsNullOrEmpty (DedupAssembly)) - { - aotArgs.Add("dedup-skip"); - } - - // compute output mode and file names - if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.LLVMOnlyInterp) - { - aotArgs.Add("llvmonly"); - - string llvmBitcodeFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.bc")); - ProxyFile proxyFile = _cache!.NewFile(llvmBitcodeFile); - proxyFiles.Add(proxyFile); - aotAssembly.SetMetadata("LlvmBitcodeFile", proxyFile.TargetFile); - - if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) - { - aotArgs.Add("interp"); - } - - if (parsedOutputType == MonoAotOutputType.AsmOnly) - { - aotArgs.Add("asmonly"); - aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); - } - else - { - aotArgs.Add($"outfile={proxyFile.TempFile}"); - } - } - else - { - if (parsedAotMode == MonoAotMode.Full || parsedAotMode == MonoAotMode.FullInterp) - { - aotArgs.Add("full"); - } - - if (parsedAotMode == MonoAotMode.Hybrid) - { - aotArgs.Add("hybrid"); - } - - if (parsedAotMode == MonoAotMode.FullInterp || parsedAotMode == MonoAotMode.JustInterp) - { - aotArgs.Add("interp"); - } - - switch (parsedOutputType) - { - case MonoAotOutputType.ObjectFile: - { - string objectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.o")); - ProxyFile proxyFile = _cache!.NewFile(objectFile); - proxyFiles.Add((proxyFile)); - aotArgs.Add($"outfile={proxyFile.TempFile}"); - aotAssembly.SetMetadata("ObjectFile", proxyFile.TargetFile); - } - break; - - case MonoAotOutputType.AsmOnly: - { - aotArgs.Add("asmonly"); - - string assemblerFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll.s")); - ProxyFile proxyFile = _cache!.NewFile(assemblerFile); - proxyFiles.Add(proxyFile); - aotArgs.Add($"outfile={proxyFile.TempFile}"); - aotAssembly.SetMetadata("AssemblerFile", proxyFile.TargetFile); - } - break; - - case MonoAotOutputType.Library: - { - string extension = parsedLibraryFormat switch { - MonoAotLibraryFormat.Dll => ".dll", - MonoAotLibraryFormat.Dylib => ".dylib", - MonoAotLibraryFormat.So => ".so", - _ => throw new ArgumentOutOfRangeException() - }; - string libraryFileName = $"{LibraryFilePrefix}{assemblyFilename}{extension}"; - string libraryFilePath = Path.Combine(OutputDir, libraryFileName); - ProxyFile proxyFile = _cache!.NewFile(libraryFilePath); - proxyFiles.Add(proxyFile); - - aotArgs.Add($"outfile={proxyFile.TempFile}"); - aotAssembly.SetMetadata("LibraryFile", proxyFile.TargetFile); - } - break; - - default: - throw new Exception($"Bug: Unhandled MonoAotOutputType: {parsedAotMode}"); - } - - if (UseLLVM) - { - string llvmObjectFile = Path.Combine(OutputDir, Path.ChangeExtension(assemblyFilename, ".dll-llvm.o")); - ProxyFile proxyFile = _cache.NewFile(llvmObjectFile); - proxyFiles.Add(proxyFile); - aotArgs.Add($"llvm-outfile={proxyFile.TempFile}"); - aotAssembly.SetMetadata("LlvmObjectFile", proxyFile.TargetFile); - } - } - - // pass msym-dir if specified - if (MsymPath != null) - { - aotArgs.Add($"msym-dir={MsymPath}"); - } - - if (UseAotDataFile) - { - string aotDataFile = Path.ChangeExtension(assembly, ".aotdata"); - ProxyFile proxyFile = _cache.NewFile(aotDataFile); - proxyFiles.Add(proxyFile); - aotArgs.Add($"data-outfile={proxyFile.TempFile}"); - aotAssembly.SetMetadata("AotDataFile", proxyFile.TargetFile); - } - - if (AotProfilePath?.Length > 0) - { - aotArgs.Add("profile-only"); - foreach (var path in AotProfilePath) - { - aotArgs.Add($"profile={path}"); - } - } - - if (MibcProfilePath.Length > 0) - { - aotArgs.Add("profile-only"); - foreach (var path in MibcProfilePath) - { - aotArgs.Add($"mibc-profile={path}"); - } - } - - if (!string.IsNullOrEmpty(AotArguments)) - { - aotArgs.Add(AotArguments); - } - - if (!string.IsNullOrEmpty(TempPath)) - { - aotArgs.Add($"temp-path={TempPath}"); - } - - if (!string.IsNullOrEmpty(LdName)) - { - aotArgs.Add($"ld-name={LdName}"); - } - - if (!string.IsNullOrEmpty(LdFlags)) - { - aotArgs.Add($"ld-flags={LdFlags}"); - } - - // we need to quote the entire --aot arguments here to make sure it is parsed - // on Windows as one argument. Otherwise it will be split up into multiple - // values, which wont work. - processArgs.Add($"\"--aot={string.Join(",", aotArgs)}\""); - - if (isDedup) - { - foreach (var aItem in _assembliesToCompile!) - processArgs.Add(aItem.ItemSpec); - } - else - { - if (string.IsNullOrEmpty(WorkingDirectory)) - { - processArgs.Add('"' + assemblyFilename + '"'); - } - else - { - // If WorkingDirectory is supplied, the caller could be passing in a relative path - // Use the original ItemSpec that was passed in. - processArgs.Add('"' + assemblyItem.ItemSpec + '"'); - } - } - - monoPaths = $"{assemblyDir}{Path.PathSeparator}{monoPaths}"; - var envVariables = new Dictionary - { - {"MONO_PATH", monoPaths }, - {"MONO_ENV_OPTIONS", string.Empty} // we do not want options to be provided out of band to the cross compilers - }; - - var responseFileContent = string.Join(" ", processArgs); - var responseFilePath = Path.GetTempFileName(); - using (var sw = new StreamWriter(responseFilePath, append: false, encoding: s_utf8Encoding)) - { - sw.WriteLine(responseFileContent); - } - - return new PrecompileArguments(ResponseFilePath: responseFilePath, - EnvironmentVariables: envVariables, - WorkingDir: string.IsNullOrEmpty(WorkingDirectory) ? assemblyDir : WorkingDirectory, - AOTAssembly: aotAssembly, - ProxyFiles: proxyFiles); - } - - private bool PrecompileLibrary(PrecompileArguments args) - { - string assembly = args.AOTAssembly.GetMetadata("FullPath"); - string output; - try - { - string msgPrefix = $"[{Path.GetFileName(assembly)}] "; - - // run the AOT compiler - (int exitCode, output) = Utils.TryRunProcess(Log, - CompilerBinaryPath, - $"--response=\"{args.ResponseFilePath}\"", - args.EnvironmentVariables, - args.WorkingDir, - silent: true, - debugMessageImportance: MessageImportance.Low, - label: Path.GetFileName(assembly)); - - var importance = exitCode == 0 ? MessageImportance.Low : MessageImportance.High; - // Log the command in a compact format which can be copy pasted - { - StringBuilder envStr = new StringBuilder(string.Empty); - foreach (KeyValuePair kvp in args.EnvironmentVariables) - envStr.Append($"{kvp.Key}={kvp.Value} "); - Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {args.WorkingDir}: {envStr}{CompilerBinaryPath} {File.ReadAllText(args.ResponseFilePath, s_utf8Encoding)}"); - } - - if (exitCode != 0) - { - Log.LogError($"Precompiling failed for {assembly}.{Environment.NewLine}{output}"); - return false; - } - - Log.LogMessage(importance, output); - } - catch (Exception ex) - { - Log.LogMessage(MessageImportance.Low, ex.ToString()); - Log.LogError($"Precompiling failed for {assembly}: {ex.Message}"); - return false; - } - finally - { - File.Delete(args.ResponseFilePath); - } - - bool copied = false; - foreach (var proxyFile in args.ProxyFiles) - { - if (!File.Exists(proxyFile.TempFile)) - { - Log.LogError($"Precompile command succeeded, but can't find the expected temporary output file - {proxyFile.TempFile} for {assembly}.{Environment.NewLine}{output}"); - return false; - } - - copied |= proxyFile.CopyOutputFileIfChanged(); - _fileWrites.Add(proxyFile.TargetFile); - } - - if (copied) - { - string copiedFiles = string.Join(", ", args.ProxyFiles.Select(tf => Path.GetFileName(tf.TargetFile))); - int count = Interlocked.Increment(ref _numCompiled); - Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assembly)} -> {copiedFiles}"); - } - - compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly); + MibcProfilePath = outputMibcPath; + Log.LogMessage(MessageImportance.Low, $"Generated {outputMibcPath} from {MibcConverterBinaryPath}"); return true; } - - private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopState state) - { - try - { - if (PrecompileLibrary(args)) - return; - } - catch (LogAsErrorException laee) - { - Log.LogError($"Precompiling failed for {args.AOTAssembly}: {laee.Message}"); - } - catch (Exception ex) - { - if (Log.HasLoggedErrors) - Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}"); - else - Log.LogError($"Precompiling failed for {args.AOTAssembly}: {ex}"); - } - - state.Break(); - } - - private bool GenerateAotModulesTable(IEnumerable assemblies, string[]? profilers, string outputFile) - { - var symbols = new List(); - foreach (var asm in assemblies) - { - string asmPath = asm.ItemSpec; - if (!File.Exists(asmPath)) - { - Log.LogError($"Could not find assembly {asmPath}"); - return false; - } - - if (!TryGetAssemblyName(asmPath, out string? assemblyName)) - return false; - - string symbolName = assemblyName.Replace ('.', '_').Replace ('-', '_').Replace(' ', '_'); - symbols.Add($"mono_aot_module_{symbolName}_info"); - } - - Directory.CreateDirectory(Path.GetDirectoryName(outputFile)!); - - string tmpAotModulesTablePath = Path.GetTempFileName(); - using (var writer = File.CreateText(tmpAotModulesTablePath)) - { - if (parsedAotModulesTableLanguage == MonoAotModulesTableLanguage.C) - { - writer.WriteLine("#include "); - - foreach (var symbol in symbols) - { - writer.WriteLine($"extern void *{symbol};"); - } - writer.WriteLine("void register_aot_modules ()"); - writer.WriteLine("{"); - foreach (var symbol in symbols) - { - writer.WriteLine($"\tmono_aot_register_module ({symbol});"); - } - writer.WriteLine("}"); - - foreach (var profiler in profilers ?? Enumerable.Empty()) - { - writer.WriteLine($"void mono_profiler_init_{profiler} (const char *desc);"); - writer.WriteLine("EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_" + profiler + " (const char *desc) { mono_profiler_init_" + profiler + " (desc); }"); - } - - if (parsedAotMode == MonoAotMode.LLVMOnly) - { - writer.WriteLine("#define EE_MODE_LLVMONLY 1"); - } - - if (parsedAotMode == MonoAotMode.LLVMOnlyInterp) - { - writer.WriteLine("#define EE_MODE_LLVMONLY_INTERP 1"); - } - } - else if (parsedAotModulesTableLanguage == MonoAotModulesTableLanguage.ObjC) - { - writer.WriteLine("#include "); - writer.WriteLine("#include "); - writer.WriteLine(""); - writer.WriteLine("#if TARGET_OS_IPHONE && (!TARGET_IPHONE_SIMULATOR || FORCE_AOT)"); - - foreach (var symbol in symbols) - { - writer.WriteLine($"extern void *{symbol};"); - } - - writer.WriteLine("void register_aot_modules (void)"); - writer.WriteLine("{"); - foreach (var symbol in symbols) - { - writer.WriteLine($"\tmono_aot_register_module ({symbol});"); - } - writer.WriteLine("}"); - writer.WriteLine("#endif"); - } - else - { - throw new NotSupportedException(); - } - } - - if (Utils.CopyIfDifferent(tmpAotModulesTablePath, outputFile, useHash: false)) - { - _fileWrites.Add(outputFile); - Log.LogMessage(MessageImportance.Low, $"Generated {outputFile}"); - } - - return true; - } - - private bool TryGetAssemblyName(string asmPath, [NotNullWhen(true)] out string? assemblyName) - { - assemblyName = null; - - try - { - using var fs = new FileStream(asmPath, FileMode.Open, FileAccess.Read); - using var peReader = new PEReader(fs); - MetadataReader mr = peReader.GetMetadataReader(); - assemblyName = mr.GetAssemblyDefinition().GetAssemblyName().Name; - - if (string.IsNullOrEmpty(assemblyName)) - { - Log.LogError($"Could not get assembly name for {asmPath}"); - return false; - } - - return true; - } - catch (Exception ex) - { - Log.LogError($"Failed to get assembly name for {asmPath}: {ex.Message}"); - return false; - } - } - - private static IList ConvertAssembliesDictToOrderedList(ConcurrentDictionary dict, IList originalAssemblies) - { - List outItems = new(originalAssemblies.Count); - foreach (ITaskItem item in originalAssemblies) - { - if (dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem)) - outItems.Add(dictItem); - } - return outItems; - } - - internal sealed class PrecompileArguments - { - public PrecompileArguments(string ResponseFilePath, IDictionary EnvironmentVariables, string WorkingDir, ITaskItem AOTAssembly, IList ProxyFiles) - { - this.ResponseFilePath = ResponseFilePath; - this.EnvironmentVariables = EnvironmentVariables; - this.WorkingDir = WorkingDir; - this.AOTAssembly = AOTAssembly; - this.ProxyFiles = ProxyFiles; - } - - public string ResponseFilePath { get; private set; } - public IDictionary EnvironmentVariables { get; private set; } - public string WorkingDir { get; private set; } - public ITaskItem AOTAssembly { get; private set; } - public IList ProxyFiles { get; private set; } - } -} - -internal sealed class FileCache -{ - private CompilerCache? _newCache; - private CompilerCache? _oldCache; - - public bool Enabled { get; } - public TaskLoggingHelper Log { get; } - - public FileCache(string? cacheFilePath, TaskLoggingHelper log) - { - Log = log; - if (string.IsNullOrEmpty(cacheFilePath)) - { - Log.LogMessage(MessageImportance.Low, $"Disabling cache, because CacheFilePath is not set"); - return; - } - - Enabled = true; - if (File.Exists(cacheFilePath)) - { - _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), - typeof(CompilerCache), - new JsonSerializerOptions()); - } - - _oldCache ??= new(); - _newCache = new(_oldCache.FileHashes); - } - - public bool UpdateAndCheckHasFileChanged(string filePath, string newHash) - { - if (!Enabled) - throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); - - _newCache!.FileHashes[filePath] = newHash; - return !_oldCache!.FileHashes.TryGetValue(filePath, out string? oldHash) || oldHash != newHash; - } - - public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause) - { - if (!Enabled) - throw new InvalidOperationException("Cache is not enabled. Make sure the cache file path is set"); - - cause = null; - - string newHash = Utils.ComputeHash(proxyFile.TempFile); - _newCache!.FileHashes[proxyFile.TargetFile] = newHash; - - if (!File.Exists(proxyFile.TargetFile)) - { - cause = $"the output file didn't exist"; - return true; - } - - string? oldHash; - if (!_oldCache!.FileHashes.TryGetValue(proxyFile.TargetFile, out oldHash)) - oldHash = Utils.ComputeHash(proxyFile.TargetFile); - - if (oldHash != newHash) - { - cause = $"hash for the file changed"; - return true; - } - - return false; - } - - public bool Save(string? cacheFilePath) - { - if (!Enabled || string.IsNullOrEmpty(cacheFilePath)) - return false; - - var json = JsonSerializer.Serialize (_newCache, new JsonSerializerOptions { WriteIndented = true }); - File.WriteAllText(cacheFilePath!, json); - return true; - } - - public ProxyFile NewFile(string targetFile) => new ProxyFile(targetFile, this); -} - -internal sealed class ProxyFile -{ - public string TargetFile { get; } - public string TempFile { get; } - private FileCache _cache; - - public ProxyFile(string targetFile, FileCache cache) - { - _cache = cache; - this.TargetFile = targetFile; - this.TempFile = _cache.Enabled ? targetFile + ".tmp" : targetFile; - } - - public bool CopyOutputFileIfChanged() - { - if (!_cache.Enabled) - return true; - - try - { - if (!File.Exists(TempFile)) - throw new LogAsErrorException($"Could not find the temporary file {TempFile} for target file {TargetFile}. Look for any errors/warnings generated earlier in the build."); - - if (!_cache.ShouldCopy(this, out string? cause)) - { - _cache.Log.LogMessage(MessageImportance.Low, $"Skipping copying over {TargetFile} as the contents are unchanged"); - return false; - } - - if (File.Exists(TargetFile)) - File.Delete(TargetFile); - - File.Copy(TempFile, TargetFile); - - _cache.Log.LogMessage(MessageImportance.Low, $"Copying {TempFile} to {TargetFile} because {cause}"); - return true; - } - finally - { - File.Delete(TempFile); - } - } - -} - -public enum MonoAotMode -{ - Normal, - JustInterp, - Full, - FullInterp, - Hybrid, - LLVMOnly, - LLVMOnlyInterp -} - -public enum MonoAotOutputType -{ - ObjectFile, - AsmOnly, - Library, -} - -public enum MonoAotLibraryFormat -{ - Dll, - Dylib, - So, -} - -public enum MonoAotModulesTableLanguage -{ - C, - ObjC -} - -internal sealed class CompilerCache -{ - public CompilerCache() => FileHashes = new(); - public CompilerCache(IDictionary oldHashes) - => FileHashes = new(oldHashes); - - [JsonPropertyName("file_hashes")] - public ConcurrentDictionary FileHashes { get; set; } } diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj index 87a6349afea5e9..293685bb00a58a 100644 --- a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj +++ b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj @@ -1,45 +1,34 @@ - $(TargetFrameworkForNETCoreTasks);$(TargetFrameworkForNETFrameworkTasks) + $(TargetFrameworkForNETCoreTasks) Library true false enable $(NoWarn),CA1050 - - - $(NoWarn),CS8604,CS8602 true - - + + + + - - - - - PreserveNewest - - + <_PublishFramework Remove="@(_PublishFramework)" /> - <_PublishFramework Include="$(TargetFrameworks)" /> + <_PublishFramework Include="$(TargetFramework)" /> - - - From 3363776701ec81e684a506bba5334e5f6b735dc4 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Mon, 18 Jul 2022 15:57:35 -0400 Subject: [PATCH 03/13] [Android] Introduce NetTraceToMibcConverter task & streamline testing targets NetTraceToMibcConverter - Used in profiled AOT scenarios where a .nettrace file is given as input and is converted to a .mibc file that can be fed into the AOT compiler. This previously was in the AotCompiler task, but for clarity purposes is now separated out. Streamline Android testing targets - The testing targets function the same, but are now structured similarly to iOS and Wasm. - Introduced new testing properties to support profiled AOT: RunProfileAOT - true/false to control the mode the aot compiler is set in. NetTracePath - The path to a .nettrace file that will be converted into a .mibc file and fed into the aot compiler RuntimeComponents - The list of native components to include in the test app build (diagnostics_tracing) DiagnosticsPorts - The ip address:port where the runtime will listen when running diagnostic tooling DiagnosticStartup - The mode the runtime will use at startup for diagnostic scenarios. Suspend will halt the app very early and wait, while nosuspend will wait for a connection, but not halt the runtime --- eng/Subsets.props | 1 + .../msbuild/android/build/AndroidApp.targets | 3 ++- ...t.NET.Runtime.MonoAOTCompiler.Task.pkgproj | 1 + .../Sdk/Sdk.props.in | 3 +++ .../AndroidAppBuilder/AndroidAppBuilder.cs | 6 +++++ src/tasks/AndroidAppBuilder/ApkBuilder.cs | 6 +++++ .../AndroidAppBuilder/Templates/monodroid.c | 9 +++++-- .../NetTraceToMibcConverter.cs | 27 ------------------- .../NetTraceToMibcConverter.csproj | 10 +++---- 9 files changed, 30 insertions(+), 36 deletions(-) diff --git a/eng/Subsets.props b/eng/Subsets.props index 66d0c6498665b2..45b2a49471e5c6 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -62,6 +62,7 @@ $(DefaultMonoSubsets)mono.wasmruntime+ $(DefaultMonoSubsets)mono.aotcross+ $(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages+mono.tools+ + $(DefaultMonoSubsets)host+ - + @@ -137,6 +137,7 @@ EnvironmentVariables="@(AndroidEnv)" ForceAOT="$(RunAOTCompilation)" ForceInterpreter="$(MonoForceInterpreter)" + ProfileAOT="$(RunProfileAOT)" StripDebugSymbols="False" RuntimeComponents="$(RuntimeComponents)" DiagnosticPorts="$(DiagnosticPorts)" diff --git a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj index e5a5b52244306c..2036fc6edb343c 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj +++ b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj @@ -7,6 +7,7 @@ + diff --git a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in index bc14a619299e1e..648f42aa2a4ecc 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in +++ b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in @@ -2,6 +2,9 @@ $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\MonoAOTCompiler.dll $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\MonoAOTCompiler.dll + $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\NetTraceToMibcConverter.dll + $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\NetTraceToMibcConverter.dll + diff --git a/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs b/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs index b2ae7276fe90b9..5e586f93c89183 100644 --- a/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs +++ b/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs @@ -38,6 +38,11 @@ public class AndroidAppBuilderTask : Task /// public bool ForceAOT { get; set; } + /// + /// Indicates if the assemblies to aot were gathered from a profile + /// + public bool ProfileAOT { get; set; } + /// /// Static linked runtime /// @@ -108,6 +113,7 @@ public override bool Execute() apkBuilder.KeyStorePath = KeyStorePath; apkBuilder.ForceInterpreter = ForceInterpreter; apkBuilder.ForceAOT = ForceAOT; + apkBuilder.ProfileAOT = ProfileAOT; apkBuilder.EnvironmentVariables = EnvironmentVariables; apkBuilder.StaticLinkedRuntime = StaticLinkedRuntime; apkBuilder.RuntimeComponents = RuntimeComponents; diff --git a/src/tasks/AndroidAppBuilder/ApkBuilder.cs b/src/tasks/AndroidAppBuilder/ApkBuilder.cs index 9ed098ac55a61e..5ce926e3e597d5 100644 --- a/src/tasks/AndroidAppBuilder/ApkBuilder.cs +++ b/src/tasks/AndroidAppBuilder/ApkBuilder.cs @@ -27,6 +27,7 @@ public class ApkBuilder public string? KeyStorePath { get; set; } public bool ForceInterpreter { get; set; } public bool ForceAOT { get; set; } + public bool ProfileAOT { get; set; } public ITaskItem[] EnvironmentVariables { get; set; } = Array.Empty(); public bool InvariantGlobalization { get; set; } public bool EnableRuntimeLogging { get; set; } @@ -322,6 +323,11 @@ public ApkBuilder(TaskLoggingHelper logger) } } + if (ProfileAOT) + { + defines.AppendLine("add_definitions(-DPROFILE_AOT=1)"); + } + if (!string.IsNullOrEmpty(DiagnosticPorts)) { defines.AppendLine("add_definitions(-DDIAGNOSTIC_PORTS=\"" + DiagnosticPorts + "\")"); diff --git a/src/tasks/AndroidAppBuilder/Templates/monodroid.c b/src/tasks/AndroidAppBuilder/Templates/monodroid.c index 302324efd82df6..6f88f1d0542145 100644 --- a/src/tasks/AndroidAppBuilder/Templates/monodroid.c +++ b/src/tasks/AndroidAppBuilder/Templates/monodroid.c @@ -273,9 +273,14 @@ mono_droid_runtime_init (const char* executable, int managed_argc, char* managed LOG_INFO("AOT Enabled"); #if STATIC_AOT register_aot_modules(); -#endif +#endif // STATIC_AOT + +#if PROFILE_AOT + mono_jit_set_aot_mode(MONO_AOT_MODE_NORMAL); +#else mono_jit_set_aot_mode(MONO_AOT_MODE_FULL); -#endif +#endif // PROFILE_AOT +#endif // FORCE_INTERPRETER MonoDomain *domain = mono_jit_init_version ("dotnet.android", "mobile"); assert (domain); diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs index 0c3e1229dca658..8c6c0e6a10d007 100644 --- a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs +++ b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs @@ -7,14 +7,12 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; -using System.Reflection.Metadata; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -using System.Reflection.PortableExecutable; using System.Text.Json.Serialization; public class NetTraceToMibcConverter : Microsoft.Build.Utilities.Task @@ -56,8 +54,6 @@ public class NetTraceToMibcConverter : Microsoft.Build.Utilities.Task [Output] public string? MibcProfilePath { get; set; } - private FileCache? _cache; - private bool ProcessAndValidateArguments() { if (!File.Exists(MibcConverterBinaryPath)) @@ -99,13 +95,6 @@ public override bool Execute() Log.LogError(laee.Message); return false; } - /*finally - { - //if (_cache != null) - //{ - // _cache.Save(CacheFilePath!); - //} - }*/ } private bool ExecuteInternal() @@ -113,8 +102,6 @@ private bool ExecuteInternal() if (!ProcessAndValidateArguments()) return false; - _cache = new FileCache(CacheFilePath, Log); - if (!ProcessNettrace(NetTracePath)) return false; @@ -125,20 +112,6 @@ private bool ProcessNettrace(string netTraceFile) { var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(netTraceFile), ".mibc")); - if (_cache!.Enabled) - { - string hash = Utils.ComputeHash(netTraceFile); - if (!_cache!.UpdateAndCheckHasFileChanged($"-mibc-source-file-{Path.GetFileName(netTraceFile)}", hash)) - { - Log.LogMessage(MessageImportance.Low, $"Skipping generating {outputMibcPath} from {netTraceFile} because source file hasn't changed"); - return true; - } - else - { - Log.LogMessage(MessageImportance.Low, $"Generating {outputMibcPath} from {netTraceFile} because the source file's hash has changed."); - } - } - StringBuilder pgoArgsStr = new StringBuilder(string.Empty); pgoArgsStr.Append($"create-mibc"); pgoArgsStr.Append($" --trace {netTraceFile} "); diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj index 293685bb00a58a..2829984e53cacc 100644 --- a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj +++ b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj @@ -1,6 +1,6 @@ - $(TargetFrameworkForNETCoreTasks) + $(TargetFrameworkForNETCoreTasks);$(TargetFrameworkForNETFrameworkTasks) Library true false @@ -14,18 +14,16 @@ - - - + - + <_PublishFramework Remove="@(_PublishFramework)" /> - <_PublishFramework Include="$(TargetFramework)" /> + <_PublishFramework Include="$(TargetFrameworks)" /> From 55c411a58a5caf285617f0c6edff579e9763ef5c Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Tue, 19 Jul 2022 14:51:26 -0400 Subject: [PATCH 04/13] Update src/mono/msbuild/android/build/AndroidApp.InTree.props Co-authored-by: Mitchell Hwang <16830051+mdh1418@users.noreply.github.com> --- src/mono/msbuild/android/build/AndroidApp.InTree.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/msbuild/android/build/AndroidApp.InTree.props b/src/mono/msbuild/android/build/AndroidApp.InTree.props index ff68f1085a0ac5..26fb16475833a8 100644 --- a/src/mono/msbuild/android/build/AndroidApp.InTree.props +++ b/src/mono/msbuild/android/build/AndroidApp.InTree.props @@ -1,7 +1,7 @@ - + $(NetCoreAppCurrent) false From 958b162f1a5f52cdf69a83a16442116721ac3832 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Tue, 19 Jul 2022 16:45:05 -0400 Subject: [PATCH 05/13] PR feedback --- eng/testing/tests.android.targets | 10 ------ eng/testing/tests.mobile.targets | 7 +--- .../android/build/AndroidApp.InTree.targets | 1 + .../msbuild/android/build/AndroidApp.targets | 36 ++++++++----------- .../NetTraceToMibcConverter.cs | 12 +++---- 5 files changed, 22 insertions(+), 44 deletions(-) diff --git a/eng/testing/tests.android.targets b/eng/testing/tests.android.targets index ca5251ed16da13..a8b279bf853178 100644 --- a/eng/testing/tests.android.targets +++ b/eng/testing/tests.android.targets @@ -21,22 +21,12 @@ - true AndroidTestRunner.dll $(PublishDir) $(BundleDir) - - arm64-v8a - armeabi-v7a - x86_64 - x86 - - AndroidTestRunner.dll - - 1 diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index aade6dc5fa2553..bb64a34cfd4eae 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -8,7 +8,6 @@ $([MSBuild]::NormalizePath('$(BundleDir)', '$(RunScriptOutputName)')) true - BundleTestAndroidApp Publish @@ -75,10 +74,6 @@ <_MobileIntermediateOutputPath>$(IntermediateOutputPath)mobile - - @@ -146,7 +141,7 @@ + DependsOnTargets="$(PublishTestAsSelfContainedDependsOn);ArchiveTests" /> + diff --git a/src/mono/msbuild/android/build/AndroidApp.targets b/src/mono/msbuild/android/build/AndroidApp.targets index 34b5e6492026fd..eb62d6e1c67bf5 100644 --- a/src/mono/msbuild/android/build/AndroidApp.targets +++ b/src/mono/msbuild/android/build/AndroidApp.targets @@ -1,8 +1,6 @@ - @@ -70,11 +68,6 @@ <_AotExcludeAssemblies Include="*System.Runtime.WindowsRuntime.dll" /> - - <_InternalForceInterpret>%(_AndroidAssembliesInternal._InternalForceInterpret) - <_IsNative>%(_AndroidAssembliesInternal._IsNative) - - <_AotInputAssemblies Include="@(_AndroidAssembliesInternal)" Condition="'%(_AndroidAssembliesInternal._InternalForceInterpret)' != 'true'"> $(AotArguments) @@ -88,6 +81,20 @@ + + + + + + + @@ -111,21 +118,6 @@ - - - - - - - - Date: Wed, 20 Jul 2022 08:50:51 -0400 Subject: [PATCH 06/13] Cleanup on asile 5 --- eng/testing/tests.mobile.targets | 2 +- src/mono/msbuild/android/build/AndroidApp.targets | 3 --- src/mono/msbuild/apple/build/AppleApp.targets | 13 ------------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index bb64a34cfd4eae..e0aa2217e14800 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -141,7 +141,7 @@ + DependsOnTargets="$(PublishTestAsSelfContainedDependsOn);BundleTestAppTargets;ArchiveTests" /> - @@ -56,8 +55,6 @@ - - diff --git a/src/mono/msbuild/apple/build/AppleApp.targets b/src/mono/msbuild/apple/build/AppleApp.targets index 89ec14ff767dec..d9a9841b792c8e 100644 --- a/src/mono/msbuild/apple/build/AppleApp.targets +++ b/src/mono/msbuild/apple/build/AppleApp.targets @@ -24,15 +24,7 @@ <_AppleRuntimeConfigFilePath Condition="'$(_AppleRuntimeConfigFilePath)' == ''">$([MSBuild]::NormalizePath($(AppleAppDir), '$(AssemblyName).runtimeconfig.json')) <_ParsedRuntimeConfigFilePath Condition="'$(_ParsedRuntimeConfigFilePath)' == ''">$([MSBuild]::NormalizePath($(AppleAppDir), 'runtimeconfig.bin')) - @@ -83,11 +75,6 @@ <_AotExcludeAssemblies Include="*System.Runtime.WindowsRuntime.dll" /> - - - <_InternalForceInterpret>%(_AppleAssembliesInternal._InternalForceInterpret) - <_IsNative>%(_AppleAssembliesInternal._IsNative) - <_AotInputAssemblies Include="@(_AppleAssembliesInternal)" Condition="'%(_AppleAssembliesInternal._InternalForceInterpret)' != 'true'"> From d4ac4a86ff439a39cb8b8125318efef7c7e1a906 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Tue, 26 Jul 2022 09:01:37 -0400 Subject: [PATCH 07/13] Missing around BundleTestAppTargets --- eng/testing/tests.mobile.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index e0aa2217e14800..a3cb14fa052e1c 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -141,7 +141,7 @@ + DependsOnTargets="$(PublishTestAsSelfContainedDependsOn);$(BundleTestAppTargets);ArchiveTests" /> Date: Tue, 26 Jul 2022 17:52:25 -0400 Subject: [PATCH 08/13] PR feedback --- Directory.Build.props | 2 - .../android/build/AndroidApp.InTree.targets | 2 +- .../msbuild/android/build/AndroidApp.targets | 8 +- ...t.NET.Runtime.MonoAOTCompiler.Task.pkgproj | 1 - .../Sdk/Sdk.props.in | 3 - .../Sdk/MonoTargetsTasks.props.in | 2 + src/tasks/Common/CompilerCache.cs | 2 + src/tasks/Common/FileCache.cs | 2 + src/tasks/Common/ProxyFile.cs | 2 + .../MonoTargetsTasks/MonoTargetsTasks.csproj | 2 + .../NetTraceToMibcConverter.cs | 87 +++++++------------ .../NetTraceToMibcConverter.csproj | 32 ------- 12 files changed, 49 insertions(+), 96 deletions(-) rename src/tasks/{ => MonoTargetsTasks}/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs (60%) delete mode 100644 src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj diff --git a/Directory.Build.props b/Directory.Build.props index c70619a9cb8ecb..f3b2b5a217d83c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -100,7 +100,6 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)')) - $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'NetTraceToMibcConverter', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll')) @@ -114,7 +113,6 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WasmAppHost', 'wasm', '$(Configuration)')) $([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) - $([MSBuild]::NormalizePath('$(NetTraceToMibcConverterDir)', 'NetTraceToMibcConverter.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) $([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)')) diff --git a/src/mono/msbuild/android/build/AndroidApp.InTree.targets b/src/mono/msbuild/android/build/AndroidApp.InTree.targets index 1a1f45458504bb..2b374f617ae532 100644 --- a/src/mono/msbuild/android/build/AndroidApp.InTree.targets +++ b/src/mono/msbuild/android/build/AndroidApp.InTree.targets @@ -1,7 +1,7 @@ - + diff --git a/src/mono/msbuild/android/build/AndroidApp.targets b/src/mono/msbuild/android/build/AndroidApp.targets index 53bc35ced55b98..1e48e0a04ae6f4 100644 --- a/src/mono/msbuild/android/build/AndroidApp.targets +++ b/src/mono/msbuild/android/build/AndroidApp.targets @@ -79,12 +79,16 @@ + + <_ToolPath>$([System.IO.Path]::GetDirectoryName('$(DotnetPgoToolPath)')) + <_ToolExe>$([System.IO.Path]::GetFileName('$(DotnetPgoToolPath)')) + diff --git a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj index 2036fc6edb343c..e5a5b52244306c 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj +++ b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Microsoft.NET.Runtime.MonoAOTCompiler.Task.pkgproj @@ -7,7 +7,6 @@ - diff --git a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in index 648f42aa2a4ecc..bc14a619299e1e 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in +++ b/src/mono/nuget/Microsoft.NET.Runtime.MonoAOTCompiler.Task/Sdk/Sdk.props.in @@ -2,9 +2,6 @@ $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\MonoAOTCompiler.dll $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\MonoAOTCompiler.dll - $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETCoreTasks}\NetTraceToMibcConverter.dll - $(MSBuildThisFileDirectory)..\tasks\${TargetFrameworkForNETFrameworkTasks}\NetTraceToMibcConverter.dll - diff --git a/src/mono/nuget/Microsoft.NET.Runtime.MonoTargets.Sdk/Sdk/MonoTargetsTasks.props.in b/src/mono/nuget/Microsoft.NET.Runtime.MonoTargets.Sdk/Sdk/MonoTargetsTasks.props.in index 4ef325f421be72..ad5db32cbf581c 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.MonoTargets.Sdk/Sdk/MonoTargetsTasks.props.in +++ b/src/mono/nuget/Microsoft.NET.Runtime.MonoTargets.Sdk/Sdk/MonoTargetsTasks.props.in @@ -7,6 +7,8 @@ + + diff --git a/src/tasks/Common/CompilerCache.cs b/src/tasks/Common/CompilerCache.cs index 22e7e6c87122cd..1d28125511996b 100644 --- a/src/tasks/Common/CompilerCache.cs +++ b/src/tasks/Common/CompilerCache.cs @@ -8,6 +8,8 @@ using System.Text.Json.Serialization; using Microsoft.Build.Utilities; +#nullable enable + internal sealed class CompilerCache { public CompilerCache() => FileHashes = new(); diff --git a/src/tasks/Common/FileCache.cs b/src/tasks/Common/FileCache.cs index 098a4b7c5528b3..5f090693f7cee6 100644 --- a/src/tasks/Common/FileCache.cs +++ b/src/tasks/Common/FileCache.cs @@ -10,6 +10,8 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; +#nullable enable + internal sealed class FileCache { private CompilerCache? _newCache; diff --git a/src/tasks/Common/ProxyFile.cs b/src/tasks/Common/ProxyFile.cs index 79d96c83dada6a..ec38b3eb3c6d14 100644 --- a/src/tasks/Common/ProxyFile.cs +++ b/src/tasks/Common/ProxyFile.cs @@ -8,6 +8,8 @@ using System.Text.Json; using Microsoft.Build.Framework; +#nullable enable + internal sealed class ProxyFile { public string TargetFile { get; } diff --git a/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj b/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj index d470c822c08e96..a37d59c75ec5ae 100644 --- a/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj +++ b/src/tasks/MonoTargetsTasks/MonoTargetsTasks.csproj @@ -29,6 +29,8 @@ + + diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs similarity index 60% rename from src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs rename to src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs index 20d35d7d8074ce..3c6700f70e06a3 100644 --- a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs +++ b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs @@ -15,14 +15,9 @@ using Microsoft.Build.Utilities; using System.Text.Json.Serialization; -public class NetTraceToMibcConverter : Microsoft.Build.Utilities.Task -{ - - /// - /// Path to the mibc converter tool - /// - public string? MibcConverterBinaryPath { get; set; } +public class NetTraceToMibcConverter : ToolTask +{ /// /// Entries for assemblies referenced in a .nettrace file. Important when you run traces against an executable on a different machine / device /// @@ -35,12 +30,6 @@ public class NetTraceToMibcConverter : Microsoft.Build.Utilities.Task [Required] public string NetTracePath { get; set; } = ""; - /// - /// File used to track hashes of assemblies, to act as a cache - /// Output files don't get written, if they haven't changed - /// - public string? CacheFilePath { get; set; } - /// /// Directory where the mibc file will be placed /// @@ -54,11 +43,32 @@ public class NetTraceToMibcConverter : Microsoft.Build.Utilities.Task [Output] public string? MibcProfilePath { get; set; } - private bool ProcessAndValidateArguments() + protected override string ToolName { get; } = "NetTraceToMibcConverter"; + + protected override string GenerateFullPathToTool() + { + return ToolPath; + } + + protected override bool ValidateParameters() { - if (!File.Exists(MibcConverterBinaryPath)) + if (string.IsNullOrEmpty(ToolPath)) + { + Log.LogError($"{nameof(ToolPath)}='{ToolPath}' must be specified."); + return false; + } + + if (string.IsNullOrEmpty(ToolExe)) { - Log.LogError($"{nameof(MibcConverterBinaryPath)}='{MibcConverterBinaryPath}' doesn't exist."); + Log.LogError($"{nameof(ToolExe)}='{ToolExe}' must be specified."); + return false; + } + + string mibcConverterBinaryPath = Path.Combine(ToolPath, ToolExe); + + if (!File.Exists(mibcConverterBinaryPath)) + { + Log.LogError($"{nameof(mibcConverterBinaryPath)}='{mibcConverterBinaryPath}' doesn't exist."); return false; } @@ -84,55 +94,22 @@ private bool ProcessAndValidateArguments() return !Log.HasLoggedErrors; } - public override bool Execute() + protected override string GenerateCommandLineCommands() { - try - { - return ExecuteInternal(); - } - catch (LogAsErrorException laee) - { - Log.LogError(laee.Message); - return false; - } - } - - private bool ExecuteInternal() - { - if (!ProcessAndValidateArguments()) - return false; - - if (!ProcessNettrace(NetTracePath)) - return false; - - return !Log.HasLoggedErrors; - } - - private bool ProcessNettrace(string netTraceFile) - { - var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(netTraceFile), ".mibc")); + var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(NetTracePath), ".mibc")); StringBuilder mibcConverterArgsStr = new StringBuilder(string.Empty); mibcConverterArgsStr.Append($"create-mibc"); - mibcConverterArgsStr.Append($" --trace {netTraceFile} "); + mibcConverterArgsStr.Append($" --trace {NetTracePath} "); + foreach (var refAsmItem in Assemblies) { string? fullPath = refAsmItem.GetMetadata("FullPath"); mibcConverterArgsStr.Append($" --reference \"{fullPath}\" "); } - mibcConverterArgsStr.Append($" --output {outputMibcPath} "); - (int exitCode, string output) = Utils.TryRunProcess(Log, - MibcConverterBinaryPath!, - mibcConverterArgsStr.ToString()); - if (exitCode != 0) - { - Log.LogError($"dotnet-pgo({MibcConverterBinaryPath}) failed for {netTraceFile}:{output}"); - return false; - } + mibcConverterArgsStr.Append($" --output {outputMibcPath} "); - MibcProfilePath = outputMibcPath; - Log.LogMessage(MessageImportance.Low, $"Generated {outputMibcPath} from {MibcConverterBinaryPath}"); - return true; + return mibcConverterArgsStr.ToString(); } } diff --git a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj b/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj deleted file mode 100644 index 2829984e53cacc..00000000000000 --- a/src/tasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.csproj +++ /dev/null @@ -1,32 +0,0 @@ - - - $(TargetFrameworkForNETCoreTasks);$(TargetFrameworkForNETFrameworkTasks) - Library - true - false - enable - $(NoWarn),CA1050 - true - - - - - - - - - - - - - - - - <_PublishFramework Remove="@(_PublishFramework)" /> - <_PublishFramework Include="$(TargetFrameworks)" /> - - - - - - From a1a69e9dd31df76b12c9bbc72689e3aac8e18d25 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Tue, 26 Jul 2022 22:02:39 -0400 Subject: [PATCH 09/13] Cleanup aot task and remove nettrace conversion --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 97 ++------------------ 1 file changed, 6 insertions(+), 91 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 8aab73d53e39e3..ac6f712fcab1dd 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -111,21 +111,6 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// public bool UseDwarfDebug { get; set; } - /// - /// Path to Dotnet PGO binary (dotnet-pgo) - /// - public string? PgoBinaryPath { get; set; } - - /// - /// NetTrace file to use when invoking dotnet-pgo for - /// - public string? NetTracePath { get; set; } - - /// - /// Directory containing all assemblies referenced in a .nettrace collected from a separate device needed by dotnet-pgo. Necessary for mobile platforms. - /// - public ITaskItem[] ReferenceAssemblyPathsForPGO { get; set; } = Array.Empty(); - /// /// File to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. /// @@ -134,7 +119,7 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// /// Mibc file to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. /// - public string[] MibcProfilePath { get; set; } = Array.Empty(); + public ITaskItem[] MibcProfilePath { get; set; } = Array.Empty(); /// /// List of profilers to use. @@ -286,31 +271,6 @@ private bool ProcessAndValidateArguments() if (!Directory.Exists(IntermediateOutputPath)) Directory.CreateDirectory(IntermediateOutputPath); - if (!string.IsNullOrEmpty(NetTracePath)) - { - if (!File.Exists(NetTracePath)) - { - Log.LogError($"{nameof(NetTracePath)}='{NetTracePath}' doesn't exist"); - return false; - } - if (!File.Exists(PgoBinaryPath)) - { - Log.LogError($"NetTracePath was provided, but {nameof(PgoBinaryPath)}='{PgoBinaryPath}' doesn't exist"); - return false; - } - if (ReferenceAssemblyPathsForPGO.Length == 0) - { - Log.LogError($"NetTracePath was provided, but {nameof(ReferenceAssemblyPathsForPGO)} is empty"); - return false; - } - foreach (var refAsmItem in ReferenceAssemblyPathsForPGO) - { - string? fullPath = refAsmItem.GetMetadata("FullPath"); - if (!File.Exists(fullPath)) - throw new LogAsErrorException($"ReferenceAssembly '{fullPath}' doesn't exist"); - } - } - if (AotProfilePath != null) { foreach (var path in AotProfilePath) @@ -323,11 +283,11 @@ private bool ProcessAndValidateArguments() } } - foreach (var path in MibcProfilePath) + foreach (var item in MibcProfilePath) { - if (!File.Exists(path)) + if (!File.Exists(item.ItemSpec)) { - Log.LogError($"MibcProfilePath '{path}' doesn't exist."); + Log.LogError($"MibcProfilePath '{item.ItemSpec}' doesn't exist."); return false; } } @@ -437,48 +397,6 @@ public override bool Execute() } } - private bool ProcessNettrace(string netTraceFile) - { - var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(netTraceFile), ".mibc")); - - if (_cache!.Enabled) - { - string hash = Utils.ComputeHash(netTraceFile); - if (!_cache!.UpdateAndCheckHasFileChanged($"-mibc-source-file-{Path.GetFileName(netTraceFile)}", hash)) - { - Log.LogMessage(MessageImportance.Low, $"Skipping generating {outputMibcPath} from {netTraceFile} because source file hasn't changed"); - return true; - } - else - { - Log.LogMessage(MessageImportance.Low, $"Generating {outputMibcPath} from {netTraceFile} because the source file's hash has changed."); - } - } - - StringBuilder pgoArgsStr = new StringBuilder(string.Empty); - pgoArgsStr.Append($"create-mibc"); - pgoArgsStr.Append($" --trace {netTraceFile} "); - foreach (var refAsmItem in ReferenceAssemblyPathsForPGO) - { - string? fullPath = refAsmItem.GetMetadata("FullPath"); - pgoArgsStr.Append($" --reference \"{fullPath}\" "); - } - pgoArgsStr.Append($" --output {outputMibcPath} "); - (int exitCode, string output) = Utils.TryRunProcess(Log, - PgoBinaryPath!, - pgoArgsStr.ToString()); - - if (exitCode != 0) - { - Log.LogError($"dotnet-pgo({PgoBinaryPath}) failed for {netTraceFile}:{output}"); - return false; - } - - MibcProfilePath = MibcProfilePath.Append(outputMibcPath).ToArray(); - Log.LogMessage(MessageImportance.Low, $"Generated {outputMibcPath} from {PgoBinaryPath}"); - return true; - } - private bool ExecuteInternal() { if (!ProcessAndValidateArguments()) @@ -496,9 +414,6 @@ private bool ExecuteInternal() _cache = new FileCache(CacheFilePath, Log); - if (!string.IsNullOrEmpty(NetTracePath) && !ProcessNettrace(NetTracePath)) - return false; - List argsList = new(); foreach (var assemblyItem in _assembliesToCompile) argsList.Add(GetPrecompileArgumentsFor(assemblyItem, monoPaths)); @@ -841,9 +756,9 @@ private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, st if (MibcProfilePath.Length > 0) { aotArgs.Add("profile-only"); - foreach (var path in MibcProfilePath) + foreach (var item in MibcProfilePath) { - aotArgs.Add($"mibc-profile={path}"); + aotArgs.Add($"mibc-profile={item.ItemSpec}"); } } From b4b107e25d8e072c0dc872bf4fc492d5397cb607 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Thu, 28 Jul 2022 13:49:24 -0400 Subject: [PATCH 10/13] PR Feedback --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 8 +++---- .../NetTraceToMibcConverter.cs | 22 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index ac6f712fcab1dd..053c5429aea296 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -119,7 +119,7 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// /// Mibc file to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. /// - public ITaskItem[] MibcProfilePath { get; set; } = Array.Empty(); + public string[] MibcProfilePath { get; set; } = Array.Empty(); /// /// List of profilers to use. @@ -285,9 +285,9 @@ private bool ProcessAndValidateArguments() foreach (var item in MibcProfilePath) { - if (!File.Exists(item.ItemSpec)) + if (!File.Exists(item)) { - Log.LogError($"MibcProfilePath '{item.ItemSpec}' doesn't exist."); + Log.LogError($"MibcProfilePath '{item}' doesn't exist."); return false; } } @@ -758,7 +758,7 @@ private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, st aotArgs.Add("profile-only"); foreach (var item in MibcProfilePath) { - aotArgs.Add($"mibc-profile={item.ItemSpec}"); + aotArgs.Add($"mibc-profile={item}"); } } diff --git a/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs index 3c6700f70e06a3..f6e5d250e7fb39 100644 --- a/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs +++ b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs @@ -15,20 +15,21 @@ using Microsoft.Build.Utilities; using System.Text.Json.Serialization; +#nullable enable public class NetTraceToMibcConverter : ToolTask { /// - /// Entries for assemblies referenced in a .nettrace file. Important when you run traces against an executable on a different machine / device + /// List of all assemblies referenced in a .nettrace file. Important when you run traces against an executable on a different machine / device /// [Required] public ITaskItem[] Assemblies { get; set; } = Array.Empty(); /// - /// NetTrace file to use when invoking dotnet-pgo for + /// Path to .nettrace file which should be converted to .mibc /// [Required] - public string NetTracePath { get; set; } = ""; + public string NetTraceFilePath { get; set; } = ""; /// /// Directory where the mibc file will be placed @@ -41,7 +42,7 @@ public class NetTraceToMibcConverter : ToolTask /// The path to the mibc file generated from the converter. /// [Output] - public string? MibcProfilePath { get; set; } + public string? MibcFilePath { get; set; } protected override string ToolName { get; } = "NetTraceToMibcConverter"; @@ -78,9 +79,9 @@ protected override bool ValidateParameters() return false; } - if (!File.Exists(NetTracePath)) + if (!File.Exists(NetTraceFilePath)) { - Log.LogError($"{nameof(NetTracePath)}='{NetTracePath}' doesn't exist"); + Log.LogError($"{nameof(NetTraceFilePath)}='{NetTraceFilePath}' doesn't exist"); return false; } @@ -96,11 +97,10 @@ protected override bool ValidateParameters() protected override string GenerateCommandLineCommands() { - var outputMibcPath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(NetTracePath), ".mibc")); + MibcFilePath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(NetTraceFilePath), ".mibc")); - StringBuilder mibcConverterArgsStr = new StringBuilder(string.Empty); - mibcConverterArgsStr.Append($"create-mibc"); - mibcConverterArgsStr.Append($" --trace {NetTracePath} "); + StringBuilder mibcConverterArgsStr = new StringBuilder("create-mibc"); + mibcConverterArgsStr.Append($" --trace \"{NetTraceFilePath}\" "); foreach (var refAsmItem in Assemblies) { @@ -108,7 +108,7 @@ protected override string GenerateCommandLineCommands() mibcConverterArgsStr.Append($" --reference \"{fullPath}\" "); } - mibcConverterArgsStr.Append($" --output {outputMibcPath} "); + mibcConverterArgsStr.Append($" --output \"{MibcFilePath}\""); return mibcConverterArgsStr.ToString(); } From 2144902db167801d2b01fe6f155f7bbc4b82b265 Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Thu, 28 Jul 2022 14:04:43 -0400 Subject: [PATCH 11/13] More PR Feedback --- src/mono/msbuild/android/build/AndroidApp.targets | 8 ++++---- src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs | 6 +++--- src/tasks/AndroidAppBuilder/ApkBuilder.cs | 6 +++--- src/tasks/AndroidAppBuilder/Templates/monodroid.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/mono/msbuild/android/build/AndroidApp.targets b/src/mono/msbuild/android/build/AndroidApp.targets index 1e48e0a04ae6f4..7e68fb07adcf33 100644 --- a/src/mono/msbuild/android/build/AndroidApp.targets +++ b/src/mono/msbuild/android/build/AndroidApp.targets @@ -35,9 +35,9 @@ - <_AOTMode Condition="'$(UseMonoJustInterp)' != 'true'">Full + <_AOTMode Condition="'$(UseMonoJustInterp)' != 'true'">Normal <_AOTMode Condition="'$(UseMonoJustInterp)' == 'true'">JustInterp - <_AOTMode Condition="'$(RunProfileAOT)' == 'true'">Normal + <_AOTMode Condition="'$(ForceFullAOT)' == 'true'">Full @@ -77,7 +77,7 @@ <_ToolPath>$([System.IO.Path]::GetDirectoryName('$(DotnetPgoToolPath)')) @@ -128,8 +128,8 @@ IncludeNetworkSecurityConfig="$(IncludeNetworkSecurityConfig)" EnvironmentVariables="@(AndroidEnv)" ForceAOT="$(RunAOTCompilation)" + ForceFullAOT="$(ForceFullAOT)" ForceInterpreter="$(MonoForceInterpreter)" - ProfileAOT="$(RunProfileAOT)" StripDebugSymbols="False" RuntimeComponents="$(RuntimeComponents)" DiagnosticPorts="$(DiagnosticPorts)" diff --git a/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs b/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs index 0fd142f3b17213..0f409c1745f7ca 100644 --- a/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs +++ b/src/tasks/AndroidAppBuilder/AndroidAppBuilder.cs @@ -39,9 +39,9 @@ public class AndroidAppBuilderTask : Task public bool ForceAOT { get; set; } /// - /// Indicates if the assemblies to aot were gathered from a profile + /// Indicates if we want to AOT all assemblies or not /// - public bool ProfileAOT { get; set; } + public bool ForceFullAOT { get; set; } /// /// Static linked runtime @@ -116,7 +116,7 @@ public override bool Execute() apkBuilder.KeyStorePath = KeyStorePath; apkBuilder.ForceInterpreter = ForceInterpreter; apkBuilder.ForceAOT = ForceAOT; - apkBuilder.ProfileAOT = ProfileAOT; + apkBuilder.ForceFullAOT = ForceFullAOT; apkBuilder.EnvironmentVariables = EnvironmentVariables; apkBuilder.StaticLinkedRuntime = StaticLinkedRuntime; apkBuilder.RuntimeComponents = RuntimeComponents; diff --git a/src/tasks/AndroidAppBuilder/ApkBuilder.cs b/src/tasks/AndroidAppBuilder/ApkBuilder.cs index 8294a010d705b4..84ecf5c049d60c 100644 --- a/src/tasks/AndroidAppBuilder/ApkBuilder.cs +++ b/src/tasks/AndroidAppBuilder/ApkBuilder.cs @@ -28,7 +28,7 @@ public class ApkBuilder public string? KeyStorePath { get; set; } public bool ForceInterpreter { get; set; } public bool ForceAOT { get; set; } - public bool ProfileAOT { get; set; } + public bool ForceFullAOT { get; set; } public ITaskItem[] EnvironmentVariables { get; set; } = Array.Empty(); public bool InvariantGlobalization { get; set; } public bool EnableRuntimeLogging { get; set; } @@ -343,9 +343,9 @@ public ApkBuilder(TaskLoggingHelper logger) } } - if (ProfileAOT) + if (ForceFullAOT) { - defines.AppendLine("add_definitions(-DPROFILE_AOT=1)"); + defines.AppendLine("add_definitions(-DFULL_AOT=1)"); } if (!string.IsNullOrEmpty(DiagnosticPorts)) diff --git a/src/tasks/AndroidAppBuilder/Templates/monodroid.c b/src/tasks/AndroidAppBuilder/Templates/monodroid.c index 6f88f1d0542145..1fda6a85f849fe 100644 --- a/src/tasks/AndroidAppBuilder/Templates/monodroid.c +++ b/src/tasks/AndroidAppBuilder/Templates/monodroid.c @@ -275,10 +275,10 @@ mono_droid_runtime_init (const char* executable, int managed_argc, char* managed register_aot_modules(); #endif // STATIC_AOT -#if PROFILE_AOT - mono_jit_set_aot_mode(MONO_AOT_MODE_NORMAL); -#else +#if FULL_AOT mono_jit_set_aot_mode(MONO_AOT_MODE_FULL); +#else + mono_jit_set_aot_mode(MONO_AOT_MODE_NORMAL); #endif // PROFILE_AOT #endif // FORCE_INTERPRETER From a6ae5f057525b59fbcbb533d03d5e68c5ade2eba Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Mon, 1 Aug 2022 15:27:06 -0400 Subject: [PATCH 12/13] More feedback --- src/mono/msbuild/android/build/AndroidApp.targets | 12 ++++++++---- src/tasks/AndroidAppBuilder/Templates/monodroid.c | 2 +- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 10 +++++----- .../NetTraceToMibcConverter.cs | 6 +++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/mono/msbuild/android/build/AndroidApp.targets b/src/mono/msbuild/android/build/AndroidApp.targets index 7e68fb07adcf33..dc92fd3e263771 100644 --- a/src/mono/msbuild/android/build/AndroidApp.targets +++ b/src/mono/msbuild/android/build/AndroidApp.targets @@ -77,7 +77,7 @@ <_ToolPath>$([System.IO.Path]::GetDirectoryName('$(DotnetPgoToolPath)')) @@ -88,9 +88,9 @@ ToolPath="$(_ToolPath)" ToolExe="$(_ToolExe)" Assemblies="@(_AotInputAssemblies)" - NetTracePath="$(NetTracePath)" + NetTraceFilePath="$(NetTraceFilePath)" OutputDir="$(_MobileIntermediateOutputPath)"> - + @@ -98,6 +98,10 @@ Condition="'$(RunAOTCompilation)' == 'true'" DependsOnTargets="_AndroidBeforeAotCompileApp"> + + + + diff --git a/src/tasks/AndroidAppBuilder/Templates/monodroid.c b/src/tasks/AndroidAppBuilder/Templates/monodroid.c index 1fda6a85f849fe..1d8d3f1bd529cf 100644 --- a/src/tasks/AndroidAppBuilder/Templates/monodroid.c +++ b/src/tasks/AndroidAppBuilder/Templates/monodroid.c @@ -279,7 +279,7 @@ mono_droid_runtime_init (const char* executable, int managed_argc, char* managed mono_jit_set_aot_mode(MONO_AOT_MODE_FULL); #else mono_jit_set_aot_mode(MONO_AOT_MODE_NORMAL); -#endif // PROFILE_AOT +#endif // FULL_AOT #endif // FORCE_INTERPRETER MonoDomain *domain = mono_jit_init_version ("dotnet.android", "mobile"); diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 4a99d2d4736c38..291af85bde42a6 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -284,11 +284,11 @@ private bool ProcessAndValidateArguments() } } - foreach (var item in MibcProfilePath) + foreach (var path in MibcProfilePath) { - if (!File.Exists(item)) + if (!File.Exists(path)) { - Log.LogError($"MibcProfilePath '{item}' doesn't exist."); + Log.LogError($"MibcProfilePath '{path}' doesn't exist."); return false; } } @@ -760,9 +760,9 @@ private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, st if (MibcProfilePath.Length > 0) { aotArgs.Add("profile-only"); - foreach (var item in MibcProfilePath) + foreach (var path in MibcProfilePath) { - aotArgs.Add($"mibc-profile={item}"); + aotArgs.Add($"mibc-profile={path}"); } } diff --git a/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs index f6e5d250e7fb39..eaf0090d2b7b8a 100644 --- a/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs +++ b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs @@ -42,7 +42,7 @@ public class NetTraceToMibcConverter : ToolTask /// The path to the mibc file generated from the converter. /// [Output] - public string? MibcFilePath { get; set; } + public string[] MibcFilePath { get; set; } = new string[1]; protected override string ToolName { get; } = "NetTraceToMibcConverter"; @@ -97,7 +97,7 @@ protected override bool ValidateParameters() protected override string GenerateCommandLineCommands() { - MibcFilePath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(NetTraceFilePath), ".mibc")); + MibcFilePath[0] = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(NetTraceFilePath), ".mibc")); StringBuilder mibcConverterArgsStr = new StringBuilder("create-mibc"); mibcConverterArgsStr.Append($" --trace \"{NetTraceFilePath}\" "); @@ -108,7 +108,7 @@ protected override string GenerateCommandLineCommands() mibcConverterArgsStr.Append($" --reference \"{fullPath}\" "); } - mibcConverterArgsStr.Append($" --output \"{MibcFilePath}\""); + mibcConverterArgsStr.Append($" --output \"{MibcFilePath[0]}\""); return mibcConverterArgsStr.ToString(); } From 96d4bcbfa150bf5b279d0dd422224b2829de448f Mon Sep 17 00:00:00 2001 From: Steve Pfister Date: Mon, 1 Aug 2022 22:02:31 -0400 Subject: [PATCH 13/13] More feedback --- Directory.Build.props | 3 ++- eng/testing/tests.mobile.targets | 4 ++-- src/mono/msbuild/android/build/AndroidApp.targets | 2 -- .../NetTraceToMibcConverter.cs | 12 +++++++----- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index f3b2b5a217d83c..6c5c1b4bceac58 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -115,7 +115,8 @@ $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) $([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll')) - $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)')) + $(CoreCLRToolPath) diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets index a3cb14fa052e1c..d4ef535566364c 100644 --- a/eng/testing/tests.mobile.targets +++ b/eng/testing/tests.mobile.targets @@ -16,7 +16,7 @@ - $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)', 'dotnet-pgo')) + $([MSBuild]::NormalizeDirectory('$(CoreCLRToolPath)', 'dotnet-pgo')) $([MSBuild]::NormalizePath('$(DotnetPgoToolDir)', 'dotnet-pgo')) @@ -44,7 +44,7 @@ - $(DiagnosticPorts),$(DiagnosticStartup) + $(DiagnosticPorts),$(DiagnosticStartupMode) diff --git a/src/mono/msbuild/android/build/AndroidApp.targets b/src/mono/msbuild/android/build/AndroidApp.targets index dc92fd3e263771..6fd26641cd1e10 100644 --- a/src/mono/msbuild/android/build/AndroidApp.targets +++ b/src/mono/msbuild/android/build/AndroidApp.targets @@ -81,12 +81,10 @@ DependsOnTargets="_AndroidBeforeAotCompileApp"> <_ToolPath>$([System.IO.Path]::GetDirectoryName('$(DotnetPgoToolPath)')) - <_ToolExe>$([System.IO.Path]::GetFileName('$(DotnetPgoToolPath)')) diff --git a/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs index eaf0090d2b7b8a..205b115a4685e3 100644 --- a/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs +++ b/src/tasks/MonoTargetsTasks/NetTraceToMibcConverterTask/NetTraceToMibcConverter.cs @@ -7,6 +7,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Text.Json; using System.Threading; @@ -42,7 +43,9 @@ public class NetTraceToMibcConverter : ToolTask /// The path to the mibc file generated from the converter. /// [Output] - public string[] MibcFilePath { get; set; } = new string[1]; + public string MibcFilePath { get; set; } = ""; + + public override string ToolExe { get; set; } = ""; protected override string ToolName { get; } = "NetTraceToMibcConverter"; @@ -61,8 +64,7 @@ protected override bool ValidateParameters() if (string.IsNullOrEmpty(ToolExe)) { - Log.LogError($"{nameof(ToolExe)}='{ToolExe}' must be specified."); - return false; + ToolExe = (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) ? "dotnet-pgo.exe" : "dotnet-pgo"; } string mibcConverterBinaryPath = Path.Combine(ToolPath, ToolExe); @@ -97,7 +99,7 @@ protected override bool ValidateParameters() protected override string GenerateCommandLineCommands() { - MibcFilePath[0] = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(NetTraceFilePath), ".mibc")); + MibcFilePath = Path.Combine(OutputDir, Path.ChangeExtension(Path.GetFileName(NetTraceFilePath), ".mibc")); StringBuilder mibcConverterArgsStr = new StringBuilder("create-mibc"); mibcConverterArgsStr.Append($" --trace \"{NetTraceFilePath}\" "); @@ -108,7 +110,7 @@ protected override string GenerateCommandLineCommands() mibcConverterArgsStr.Append($" --reference \"{fullPath}\" "); } - mibcConverterArgsStr.Append($" --output \"{MibcFilePath[0]}\""); + mibcConverterArgsStr.Append($" --output \"{MibcFilePath}\""); return mibcConverterArgsStr.ToString(); }