From 923e348aff8eb28f2f9616bae3a12d1f54d367ec Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sat, 30 Dec 2017 20:05:32 +0100 Subject: [PATCH] build benchmarks in Parallel, part of #550 --- .../Running/BenchmarkRunnerCore.cs | 133 ++++++++---------- .../Toolchains/Results/BuildResult.cs | 2 +- 2 files changed, 62 insertions(+), 73 deletions(-) diff --git a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs index 50aa448274..c5de14106e 100644 --- a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs +++ b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Configs; @@ -98,12 +99,32 @@ public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, str var globalChronometer = Chronometer.Start(); var reports = new List(); + + var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, toolchainProvider, resolver, benchmarks, config, ref globalChronometer); + foreach (var benchmark in benchmarks) { - var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup); - reports.Add(report); - if (report.GetResultRuns().Any()) - logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); + var buildResult = buildResults[benchmark]; + + if (!config.KeepBenchmarkFiles) + artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); + + if (buildResult.IsBuildSuccess) + { + var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, buildResult); + reports.Add(report); + if (report.GetResultRuns().Any()) + logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr()); + } + else + { + reports.Add(new BenchmarkReport(benchmark, buildResult, buildResult, null, null, default)); + + if (buildResult.GenerateException != null) + logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}"); + if (buildResult.BuildException != null) + logger.WriteLineError($"// Build Exception: {buildResult.BuildException.Message}"); + } logger.WriteLine(); } @@ -189,93 +210,61 @@ private static ValidationError[] Validate(IReadOnlyList benchmarks, I return validationErrors; } - internal static void LogTotalTime(ILogger logger, TimeSpan time, string message = "Total time") + private static Dictionary BuildInParallel(ILogger logger, string rootArtifactsFolderPath, Func toolchainProvider, IResolver resolver, Benchmark[] benchmarks, ReadOnlyConfig config, ref StartedClock globalChronometer) { - logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime()}"); + using (benchmarks.Select(benchmark => GetAssemblyResolveHelper(toolchainProvider(benchmark.Job), logger)).FirstOrDefault(helper => helper != null)) + { + logger.WriteLineHeader($"// ***** Building {benchmarks.Length} benchmark(s) in Parallel: Start *****"); + + var buildResults = benchmarks + .AsParallel() + .Select(benchmark => { return (benchmark, buildResult: Build(benchmark, config, rootArtifactsFolderPath, toolchainProvider, resolver)); }) + .ToDictionary(result => result.benchmark, result => result.buildResult); + + logger.WriteLineHeader($"// ***** Done, took {globalChronometer.GetElapsed().GetTimeSpan().ToFormattedTotalTime()} *****"); + + return buildResults; + } } - private static BenchmarkReport RunCore(Benchmark benchmark, ILogger logger, ReadOnlyConfig config, string rootArtifactsFolderPath, Func toolchainProvider, IResolver resolver, List artifactsToCleanup) + private static BuildResult Build(Benchmark benchmark, ReadOnlyConfig config, string rootArtifactsFolderPath, Func toolchainProvider, IResolver resolver) { var toolchain = toolchainProvider(benchmark.Job); - logger.WriteLineHeader("// **************************"); - logger.WriteLineHeader("// Benchmark: " + benchmark.DisplayInfo); - - var assemblyResolveHelper = GetAssemblyResolveHelper(toolchain, logger); - var generateResult = Generate(logger, toolchain, benchmark, rootArtifactsFolderPath, config, resolver); + var generateResult = toolchain.Generator.GenerateProject(benchmark, NullLogger.Instance, rootArtifactsFolderPath, config, resolver); try { if (!generateResult.IsGenerateSuccess) - return new BenchmarkReport(benchmark, generateResult, null, null, null, default(GcStats)); + return BuildResult.Failure(generateResult); - var buildResult = Build(logger, toolchain, generateResult, benchmark, resolver); - if (!buildResult.IsBuildSuccess) - return new BenchmarkReport(benchmark, generateResult, buildResult, null, null, default(GcStats)); + return toolchain.Builder.Build(generateResult, NullLogger.Instance, benchmark, resolver); - var (executeResults, gcStats) = Execute(logger, benchmark, toolchain, buildResult, config, resolver); - - var runs = new List(); - - for (int index = 0; index < executeResults.Count; index++) - { - var executeResult = executeResults[index]; - runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown)); - } - - return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs, gcStats); } catch (Exception e) { - logger.WriteLineError("// Exception: " + e); - return new BenchmarkReport(benchmark, generateResult, BuildResult.Failure(generateResult, e), Array.Empty(), Array.Empty(), GcStats.Empty); - } - finally - { - if (!config.KeepBenchmarkFiles) - { - artifactsToCleanup.AddRange(generateResult.ArtifactsToCleanup); - } - - assemblyResolveHelper?.Dispose(); + return BuildResult.Failure(generateResult, e); } } - private static GenerateResult Generate(ILogger logger, IToolchain toolchain, Benchmark benchmark, string rootArtifactsFolderPath, IConfig config, IResolver resolver) + private static BenchmarkReport RunCore(Benchmark benchmark, ILogger logger, ReadOnlyConfig config, string rootArtifactsFolderPath, Func toolchainProvider, IResolver resolver, BuildResult buildResult) { - logger.WriteLineInfo("// *** Generate *** "); - var generateResult = toolchain.Generator.GenerateProject(benchmark, logger, rootArtifactsFolderPath, config, resolver); - if (generateResult.IsGenerateSuccess) - { - logger.WriteLineInfo("// Result = Success"); - logger.WriteLineInfo($"// {nameof(generateResult.ArtifactsPaths.BinariesDirectoryPath)} = {generateResult.ArtifactsPaths?.BinariesDirectoryPath}"); - } - else - { - logger.WriteLineError("// Result = Failure"); - if (generateResult.GenerateException != null) - logger.WriteLineError($"// Exception: {generateResult.GenerateException}"); - } - logger.WriteLine(); - return generateResult; - } + var toolchain = toolchainProvider(benchmark.Job); - private static BuildResult Build(ILogger logger, IToolchain toolchain, GenerateResult generateResult, Benchmark benchmark, IResolver resolver) - { - logger.WriteLineInfo("// *** Build ***"); - var buildResult = toolchain.Builder.Build(generateResult, logger, benchmark, resolver); - if (buildResult.IsBuildSuccess) - { - logger.WriteLineInfo("// Result = Success"); - } - else + logger.WriteLineHeader("// **************************"); + logger.WriteLineHeader("// Benchmark: " + benchmark.DisplayInfo); + + var (executeResults, gcStats) = Execute(logger, benchmark, toolchain, buildResult, config, resolver); + + var runs = new List(); + + for (int index = 0; index < executeResults.Count; index++) { - logger.WriteLineError("// Result = Failure"); - if (buildResult.BuildException != null) - logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}"); + var executeResult = executeResults[index]; + runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown)); } - logger.WriteLine(); - return buildResult; + + return new BenchmarkReport(benchmark, buildResult, buildResult, executeResults, runs, gcStats); } private static (List executeResults, GcStats gcStats) Execute( @@ -383,10 +372,10 @@ private static (List executeResults, GcStats gcStats) Execute( return (executeResults, gcStats); } + internal static void LogTotalTime(ILogger logger, TimeSpan time, string message = "Total time") => logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime()}"); + private static Benchmark[] GetSupportedBenchmarks(IList benchmarks, CompositeLogger logger, Func toolchainProvider, IResolver resolver) - { - return benchmarks.Where(benchmark => toolchainProvider(benchmark.Job).IsSupported(benchmark, logger, resolver)).ToArray(); - } + => benchmarks.Where(benchmark => toolchainProvider(benchmark.Job).IsSupported(benchmark, logger, resolver)).ToArray(); private static string GetRootArtifactsFolderPath() => CombineAndCreate(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts"); diff --git a/src/BenchmarkDotNet.Core/Toolchains/Results/BuildResult.cs b/src/BenchmarkDotNet.Core/Toolchains/Results/BuildResult.cs index 172846193f..7836b4e2e0 100644 --- a/src/BenchmarkDotNet.Core/Toolchains/Results/BuildResult.cs +++ b/src/BenchmarkDotNet.Core/Toolchains/Results/BuildResult.cs @@ -18,6 +18,6 @@ private BuildResult(GenerateResult generateResult, bool isBuildSuccess, Exceptio public static BuildResult Failure(GenerateResult generateResult, Exception exception = null) => new BuildResult(generateResult, false, exception); - public override string ToString() => "BuildResult: " + (IsBuildSuccess ? "Success" : "Fail"); + public override string ToString() => "BuildResult: " + (IsBuildSuccess ? "Success" : "Failure"); } } \ No newline at end of file