Skip to content

Commit

Permalink
build benchmarks in Parallel, part of dotnet#550
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamsitnik @alinasmirnova
adamsitnik authored and alinasmirnova committed Sep 22, 2018
1 parent 1b49262 commit 923e348
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 73 deletions.
133 changes: 61 additions & 72 deletions src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using BenchmarkDotNet.Analysers;
using BenchmarkDotNet.Characteristics;
using BenchmarkDotNet.Configs;
Expand Down Expand Up @@ -98,12 +99,32 @@ public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, str

var globalChronometer = Chronometer.Start();
var reports = new List<BenchmarkReport>();

var buildResults = BuildInParallel(logger, rootArtifactsFolderPath, toolchainProvider, resolver, benchmarks, config, ref globalChronometer);

foreach (var benchmark in benchmarks)
{
var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup);
reports.Add(report);
if (report.GetResultRuns().Any())
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
var buildResult = buildResults[benchmark];

if (!config.KeepBenchmarkFiles)
artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup);

if (buildResult.IsBuildSuccess)
{
var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, buildResult);
reports.Add(report);
if (report.GetResultRuns().Any())
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
}
else
{
reports.Add(new BenchmarkReport(benchmark, buildResult, buildResult, null, null, default));

if (buildResult.GenerateException != null)
logger.WriteLineError($"// Generate Exception: {buildResult.GenerateException.Message}");
if (buildResult.BuildException != null)
logger.WriteLineError($"// Build Exception: {buildResult.BuildException.Message}");
}

logger.WriteLine();
}
Expand Down Expand Up @@ -189,93 +210,61 @@ private static ValidationError[] Validate(IReadOnlyList<Benchmark> benchmarks, I
return validationErrors;
}

internal static void LogTotalTime(ILogger logger, TimeSpan time, string message = "Total time")
private static Dictionary<Benchmark, BuildResult> BuildInParallel(ILogger logger, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver, Benchmark[] benchmarks, ReadOnlyConfig config, ref StartedClock globalChronometer)
{
logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime()}");
using (benchmarks.Select(benchmark => GetAssemblyResolveHelper(toolchainProvider(benchmark.Job), logger)).FirstOrDefault(helper => helper != null))
{
logger.WriteLineHeader($"// ***** Building {benchmarks.Length} benchmark(s) in Parallel: Start *****");

var buildResults = benchmarks
.AsParallel()
.Select(benchmark => { return (benchmark, buildResult: Build(benchmark, config, rootArtifactsFolderPath, toolchainProvider, resolver)); })
.ToDictionary(result => result.benchmark, result => result.buildResult);

logger.WriteLineHeader($"// ***** Done, took {globalChronometer.GetElapsed().GetTimeSpan().ToFormattedTotalTime()} *****");

return buildResults;
}
}

private static BenchmarkReport RunCore(Benchmark benchmark, ILogger logger, ReadOnlyConfig config, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver, List<string> artifactsToCleanup)
private static BuildResult Build(Benchmark benchmark, ReadOnlyConfig config, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver)
{
var toolchain = toolchainProvider(benchmark.Job);

logger.WriteLineHeader("// **************************");
logger.WriteLineHeader("// Benchmark: " + benchmark.DisplayInfo);

var assemblyResolveHelper = GetAssemblyResolveHelper(toolchain, logger);
var generateResult = Generate(logger, toolchain, benchmark, rootArtifactsFolderPath, config, resolver);
var generateResult = toolchain.Generator.GenerateProject(benchmark, NullLogger.Instance, rootArtifactsFolderPath, config, resolver);

try
{
if (!generateResult.IsGenerateSuccess)
return new BenchmarkReport(benchmark, generateResult, null, null, null, default(GcStats));
return BuildResult.Failure(generateResult);

var buildResult = Build(logger, toolchain, generateResult, benchmark, resolver);
if (!buildResult.IsBuildSuccess)
return new BenchmarkReport(benchmark, generateResult, buildResult, null, null, default(GcStats));
return toolchain.Builder.Build(generateResult, NullLogger.Instance, benchmark, resolver);

var (executeResults, gcStats) = Execute(logger, benchmark, toolchain, buildResult, config, resolver);

var runs = new List<Measurement>();

for (int index = 0; index < executeResults.Count; index++)
{
var executeResult = executeResults[index];
runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown));
}

return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs, gcStats);
}
catch (Exception e)
{
logger.WriteLineError("// Exception: " + e);
return new BenchmarkReport(benchmark, generateResult, BuildResult.Failure(generateResult, e), Array.Empty<ExecuteResult>(), Array.Empty<Measurement>(), GcStats.Empty);
}
finally
{
if (!config.KeepBenchmarkFiles)
{
artifactsToCleanup.AddRange(generateResult.ArtifactsToCleanup);
}

assemblyResolveHelper?.Dispose();
return BuildResult.Failure(generateResult, e);
}
}

private static GenerateResult Generate(ILogger logger, IToolchain toolchain, Benchmark benchmark, string rootArtifactsFolderPath, IConfig config, IResolver resolver)
private static BenchmarkReport RunCore(Benchmark benchmark, ILogger logger, ReadOnlyConfig config, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver, BuildResult buildResult)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generator.GenerateProject(benchmark, logger, rootArtifactsFolderPath, config, resolver);
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.ArtifactsPaths.BinariesDirectoryPath)} = {generateResult.ArtifactsPaths?.BinariesDirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
logger.WriteLineError($"// Exception: {generateResult.GenerateException}");
}
logger.WriteLine();
return generateResult;
}
var toolchain = toolchainProvider(benchmark.Job);

private static BuildResult Build(ILogger logger, IToolchain toolchain, GenerateResult generateResult, Benchmark benchmark, IResolver resolver)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Builder.Build(generateResult, logger, benchmark, resolver);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
logger.WriteLineHeader("// **************************");
logger.WriteLineHeader("// Benchmark: " + benchmark.DisplayInfo);

var (executeResults, gcStats) = Execute(logger, benchmark, toolchain, buildResult, config, resolver);

var runs = new List<Measurement>();

for (int index = 0; index < executeResults.Count; index++)
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
var executeResult = executeResults[index];
runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown));
}
logger.WriteLine();
return buildResult;

return new BenchmarkReport(benchmark, buildResult, buildResult, executeResults, runs, gcStats);
}

private static (List<ExecuteResult> executeResults, GcStats gcStats) Execute(
Expand Down Expand Up @@ -383,10 +372,10 @@ private static (List<ExecuteResult> executeResults, GcStats gcStats) Execute(
return (executeResults, gcStats);
}

internal static void LogTotalTime(ILogger logger, TimeSpan time, string message = "Total time") => logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime()}");

private static Benchmark[] GetSupportedBenchmarks(IList<Benchmark> benchmarks, CompositeLogger logger, Func<Job, IToolchain> toolchainProvider, IResolver resolver)
{
return benchmarks.Where(benchmark => toolchainProvider(benchmark.Job).IsSupported(benchmark, logger, resolver)).ToArray();
}
=> benchmarks.Where(benchmark => toolchainProvider(benchmark.Job).IsSupported(benchmark, logger, resolver)).ToArray();

private static string GetRootArtifactsFolderPath() => CombineAndCreate(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts");

Expand Down
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet.Core/Toolchains/Results/BuildResult.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,6 @@ private BuildResult(GenerateResult generateResult, bool isBuildSuccess, Exceptio

public static BuildResult Failure(GenerateResult generateResult, Exception exception = null) => new BuildResult(generateResult, false, exception);

public override string ToString() => "BuildResult: " + (IsBuildSuccess ? "Success" : "Fail");
public override string ToString() => "BuildResult: " + (IsBuildSuccess ? "Success" : "Failure");
}
}

0 comments on commit 923e348

Please sign in to comment.