From a9664f46200aa564cfa1e6f89049a75172cccd34 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 27 May 2018 20:41:26 +0200 Subject: [PATCH] don't execute long operations more than once per iteration (#760), fixes #736 * don't execute long operations more than once per iteration, #736 * generate the right C# code for #736 * EngineParameters.Resolver was always null or ignored ;), #736 * don't forget to JIT idle, #736 * do the math right for unroll factor for JIT, #736 * generate the right IL code, #736 * Setup and Cleanup are jitted together with benchmark, #736 * engine factory is now supposed to create an engine which is ready to run (hence the method name change), #736 * addressing PR feedback, #736 * bring back the calls to DummyActions, #736 * align iteration mode * don't measure the overhead for time consuming benchmarks, don't run pilot if jitting gives the answer, #736 * fix the Linux build --- .../Intro/IntroInProcessWrongEnv.cs | 3 +- .../Diagnosers/WindowsDisassembler.cs | 2 +- src/BenchmarkDotNet/Engines/Engine.cs | 18 +- src/BenchmarkDotNet/Engines/EngineFactory.cs | 97 ++++++++- .../Engines/EngineParameters.cs | 29 ++- src/BenchmarkDotNet/Engines/IEngine.cs | 8 +- src/BenchmarkDotNet/Engines/IEngineFactory.cs | 2 +- src/BenchmarkDotNet/Engines/IterationMode.cs | 7 +- .../Engines/IterationModeExtensions.cs | 2 +- .../Extensions/ProcessExtensions.cs | 3 +- .../Portability/RuntimeInformation.cs | 4 +- src/BenchmarkDotNet/Reports/Measurement.cs | 8 +- .../Templates/BenchmarkType.txt | 83 +++++-- .../Toolchains/InProcess/InProcessRunner.cs | 27 +-- ...upAndCleanupTargetSpecificBenchmarkTest.cs | 47 ++-- .../AllSetupAndCleanupTest.cs | 23 +- .../CoreRtTests.cs | 3 + .../CustomEngineTests.cs | 13 +- .../Engine/EngineFactoryTests.cs | 206 ++++++++++++++++++ .../BenchmarkDotNet.Tests/Mocks/MockEngine.cs | 4 +- 20 files changed, 461 insertions(+), 128 deletions(-) create mode 100644 tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs diff --git a/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs b/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs index 9ba212b8ad..8fe83a3829 100644 --- a/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs +++ b/samples/BenchmarkDotNet.Samples/Intro/IntroInProcessWrongEnv.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Order; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains.InProcess; namespace BenchmarkDotNet.Samples.Intro @@ -18,7 +19,7 @@ private class Config : ManualConfig { public Config() { - var wrongPlatform = IntPtr.Size == sizeof(int) + var wrongPlatform = RuntimeInformation.GetCurrentPlatform() == Platform.X86 ? Platform.X64 : Platform.X86; diff --git a/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs b/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs index f8a4ba3998..a0a2d03b26 100644 --- a/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs +++ b/src/BenchmarkDotNet/Diagnosers/WindowsDisassembler.cs @@ -156,7 +156,7 @@ public static bool Is64Bit(Process process) return !isWow64; } - return IntPtr.Size == 8; // todo: find the way to cover all scenarios for .NET Core + return Portability.RuntimeInformation.GetCurrentPlatform() == Platform.X64; // todo: find the way to cover all scenarios for .NET Core } [DllImport("kernel32.dll", SetLastError = true, CallingConvention = CallingConvention.Winapi)] diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs index 45719d8bbd..ae99cf37cb 100644 --- a/src/BenchmarkDotNet/Engines/Engine.cs +++ b/src/BenchmarkDotNet/Engines/Engine.cs @@ -41,10 +41,10 @@ public class Engine : IEngine private readonly EngineWarmupStage warmupStage; private readonly EngineTargetStage targetStage; private readonly bool includeMemoryStats; - private bool isJitted; internal Engine( IHost host, + IResolver resolver, Action dummy1Action, Action dummy2Action, Action dummy3Action, Action idleAction, Action mainAction, Job targetJob, Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke, bool includeMemoryStats) @@ -64,7 +64,7 @@ internal Engine( OperationsPerInvoke = operationsPerInvoke; this.includeMemoryStats = includeMemoryStats; - Resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); + Resolver = resolver; Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver); ForceAllocations = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver); @@ -78,22 +78,10 @@ internal Engine( targetStage = new EngineTargetStage(this); } - public void Jitting() - { - // first signal about jitting is raised from auto-generated Program.cs, look at BenchmarkProgram.txt - Dummy1Action.Invoke(); - MainAction.Invoke(1); - Dummy2Action.Invoke(); - IdleAction.Invoke(1); - Dummy3Action.Invoke(); - isJitted = true; - } + public void Dispose() => GlobalCleanupAction?.Invoke(); public RunResults Run() { - if (Strategy.NeedsJitting() != isJitted) - throw new Exception($"You must{(Strategy.NeedsJitting() ? "" : " not")} call Jitting() first (Strategy = {Strategy})!"); - long invokeCount = InvocationCount; IReadOnlyList idle = null; diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index a5f74bda49..18719ad8d0 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -1,39 +1,114 @@ using System; +using BenchmarkDotNet.Horology; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Engines { - // TODO: Default instance? public class EngineFactory : IEngineFactory { - public IEngine Create(EngineParameters engineParameters) + public IEngine CreateReadyToRun(EngineParameters engineParameters) { - if (engineParameters.MainAction == null) - throw new ArgumentNullException(nameof(engineParameters.MainAction)); + if (engineParameters.MainSingleAction == null) + throw new ArgumentNullException(nameof(engineParameters.MainSingleAction)); + if (engineParameters.MainMultiAction == null) + throw new ArgumentNullException(nameof(engineParameters.MainMultiAction)); if (engineParameters.Dummy1Action == null) throw new ArgumentNullException(nameof(engineParameters.Dummy1Action)); if (engineParameters.Dummy2Action == null) throw new ArgumentNullException(nameof(engineParameters.Dummy2Action)); if (engineParameters.Dummy3Action == null) throw new ArgumentNullException(nameof(engineParameters.Dummy3Action)); - if (engineParameters.IdleAction == null) - throw new ArgumentNullException(nameof(engineParameters.IdleAction)); + if (engineParameters.IdleSingleAction == null) + throw new ArgumentNullException(nameof(engineParameters.IdleSingleAction)); + if (engineParameters.IdleMultiAction == null) + throw new ArgumentNullException(nameof(engineParameters.IdleMultiAction)); if(engineParameters.TargetJob == null) throw new ArgumentNullException(nameof(engineParameters.TargetJob)); - return new Engine( + engineParameters.GlobalSetupAction?.Invoke(); // whatever the settings are, we MUST call global setup here, the global cleanup is part of Engine's Dispose + + if (!engineParameters.NeedsJitting) // just create the engine, do NOT jit + return CreateMultiActionEngine(engineParameters); + + int jitIndex = 0; + + if (engineParameters.HasInvocationCount || engineParameters.HasUnrollFactor) // it's a job with explicit configuration, just create the engine and jit it + { + var warmedUpMultiActionEngine = CreateMultiActionEngine(engineParameters); + + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(warmedUpMultiActionEngine, ++jitIndex, invokeCount: engineParameters.UnrollFactor, unrollFactor: engineParameters.UnrollFactor)); + + return warmedUpMultiActionEngine; + } + + var singleActionEngine = CreateSingleActionEngine(engineParameters); + if (Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1) > engineParameters.IterationTime) + return singleActionEngine; // executing once takes longer than iteration time => long running benchmark, needs no pilot and no overhead + + var multiActionEngine = CreateMultiActionEngine(engineParameters); + int defaultUnrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, EngineParameters.DefaultResolver); + + if (Jit(multiActionEngine, ++jitIndex, invokeCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor) > engineParameters.IterationTime) + { // executing defaultUnrollFactor times takes longer than iteration time => medium running benchmark, needs no pilot and no overhead + var defaultUnrollFactorTimesPerIterationNoPilotNoOverhead = CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, + invocationCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor); // run the benchmark exactly once per iteration + + return CreateEngine(engineParameters, defaultUnrollFactorTimesPerIterationNoPilotNoOverhead, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + } + + return multiActionEngine; + } + + /// the time it took to run the benchmark + private static TimeInterval Jit(Engine engine, int jitIndex, int invokeCount, int unrollFactor) + { + engine.Dummy1Action.Invoke(); + + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, jitIndex, invokeCount, unrollFactor))); // don't forget to JIT idle + + engine.Dummy2Action.Invoke(); + + var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, jitIndex, invokeCount, unrollFactor)); + + engine.Dummy3Action.Invoke(); + + engine.WriteLine(); + + return TimeInterval.FromNanoseconds(result.Nanoseconds); + } + + private static Engine CreateMultiActionEngine(EngineParameters engineParameters) + => CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + + private static Engine CreateSingleActionEngine(EngineParameters engineParameters) + => CreateEngine(engineParameters, + CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, invocationCount: 1, unrollFactor: 1), // run the benchmark exactly once per iteration + engineParameters.IdleSingleAction, + engineParameters.MainSingleAction); + + private static Job CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(Job sourceJob, int invocationCount, int unrollFactor) + => sourceJob + .WithInvocationCount(invocationCount).WithUnrollFactor(unrollFactor) + .WithEvaluateOverhead(false); // it's very time consuming, don't evaluate the overhead which would be 0,000025% of the target run or even less + // todo: consider if we should set the warmup count to 2 + + private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action idle, Action main) + => new Engine( engineParameters.Host, + EngineParameters.DefaultResolver, engineParameters.Dummy1Action, engineParameters.Dummy2Action, engineParameters.Dummy3Action, - engineParameters.IdleAction, - engineParameters.MainAction, - engineParameters.TargetJob, + idle, + main, + job, engineParameters.GlobalSetupAction, engineParameters.GlobalCleanupAction, engineParameters.IterationSetupAction, engineParameters.IterationCleanupAction, engineParameters.OperationsPerInvoke, engineParameters.MeasureGcStats); - } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs index 49475b2eb0..9bf8e12051 100644 --- a/src/BenchmarkDotNet/Engines/EngineParameters.cs +++ b/src/BenchmarkDotNet/Engines/EngineParameters.cs @@ -1,24 +1,39 @@ using System; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Engines { public class EngineParameters { + public static readonly IResolver DefaultResolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); + public IHost Host { get; set; } - public Action MainAction { get; set; } + public Action MainSingleAction { get; set; } + public Action MainMultiAction { get; set; } public Action Dummy1Action { get; set; } public Action Dummy2Action { get; set; } public Action Dummy3Action { get; set; } - public Action IdleAction { get; set; } + public Action IdleSingleAction { get; set; } + public Action IdleMultiAction { get; set; } public Job TargetJob { get; set; } = Job.Default; public long OperationsPerInvoke { get; set; } = 1; - public Action GlobalSetupAction { get; set; } = null; - public Action GlobalCleanupAction { get; set; } = null; - public Action IterationSetupAction { get; set; } = null; - public Action IterationCleanupAction { get; set; } = null; - public IResolver Resolver { get; set; } + public Action GlobalSetupAction { get; set; } + public Action GlobalCleanupAction { get; set; } + public Action IterationSetupAction { get; set; } + public Action IterationCleanupAction { get; set; } public bool MeasureGcStats { get; set; } + + public bool NeedsJitting => TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, DefaultResolver).NeedsJitting(); + + public bool HasInvocationCount => TargetJob.HasValue(RunMode.InvocationCountCharacteristic); + + public bool HasUnrollFactor => TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); + + public int UnrollFactor => TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); + + public TimeInterval IterationTime => TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, DefaultResolver); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IEngine.cs b/src/BenchmarkDotNet/Engines/IEngine.cs index f31c6d74c1..bd4142244c 100644 --- a/src/BenchmarkDotNet/Engines/IEngine.cs +++ b/src/BenchmarkDotNet/Engines/IEngine.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Engines { - public interface IEngine + public interface IEngine : IDisposable { [NotNull] IHost Host { get; } @@ -36,12 +36,6 @@ public interface IEngine Measurement RunIteration(IterationData data); - /// - /// must perform jitting via warmup calls - /// is called after first call to GlobalSetup, from the auto-generated benchmark process - /// - void Jitting(); - RunResults Run(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IEngineFactory.cs b/src/BenchmarkDotNet/Engines/IEngineFactory.cs index 89dfb14eb3..c518441409 100644 --- a/src/BenchmarkDotNet/Engines/IEngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/IEngineFactory.cs @@ -2,6 +2,6 @@ namespace BenchmarkDotNet.Engines { public interface IEngineFactory { - IEngine Create(EngineParameters engineParameters); + IEngine CreateReadyToRun(EngineParameters engineParameters); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IterationMode.cs b/src/BenchmarkDotNet/Engines/IterationMode.cs index e60002ba68..b9ed7e6e58 100644 --- a/src/BenchmarkDotNet/Engines/IterationMode.cs +++ b/src/BenchmarkDotNet/Engines/IterationMode.cs @@ -35,6 +35,11 @@ public enum IterationMode /// /// Unknown /// - Unknown + Unknown, + + /// + /// executing benchmark for the purpose of JIT wamup + /// + MainJitting, IdleJitting } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs b/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs index 88cb925ef9..13287136d1 100644 --- a/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs +++ b/src/BenchmarkDotNet/Engines/IterationModeExtensions.cs @@ -3,6 +3,6 @@ public static class IterationModeExtensions { public static bool IsIdle(this IterationMode mode) - => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget; + => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget || mode == IterationMode.IdleJitting; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 8b52501cdc..9f2c0e1247 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -31,7 +32,7 @@ private static IntPtr FixAffinity(IntPtr processorAffinity) { int cpuMask = (1 << Environment.ProcessorCount) - 1; - return IntPtr.Size == sizeof(Int64) + return RuntimeInformation.GetCurrentPlatform() == Platform.X64 ? new IntPtr(processorAffinity.ToInt64() & cpuMask) : new IntPtr(processorAffinity.ToInt32() & cpuMask); } diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index c82af29ddf..3349edb63b 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -61,7 +61,7 @@ internal static class RuntimeInformation internal static string ScriptFileExtension => IsWindows() ? ".bat" : ".sh"; - internal static string GetArchitecture() => IntPtr.Size == 4 ? "32bit" : "64bit"; + internal static string GetArchitecture() => GetCurrentPlatform() == Platform.X86 ? "32bit" : "64bit"; internal static bool IsWindows() { @@ -247,7 +247,7 @@ internal static bool HasRyuJit() if (IsNetCore) return true; - return IntPtr.Size == 8 + return GetCurrentPlatform() == Platform.X64 && GetConfiguration() != DebugConfigurationName && !new JitHelper().IsMsX64(); } diff --git a/src/BenchmarkDotNet/Reports/Measurement.cs b/src/BenchmarkDotNet/Reports/Measurement.cs index 461d5106b5..f05a310b92 100644 --- a/src/BenchmarkDotNet/Reports/Measurement.cs +++ b/src/BenchmarkDotNet/Reports/Measurement.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; @@ -13,6 +14,8 @@ public struct Measurement : IComparable { private static readonly Measurement Error = new Measurement(-1, IterationMode.Unknown, 0, 0, 0); + private static readonly int IterationModeNameMaxWidth = Enum.GetNames(typeof(IterationMode)).Max(text => text.Length); + public IterationMode IterationMode { get; } public int LaunchIndex { get; } @@ -48,10 +51,13 @@ public Measurement(int launchIndex, IterationMode iterationMode, int iterationIn public string ToOutputLine() { + string alignedIterationMode = IterationMode.ToString().PadRight(IterationModeNameMaxWidth, ' '); + // Usually, a benchmarks takes more than 10 iterations (rarely more than 99) // PadLeft(2, ' ') looks like a good trade-off between alignment and amount of characters string alignedIterationIndex = IterationIndex.ToString().PadLeft(2, ' '); - return $"{IterationMode} {alignedIterationIndex}: {GetDisplayValue()}"; + + return $"{alignedIterationMode} {alignedIterationIndex}: {GetDisplayValue()}"; } private string GetDisplayValue() => $"{Operations} op, {Nanoseconds.ToStr("0.00")} ns, {GetAverageTime()}"; diff --git a/src/BenchmarkDotNet/Templates/BenchmarkType.txt b/src/BenchmarkDotNet/Templates/BenchmarkType.txt index e62b2fde26..e547a1181d 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkType.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkType.txt @@ -24,11 +24,13 @@ var engineParameters = new BenchmarkDotNet.Engines.EngineParameters() { Host = host, - MainAction = instance.MainMultiAction, + MainMultiAction = instance.MainMultiAction, + MainSingleAction = instance.MainSingleAction, Dummy1Action = instance.Dummy1, Dummy2Action = instance.Dummy2, Dummy3Action = instance.Dummy3, - IdleAction = instance.IdleMultiAction, + IdleSingleAction = instance.IdleSingleAction, + IdleMultiAction = instance.IdleMultiAction, GlobalSetupAction = instance.globalSetupAction, GlobalCleanupAction = instance.globalCleanupAction, IterationSetupAction = instance.iterationSetupAction, @@ -38,23 +40,14 @@ MeasureGcStats = $MeasureGcStats$ }; - var engine = new $EngineFactoryType$().Create(engineParameters); - - instance?.globalSetupAction(); - instance?.iterationSetupAction(); - - if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting()) - engine.Jitting(); // does first call to main action, must be executed after globalSetup() and iterationSetup()! - - instance?.iterationCleanupAction(); - - var results = engine.Run(); - - instance?.globalCleanupAction(); + using (var engine = new $EngineFactoryType$().CreateReadyToRun(engineParameters)) + { + var results = engine.Run(); - host.ReportResults(results); // printing costs memory, do this after runs + host.ReportResults(results); // printing costs memory, do this after runs - instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;) + instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;) + } } public delegate $IdleMethodReturnTypeName$ IdleDelegate($ArgumentsDefinition$); @@ -128,6 +121,12 @@ } } + private void IdleSingleAction(long _) + { + $LoadArguments$ + consumer.Consume(idleDelegate($PassArguments$)); + } + private void MainMultiAction(long invokeCount) { $LoadArguments$ @@ -137,6 +136,12 @@ } } + private void MainSingleAction(long _) + { + $LoadArguments$ + consumer.Consume(targetDelegate($PassArguments$)$ConsumeField$); + } + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public $TargetMethodReturnType$ $DiassemblerEntryMethodName$() { @@ -162,6 +167,14 @@ DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result); } + private void IdleSingleAction(long _) + { + $LoadArguments$ + $IdleMethodReturnTypeName$ result = default($IdleMethodReturnTypeName$); + result = idleDelegate($PassArguments$); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result); + } + private void MainMultiAction(long invokeCount) { $LoadArguments$ @@ -173,6 +186,14 @@ NonGenericKeepAliveWithoutBoxing(result); } + private void MainSingleAction(long _) + { + $LoadArguments$ + $TargetMethodReturnType$ result = default($TargetMethodReturnType$); + result = targetDelegate($PassArguments$); + NonGenericKeepAliveWithoutBoxing(result); + } + // we must not simply use DeadCodeEliminationHelper.KeepAliveWithoutBoxing because it's generic method // and stack-only types like Span can not be generic type arguments http://adamsitnik.com/Span/#span-must-not-be-a-generic-type-argument [MethodImpl(MethodImplOptions.NoInlining)] @@ -202,6 +223,14 @@ } DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); } + + private void IdleSingleAction(long _) + { + $LoadArguments$ + $IdleMethodReturnTypeName$ value = default($IdleMethodReturnTypeName$); + value = idleDelegate($PassArguments$); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value); + } private $TargetMethodReturnType$ mainDefaultValueHolder = default($TargetMethodReturnType$); @@ -215,6 +244,14 @@ } DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias); } + + private void MainSingleAction(long _) + { + $LoadArguments$ + ref $TargetMethodReturnType$ alias = ref mainDefaultValueHolder; + alias = targetDelegate($PassArguments$); + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias); + } [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public ref $TargetMethodReturnType$ $DiassemblerEntryMethodName$() @@ -238,6 +275,12 @@ } } + private void IdleSingleAction(long _) + { + $LoadArguments$ + idleDelegate($PassArguments$); + } + private void MainMultiAction(long invokeCount) { $LoadArguments$ @@ -246,6 +289,12 @@ targetDelegate($PassArguments$);@Unroll@ } } + + private void MainSingleAction(long _) + { + $LoadArguments$ + targetDelegate($PassArguments$); + } [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public void $DiassemblerEntryMethodName$() diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs index 814c23e38e..b045c8651a 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessRunner.cs @@ -112,11 +112,13 @@ public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodeg var engineParameters = new EngineParameters { Host = host, - MainAction = mainAction.InvokeMultiple, + MainSingleAction = _ => mainAction.InvokeSingle(), + MainMultiAction = mainAction.InvokeMultiple, Dummy1Action = dummy1.InvokeSingle, Dummy2Action = dummy2.InvokeSingle, Dummy3Action = dummy3.InvokeSingle, - IdleAction = idleAction.InvokeMultiple, + IdleSingleAction = _ => idleAction.InvokeSingle(), + IdleMultiAction = idleAction.InvokeMultiple, GlobalSetupAction = globalSetupAction.InvokeSingle, GlobalCleanupAction = globalCleanupAction.InvokeSingle, IterationSetupAction = iterationSetupAction.InvokeSingle, @@ -126,23 +128,14 @@ public static void RunCore(IHost host, Benchmark benchmark, BenchmarkActionCodeg MeasureGcStats = config.HasMemoryDiagnoser() }; - var engine = job + using (var engine = job .ResolveValue(InfrastructureMode.EngineFactoryCharacteristic, InfrastructureResolver.Instance) - .Create(engineParameters); - - globalSetupAction.InvokeSingle(); - iterationSetupAction.InvokeSingle(); - - if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting()) - engine.Jitting(); // does first call to main action, must be executed after setup()! - - iterationCleanupAction.InvokeSingle(); - - var results = engine.Run(); - - globalCleanupAction.InvokeSingle(); + .CreateReadyToRun(engineParameters)) + { + var results = engine.Run(); - host.ReportResults(results); // printing costs memory, do this after runs + host.ReportResults(results); // printing costs memory, do this after runs + } } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs index ef4185b145..f7e997dcbe 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTargetSpecificBenchmarkTest.cs @@ -11,7 +11,6 @@ namespace BenchmarkDotNet.IntegrationTests { public class AllSetupAndCleanupTargetSpecificBenchmarkTest : BenchmarkTestExecutor { - private const string FirstPrefix = "// ### First Called: "; private const string FirstGlobalSetupCalled = FirstPrefix + "GlobalSetup"; private const string FirstGlobalCleanupCalled = FirstPrefix + "GlobalCleanup"; @@ -31,25 +30,22 @@ public class AllSetupAndCleanupTargetSpecificBenchmarkTest : BenchmarkTestExecut private readonly string[] firstExpectedLogLines = { "// ### First Called: GlobalSetup", - "// ### First Called: IterationSetup (1)", // IterationSetup Jitting - "// ### First Called: IterationCleanup (1)", // IterationCleanup Jitting - - "// ### First Called: IterationSetup (2)", // MainWarmup1 + "// ### First Called: IterationSetup (1)", // MainWarmup1 "// ### First Called: Benchmark", // MainWarmup1 - "// ### First Called: IterationCleanup (2)", // MainWarmup1 - "// ### First Called: IterationSetup (3)", // MainWarmup2 + "// ### First Called: IterationCleanup (1)", // MainWarmup1 + "// ### First Called: IterationSetup (2)", // MainWarmup2 "// ### First Called: Benchmark", // MainWarmup2 - "// ### First Called: IterationCleanup (3)", // MainWarmup2 + "// ### First Called: IterationCleanup (2)", // MainWarmup2 - "// ### First Called: IterationSetup (4)", // MainTarget1 + "// ### First Called: IterationSetup (3)", // MainTarget1 "// ### First Called: Benchmark", // MainTarget1 - "// ### First Called: IterationCleanup (4)", // MainTarget1 - "// ### First Called: IterationSetup (5)", // MainTarget2 + "// ### First Called: IterationCleanup (3)", // MainTarget1 + "// ### First Called: IterationSetup (4)", // MainTarget2 "// ### First Called: Benchmark", // MainTarget2 - "// ### First Called: IterationCleanup (5)", // MainTarget2 - "// ### First Called: IterationSetup (6)", // MainTarget3 + "// ### First Called: IterationCleanup (4)", // MainTarget2 + "// ### First Called: IterationSetup (5)", // MainTarget3 "// ### First Called: Benchmark", // MainTarget3 - "// ### First Called: IterationCleanup (6)", // MainTarget3 + "// ### First Called: IterationCleanup (5)", // MainTarget3 "// ### First Called: GlobalCleanup" }; @@ -57,25 +53,22 @@ public class AllSetupAndCleanupTargetSpecificBenchmarkTest : BenchmarkTestExecut private readonly string[] secondExpectedLogLines = { "// ### Second Called: GlobalSetup", - "// ### Second Called: IterationSetup (1)", // IterationSetup Jitting - "// ### Second Called: IterationCleanup (1)", // IterationCleanup Jitting - - "// ### Second Called: IterationSetup (2)", // MainWarmup1 + "// ### Second Called: IterationSetup (1)", // MainWarmup1 "// ### Second Called: Benchmark", // MainWarmup1 - "// ### Second Called: IterationCleanup (2)", // MainWarmup1 - "// ### Second Called: IterationSetup (3)", // MainWarmup2 + "// ### Second Called: IterationCleanup (1)", // MainWarmup1 + "// ### Second Called: IterationSetup (2)", // MainWarmup2 "// ### Second Called: Benchmark", // MainWarmup2 - "// ### Second Called: IterationCleanup (3)", // MainWarmup2 + "// ### Second Called: IterationCleanup (2)", // MainWarmup2 - "// ### Second Called: IterationSetup (4)", // MainTarget1 + "// ### Second Called: IterationSetup (3)", // MainTarget1 "// ### Second Called: Benchmark", // MainTarget1 - "// ### Second Called: IterationCleanup (4)", // MainTarget1 - "// ### Second Called: IterationSetup (5)", // MainTarget2 + "// ### Second Called: IterationCleanup (3)", // MainTarget1 + "// ### Second Called: IterationSetup (4)", // MainTarget2 "// ### Second Called: Benchmark", // MainTarget2 - "// ### Second Called: IterationCleanup (5)", // MainTarget2 - "// ### Second Called: IterationSetup (6)", // MainTarget3 + "// ### Second Called: IterationCleanup (4)", // MainTarget2 + "// ### Second Called: IterationSetup (5)", // MainTarget3 "// ### Second Called: Benchmark", // MainTarget3 - "// ### Second Called: IterationCleanup (6)", // MainTarget3 + "// ### Second Called: IterationCleanup (5)", // MainTarget3 "// ### Second Called: GlobalCleanup" }; diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs index 140c9e97ac..4aa998e5ff 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs @@ -22,25 +22,22 @@ public class AllSetupAndCleanupTest : BenchmarkTestExecutor private readonly string[] expectedLogLines = { "// ### Called: GlobalSetup", - "// ### Called: IterationSetup (1)", // IterationSetup Jitting - "// ### Called: IterationCleanup (1)", // IterationCleanup Jitting - - "// ### Called: IterationSetup (2)", // MainWarmup1 + "// ### Called: IterationSetup (1)", // MainWarmup1 "// ### Called: Benchmark", // MainWarmup1 - "// ### Called: IterationCleanup (2)", // MainWarmup1 - "// ### Called: IterationSetup (3)", // MainWarmup2 + "// ### Called: IterationCleanup (1)", // MainWarmup1 + "// ### Called: IterationSetup (2)", // MainWarmup2 "// ### Called: Benchmark", // MainWarmup2 - "// ### Called: IterationCleanup (3)", // MainWarmup2 + "// ### Called: IterationCleanup (2)", // MainWarmup2 - "// ### Called: IterationSetup (4)", // MainTarget1 + "// ### Called: IterationSetup (3)", // MainTarget1 "// ### Called: Benchmark", // MainTarget1 - "// ### Called: IterationCleanup (4)", // MainTarget1 - "// ### Called: IterationSetup (5)", // MainTarget2 + "// ### Called: IterationCleanup (3)", // MainTarget1 + "// ### Called: IterationSetup (4)", // MainTarget2 "// ### Called: Benchmark", // MainTarget2 - "// ### Called: IterationCleanup (5)", // MainTarget2 - "// ### Called: IterationSetup (6)", // MainTarget3 + "// ### Called: IterationCleanup (4)", // MainTarget2 + "// ### Called: IterationSetup (5)", // MainTarget3 "// ### Called: Benchmark", // MainTarget3 - "// ### Called: IterationCleanup (6)", // MainTarget3 + "// ### Called: IterationCleanup (5)", // MainTarget3 "// ### Called: GlobalCleanup" }; diff --git a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs index 1b5e1f20a0..471c1ab82e 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CoreRtTests.cs @@ -17,6 +17,9 @@ public CoreRtTests(ITestOutputHelper outputHelper) : base(outputHelper) { } [Fact] public void CoreRtIsSupported() { + if (RuntimeInformation.GetCurrentPlatform() == Platform.X86) // CoreRT does not support 32bit yet + return; + var config = ManualConfig.CreateEmpty() .With(Job.Dry .With(Runtime.CoreRT) diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index ef0cdd2b92..a23cb72ab1 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -54,12 +54,18 @@ public void Empty() { } public class CustomFactory : IEngineFactory { - public IEngine Create(EngineParameters engineParameters) - => new CustomEngine + public IEngine CreateReadyToRun(EngineParameters engineParameters) + { + var engine = new CustomEngine { GlobalCleanupAction = engineParameters.GlobalCleanupAction, GlobalSetupAction = engineParameters.GlobalSetupAction }; + + engine.GlobalSetupAction?.Invoke(); // engine factory is now supposed to create an engine which is ready to run (hence the method name change) + + return engine; + } } public class CustomEngine : IEngine @@ -75,6 +81,8 @@ public RunResults Run() default); } + public void Dispose() => GlobalCleanupAction?.Invoke(); + public IHost Host { get; } public void WriteLine() { } public void WriteLine(string line) { } @@ -87,7 +95,6 @@ public void WriteLine(string line) { } public IResolver Resolver { get; } public Measurement RunIteration(IterationData data) { throw new NotImplementedException(); } - public void Jitting() { } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs new file mode 100644 index 0000000000..11edb4830c --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using Xunit; + +namespace BenchmarkDotNet.Tests.Engine +{ + public class EngineFactoryTests + { + int timesBenchmarkCalled = 0, timesIdleCalled = 0; + int timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0; + + TimeSpan IterationTime => TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds()); + + IResolver DefaultResolver => BenchmarkRunner.DefaultResolver; + + void GlobalSetup() => timesGlobalSetupCalled++; + void IterationSetup() => timesIterationSetupCalled++; + void IterationCleanup() => timesIterationCleanupCalled++; + void GlobalCleanup() => timesGlobalCleanupCalled++; + + void Throwing(long _) => throw new InvalidOperationException("must NOT be called"); + + void VeryTimeConsumingSingle(long _) + { + timesBenchmarkCalled++; + Thread.Sleep(IterationTime); + } + + void InstantSingle(long _) => timesBenchmarkCalled++; + void Instant16(long _) => timesBenchmarkCalled += 16; + + void IdleSingle(long _) => timesIdleCalled++; + void Idle16(long _) => timesIdleCalled += 16; + + public static IEnumerable JobsWhichDontRequireJitting() + { + yield return new object[]{ Job.Dry }; + yield return new object[]{ Job.Default.With(RunStrategy.ColdStart) }; + yield return new object[]{ Job.Default.With(RunStrategy.Monitoring) }; + } + + [Theory] + [MemberData(nameof(JobsWhichDontRequireJitting))] + public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(Job job) + { + var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Throwing, job: job); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(0, timesIterationSetupCalled); + Assert.Equal(0, timesBenchmarkCalled); + Assert.Equal(0, timesIdleCalled); + Assert.Equal(0, timesIterationCleanupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + [Fact] + public void ForDefaultSettingsVeryTimeConsumingBenchmarksAreExecutedOncePerIterationWithoutOverheadDeduction() + { + var engineParameters = CreateEngineParameters(mainSingleAction: VeryTimeConsumingSingle, mainMultiAction: Throwing, job: Job.Default); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(1 + 1, timesIterationSetupCalled); // 1x for Idle, 1x for Target + Assert.Equal(1, timesBenchmarkCalled); + Assert.Equal(1, timesIdleCalled); + Assert.Equal(1 + 1, timesIterationCleanupCalled); // 1x for Idle, 1x for Target + Assert.Equal(0, timesGlobalCleanupCalled); // cleanup is called as part of dispode + + Assert.Equal(1, engine.TargetJob.Run.InvocationCount); // call the benchmark once per iteration + Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor + + Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way + Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case + + engine.Dispose(); // cleanup is called as part of dispode + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + [Fact] + public void ForJobsWithExplicitUnrollFactorTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() + => AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithUnrollFactor(16)); + + [Fact] + public void ForJobsThatDontRequirePilotTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() + => AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job.Default.WithInvocationCount(100)); + + private void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job) + { + var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Instant16, job: job); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(2, timesIterationSetupCalled); + Assert.Equal(16, timesBenchmarkCalled); + Assert.Equal(16, timesIdleCalled); + Assert.Equal(2, timesIterationCleanupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + [Fact] + public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWithUnrollFactorForDefaultSettings() + { + var engineParameters = CreateEngineParameters(mainSingleAction: InstantSingle, mainMultiAction: Instant16, job: Job.Default); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal((1+1) * (1+1), timesIterationSetupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(1 + 16, timesBenchmarkCalled); + Assert.Equal(1 + 16, timesIdleCalled); + Assert.Equal((1+1) * (1+1), timesIterationCleanupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched + + Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + [Fact] + public void DontRunThePilotIfThePilotRequirementIsMetDuringWarmup() + { + var unrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); + var mediumTime = TimeSpan.FromMilliseconds((IterationTime.TotalMilliseconds / unrollFactor) * 2); + + void MediumSingle(long _) + { + timesBenchmarkCalled++; + + Thread.Sleep(mediumTime); + } + + void MediumMultiple(long _) + { + timesBenchmarkCalled += unrollFactor; + + for (int i = 0; i < unrollFactor; i++) // the real unroll factor obviously does not use loop ;) + Thread.Sleep(mediumTime); + } + + var engineParameters = CreateEngineParameters(mainSingleAction: MediumSingle, mainMultiAction: MediumMultiple, job: Job.Default); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal((1+1) * (1+1), timesIterationSetupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(1 + unrollFactor, timesBenchmarkCalled); + Assert.Equal(1 + unrollFactor, timesIdleCalled); + Assert.Equal((1+1) * (1+1), timesIterationCleanupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.Equal(unrollFactor, engine.TargetJob.Run.InvocationCount); // no need to run pilot! + Assert.Equal(unrollFactor, engine.TargetJob.Run.UnrollFactor); // remains the same! + + Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way + Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + + private EngineParameters CreateEngineParameters(Action mainSingleAction, Action mainMultiAction, Job job) + => new EngineParameters + { + Dummy1Action = () => { }, + Dummy2Action = () => { }, + Dummy3Action = () => { }, + GlobalSetupAction = GlobalSetup, + GlobalCleanupAction = GlobalCleanup, + Host = new ConsoleHost(TextWriter.Null, TextReader.Null), + IdleMultiAction = Idle16, + IdleSingleAction = IdleSingle, + IterationCleanupAction = IterationCleanup, + IterationSetupAction = IterationSetup, + MainMultiAction = mainMultiAction, + MainSingleAction = mainSingleAction, + TargetJob = job + }; + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs index b5f64d2b79..07e0a6a0b9 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockEngine.cs @@ -22,6 +22,8 @@ public MockEngine(ITestOutputHelper output, Job job, Func GlobalSetupAction?.Invoke(); [UsedImplicitly] public IHost Host { get; } @@ -52,8 +54,6 @@ public Measurement RunIteration(IterationData data) return measurement; } - public void Jitting() { } - public RunResults Run() => default; public void WriteLine() => output.WriteLine("");