From 916444c5397729522b5bb95d7daae910504a164b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 24 May 2018 13:56:01 +0200 Subject: [PATCH] don't measure the overhead for time consuming benchmarks, don't run pilot if jitting gives the answer, #736 --- src/BenchmarkDotNet/Engines/EngineFactory.cs | 83 ++++++++------ .../Engines/EngineParameters.cs | 22 +++- .../Engine/EngineFactoryTests.cs | 108 ++++++++++++++---- 3 files changed, 150 insertions(+), 63 deletions(-) diff --git a/src/BenchmarkDotNet/Engines/EngineFactory.cs b/src/BenchmarkDotNet/Engines/EngineFactory.cs index 576a196fa2..18719ad8d0 100644 --- a/src/BenchmarkDotNet/Engines/EngineFactory.cs +++ b/src/BenchmarkDotNet/Engines/EngineFactory.cs @@ -1,5 +1,4 @@ using System; -using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; @@ -28,69 +27,77 @@ public IEngine CreateReadyToRun(EngineParameters engineParameters) if(engineParameters.TargetJob == null) throw new ArgumentNullException(nameof(engineParameters.TargetJob)); - var resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); - var unrollFactor = engineParameters.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver); + engineParameters.GlobalSetupAction?.Invoke(); // whatever the settings are, we MUST call global setup here, the global cleanup is part of Engine's Dispose - engineParameters.GlobalSetupAction?.Invoke(); + if (!engineParameters.NeedsJitting) // just create the engine, do NOT jit + return CreateMultiActionEngine(engineParameters); - var needsJitting = engineParameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, resolver).NeedsJitting(); - if (!needsJitting) - { - // whatever it is, we can not interfere - return CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); - } - - var needsPilot = !engineParameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic); - var hasUnrollFactorDefined = engineParameters.TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); + int jitIndex = 0; - if (needsPilot && !hasUnrollFactorDefined) + if (engineParameters.HasInvocationCount || engineParameters.HasUnrollFactor) // it's a job with explicit configuration, just create the engine and jit it { - var singleActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); - - var iterationTime = resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic); - if (ShouldExecuteOncePerIteration(Jit(singleActionEngine, unrollFactor: 1), iterationTime)) - { - var reconfiguredJob = engineParameters.TargetJob.WithInvocationCount(1).WithUnrollFactor(1); // todo: consider if we should set the warmup count to 1! + var warmedUpMultiActionEngine = CreateMultiActionEngine(engineParameters); + + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(warmedUpMultiActionEngine, ++jitIndex, invokeCount: engineParameters.UnrollFactor, unrollFactor: engineParameters.UnrollFactor)); - return CreateEngine(engineParameters, resolver, reconfiguredJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction); - } + return warmedUpMultiActionEngine; } + + var singleActionEngine = CreateSingleActionEngine(engineParameters); + if (Jit(singleActionEngine, ++jitIndex, invokeCount: 1, unrollFactor: 1) > engineParameters.IterationTime) + return singleActionEngine; // executing once takes longer than iteration time => long running benchmark, needs no pilot and no overhead - // it's either a job with explicit configuration or not-very time consuming benchmark, just create the engine, Jit and return - var multiActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); - - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine, unrollFactor)); + var multiActionEngine = CreateMultiActionEngine(engineParameters); + int defaultUnrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, EngineParameters.DefaultResolver); + if (Jit(multiActionEngine, ++jitIndex, invokeCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor) > engineParameters.IterationTime) + { // executing defaultUnrollFactor times takes longer than iteration time => medium running benchmark, needs no pilot and no overhead + var defaultUnrollFactorTimesPerIterationNoPilotNoOverhead = CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, + invocationCount: defaultUnrollFactor, unrollFactor: defaultUnrollFactor); // run the benchmark exactly once per iteration + + return CreateEngine(engineParameters, defaultUnrollFactorTimesPerIterationNoPilotNoOverhead, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + } + return multiActionEngine; } - /// - /// returns true if it takes longer than the desired iteration time (0,5s by default) to execute benchmark once - /// - private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval iterationTime) - => TimeInterval.FromNanoseconds(jit.GetAverageNanoseconds()) > iterationTime; - - private static Measurement Jit(Engine engine, int unrollFactor) + /// the time it took to run the benchmark + private static TimeInterval Jit(Engine engine, int jitIndex, int invokeCount, int unrollFactor) { engine.Dummy1Action.Invoke(); - DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor))); // don't forget to JIT idle + DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJitting, jitIndex, invokeCount, unrollFactor))); // don't forget to JIT idle engine.Dummy2Action.Invoke(); - var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, index: 1, invokeCount: unrollFactor, unrollFactor: unrollFactor)); + var result = engine.RunIteration(new IterationData(IterationMode.MainJitting, jitIndex, invokeCount, unrollFactor)); engine.Dummy3Action.Invoke(); engine.WriteLine(); - return result; + return TimeInterval.FromNanoseconds(result.Nanoseconds); } - private static Engine CreateEngine(EngineParameters engineParameters, IResolver resolver, Job job, Action idle, Action main) + private static Engine CreateMultiActionEngine(EngineParameters engineParameters) + => CreateEngine(engineParameters, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction); + + private static Engine CreateSingleActionEngine(EngineParameters engineParameters) + => CreateEngine(engineParameters, + CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(engineParameters.TargetJob, invocationCount: 1, unrollFactor: 1), // run the benchmark exactly once per iteration + engineParameters.IdleSingleAction, + engineParameters.MainSingleAction); + + private static Job CreateJobWhichDoesNotNeedPilotAndOverheadEvaluation(Job sourceJob, int invocationCount, int unrollFactor) + => sourceJob + .WithInvocationCount(invocationCount).WithUnrollFactor(unrollFactor) + .WithEvaluateOverhead(false); // it's very time consuming, don't evaluate the overhead which would be 0,000025% of the target run or even less + // todo: consider if we should set the warmup count to 2 + + private static Engine CreateEngine(EngineParameters engineParameters, Job job, Action idle, Action main) => new Engine( engineParameters.Host, - resolver, + EngineParameters.DefaultResolver, engineParameters.Dummy1Action, engineParameters.Dummy2Action, engineParameters.Dummy3Action, diff --git a/src/BenchmarkDotNet/Engines/EngineParameters.cs b/src/BenchmarkDotNet/Engines/EngineParameters.cs index f384d32f9e..9bf8e12051 100644 --- a/src/BenchmarkDotNet/Engines/EngineParameters.cs +++ b/src/BenchmarkDotNet/Engines/EngineParameters.cs @@ -1,11 +1,15 @@ using System; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Horology; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Engines { public class EngineParameters { + public static readonly IResolver DefaultResolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance); + public IHost Host { get; set; } public Action MainSingleAction { get; set; } public Action MainMultiAction { get; set; } @@ -16,10 +20,20 @@ public class EngineParameters public Action IdleMultiAction { get; set; } public Job TargetJob { get; set; } = Job.Default; public long OperationsPerInvoke { get; set; } = 1; - public Action GlobalSetupAction { get; set; } = null; - public Action GlobalCleanupAction { get; set; } = null; - public Action IterationSetupAction { get; set; } = null; - public Action IterationCleanupAction { get; set; } = null; + public Action GlobalSetupAction { get; set; } + public Action GlobalCleanupAction { get; set; } + public Action IterationSetupAction { get; set; } + public Action IterationCleanupAction { get; set; } public bool MeasureGcStats { get; set; } + + public bool NeedsJitting => TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, DefaultResolver).NeedsJitting(); + + public bool HasInvocationCount => TargetJob.HasValue(RunMode.InvocationCountCharacteristic); + + public bool HasUnrollFactor => TargetJob.HasValue(RunMode.UnrollFactorCharacteristic); + + public int UnrollFactor => TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); + + public TimeInterval IterationTime => TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, DefaultResolver); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index 6bbfa81b68..1316142b4c 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -1,8 +1,12 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; using Xunit; namespace BenchmarkDotNet.Tests.Engine @@ -11,6 +15,10 @@ public class EngineFactoryTests { int timesBenchmarkCalled = 0, timesIdleCalled = 0; int timesGlobalSetupCalled = 0, timesGlobalCleanupCalled = 0, timesIterationSetupCalled = 0, timesIterationCleanupCalled = 0; + + TimeSpan IterationTime => TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds()); + + IResolver DefaultResolver => BenchmarkRunner.DefaultResolver; void GlobalSetup() => timesGlobalSetupCalled++; void IterationSetup() => timesIterationSetupCalled++; @@ -22,7 +30,7 @@ public class EngineFactoryTests void VeryTimeConsumingSingle(long _) { timesBenchmarkCalled++; - Thread.Sleep(TimeSpan.FromMilliseconds(EngineResolver.Instance.Resolve(Job.Default, RunMode.IterationTimeCharacteristic).ToMilliseconds())); + Thread.Sleep(IterationTime); } void InstantSingle(long _) => timesBenchmarkCalled++; @@ -31,8 +39,35 @@ void VeryTimeConsumingSingle(long _) void IdleSingle(long _) => timesIdleCalled++; void Idle16(long _) => timesIdleCalled += 16; + public static IEnumerable JobsWhichDontRequireJitting() + { + yield return new object[]{ Job.Dry }; + yield return new object[]{ Job.Default.With(RunStrategy.ColdStart) }; + yield return new object[]{ Job.Default.With(RunStrategy.Monitoring) }; + } + + [Theory] + [MemberData(nameof(JobsWhichDontRequireJitting))] + public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled(Job job) + { + var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Throwing, job: job); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal(0, timesIterationSetupCalled); + Assert.Equal(0, timesBenchmarkCalled); + Assert.Equal(0, timesIdleCalled); + Assert.Equal(0, timesIterationCleanupCalled); + Assert.Equal(0, timesGlobalCleanupCalled); + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + [Fact] - public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSettings() + public void ForDefaultSettingsVeryTimeConsumingBenchmarksAreExecutedOncePerIterationWithoutOverheadDeduction() { var engineParameters = CreateEngineParameters(mainSingleAction: VeryTimeConsumingSingle, mainMultiAction: Throwing, job: Job.Default); @@ -47,30 +82,14 @@ public void VeryTimeConsumingBenchmarksAreExecutedOncePerIterationForDefaultSett Assert.Equal(1, engine.TargetJob.Run.InvocationCount); // call the benchmark once per iteration Assert.Equal(1, engine.TargetJob.Run.UnrollFactor); // no unroll factor + + Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way + Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case engine.Dispose(); // cleanup is called as part of dispode Assert.Equal(1, timesGlobalCleanupCalled); } - - [Fact] - public void ForJobsThatDontRequireJittingOnlyGlobalSetupIsCalled() - { - var engineParameters = CreateEngineParameters(mainSingleAction: Throwing, mainMultiAction: Throwing, job: Job.Dry); - - var engine = new EngineFactory().CreateReadyToRun(engineParameters); - - Assert.Equal(1, timesGlobalSetupCalled); - Assert.Equal(0, timesIterationSetupCalled); - Assert.Equal(0, timesBenchmarkCalled); - Assert.Equal(0, timesIdleCalled); - Assert.Equal(0, timesIterationCleanupCalled); - Assert.Equal(0, timesGlobalCleanupCalled); - - engine.Dispose(); - - Assert.Equal(1, timesGlobalCleanupCalled); - } [Fact] public void ForJobsWithExplicitUnrollFactorTheGlobalSetupIsCalledAndMultiActionCodeGetsJitted() @@ -92,6 +111,8 @@ private void AssertGlobalSetupWasCalledAndMultiActionGotJitted(Job job) Assert.Equal(16, timesIdleCalled); Assert.Equal(2, timesIterationCleanupCalled); Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched engine.Dispose(); @@ -111,6 +132,8 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit Assert.Equal(1 + 16, timesIdleCalled); Assert.Equal((1+1) * (1+1), timesIterationCleanupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.False(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // remains untouched Assert.False(engine.TargetJob.Run.HasValue(RunMode.InvocationCountCharacteristic)); @@ -119,6 +142,49 @@ public void NonVeryTimeConsumingBenchmarksAreExecutedMoreThanOncePerIterationWit Assert.Equal(1, timesGlobalCleanupCalled); } + [Fact] + public void DontRunThePilotIfThePilotRequirementIsMetDuringWarmup() + { + var unrollFactor = Job.Default.ResolveValue(RunMode.UnrollFactorCharacteristic, DefaultResolver); + var mediumTime = TimeSpan.FromMilliseconds(IterationTime.TotalMilliseconds / unrollFactor); + + void MediumSingle(long _) + { + timesBenchmarkCalled++; + + Thread.Sleep(mediumTime); + } + + void MediumMultiple(long _) + { + timesBenchmarkCalled += unrollFactor; + + for (int i = 0; i < unrollFactor; i++) // the real unroll factor obviously does not use loop ;) + Thread.Sleep(mediumTime); + } + + var engineParameters = CreateEngineParameters(mainSingleAction: MediumSingle, mainMultiAction: MediumMultiple, job: Job.Default); + + var engine = new EngineFactory().CreateReadyToRun(engineParameters); + + Assert.Equal(1, timesGlobalSetupCalled); + Assert.Equal((1+1) * (1+1), timesIterationSetupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(1 + unrollFactor, timesBenchmarkCalled); + Assert.Equal(1 + unrollFactor, timesIdleCalled); + Assert.Equal((1+1) * (1+1), timesIterationCleanupCalled); // (once for single and & once for 16) x (1x for Idle + 1x for Target) + Assert.Equal(0, timesGlobalCleanupCalled); + + Assert.Equal(unrollFactor, engine.TargetJob.Run.InvocationCount); // no need to run pilot! + Assert.Equal(unrollFactor, engine.TargetJob.Run.UnrollFactor); // remains the same! + + Assert.True(engine.TargetJob.Run.HasValue(AccuracyMode.EvaluateOverheadCharacteristic)); // is set to false in explicit way + Assert.False(engine.TargetJob.Accuracy.EvaluateOverhead); // don't evaluate overhead in that case + + engine.Dispose(); + + Assert.Equal(1, timesGlobalCleanupCalled); + } + private EngineParameters CreateEngineParameters(Action mainSingleAction, Action mainMultiAction, Job job) => new EngineParameters {