don't execute long operations more than once per iteration

src/BenchmarkDotNet/Engines/Engine.cs

@@ -41,10 +41,10 @@ public class Engine : IEngine
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineTargetStage targetStage;
         private readonly bool includeMemoryStats;
-        private bool isJitted;
 
         internal Engine(
             IHost host,
+            IResolver resolver,
             Action dummy1Action, Action dummy2Action, Action dummy3Action, Action<long> idleAction, Action<long> mainAction, Job targetJob,
             Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke,
             bool includeMemoryStats)
@@ -64,7 +64,7 @@ internal Engine(
             OperationsPerInvoke = operationsPerInvoke;
             this.includeMemoryStats = includeMemoryStats;
 
-            Resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance);
+            Resolver = resolver;
 
             Clock = targetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, Resolver);
             ForceAllocations = targetJob.ResolveValue(GcMode.ForceCharacteristic, Resolver);
@@ -78,22 +78,10 @@ internal Engine(
             targetStage = new EngineTargetStage(this);
         }
 
-        public void Jitting()
-        {
-            // first signal about jitting is raised from auto-generated Program.cs, look at BenchmarkProgram.txt
-            Dummy1Action.Invoke();
-            MainAction.Invoke(1);
-            Dummy2Action.Invoke();
-            IdleAction.Invoke(1);
-            Dummy3Action.Invoke();
-            isJitted = true;
-        }
+        public void Dispose() => GlobalCleanupAction?.Invoke();
 
         public RunResults Run()
         {
-            if (Strategy.NeedsJitting() != isJitted)
-                throw new Exception($"You must{(Strategy.NeedsJitting() ? "" : " not")} call Jitting() first (Strategy = {Strategy})!");
-
             long invokeCount = InvocationCount;
             IReadOnlyList<Measurement> idle = null;
 

src/BenchmarkDotNet/Engines/EngineFactory.cs

@@ -1,39 +1,95 @@
 using System;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Horology;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 
 namespace BenchmarkDotNet.Engines
 {
-    // TODO: Default instance?
     public class EngineFactory : IEngineFactory
     {
-        public IEngine Create(EngineParameters engineParameters)
+        public IEngine CreateReadyToRun(EngineParameters engineParameters)
         {
-            if (engineParameters.MainAction == null)
-                throw new ArgumentNullException(nameof(engineParameters.MainAction));
+            if (engineParameters.MainSingleAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.MainSingleAction));
+            if (engineParameters.MainMultiAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.MainMultiAction));
             if (engineParameters.Dummy1Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy1Action));
             if (engineParameters.Dummy2Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy2Action));
             if (engineParameters.Dummy3Action == null)
                 throw new ArgumentNullException(nameof(engineParameters.Dummy3Action));
-            if (engineParameters.IdleAction == null)
-                throw new ArgumentNullException(nameof(engineParameters.IdleAction));
+            if (engineParameters.IdleSingleAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.IdleSingleAction));
+            if (engineParameters.IdleMultiAction == null)
+                throw new ArgumentNullException(nameof(engineParameters.IdleMultiAction));
             if(engineParameters.TargetJob == null)
                 throw new ArgumentNullException(nameof(engineParameters.TargetJob));
 
-            return new Engine(
+            var resolver = new CompositeResolver(BenchmarkRunner.DefaultResolver, EngineResolver.Instance);
+            var unrollFactor = engineParameters.TargetJob.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver);
+
+            engineParameters.GlobalSetupAction?.Invoke();
+
+            var needsJitting = engineParameters.TargetJob.ResolveValue(RunMode.RunStrategyCharacteristic, resolver).NeedsJitting();
+            if (!needsJitting)
+            {
+                // whatever it is, we can not interfere
+                return CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction);
+            }
+
+            var needsPilot = !engineParameters.TargetJob.HasValue(RunMode.InvocationCountCharacteristic);
+            if (needsPilot) 
+            {
+                var singleActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction);
+
+                var iterationTime = resolver.Resolve(engineParameters.TargetJob, RunMode.IterationTimeCharacteristic);
+                if (ShouldExecuteOncePerIteration(Jit(singleActionEngine, unrollFactor: 1), iterationTime))
+                {
+                    var reconfiguredJob = engineParameters.TargetJob.WithInvocationCount(1).WithUnrollFactor(1); // todo: consider if we should set the warmup count to 1!
+
+                    return CreateEngine(engineParameters, resolver, reconfiguredJob, engineParameters.IdleSingleAction, engineParameters.MainSingleAction);
+                }
+            }
+
+            // it's either a job with explicit configuration or not-very time consuming benchmark, just create the engine, Jit and return
+            var multiActionEngine = CreateEngine(engineParameters, resolver, engineParameters.TargetJob, engineParameters.IdleMultiAction, engineParameters.MainMultiAction);
+
+            DeadCodeEliminationHelper.KeepAliveWithoutBoxing(Jit(multiActionEngine, unrollFactor));
+
+            return multiActionEngine;
+        }
+
+        /// <summary>
+        /// returns true if it takes longer than the desired iteration time (0,5s by default) to execute benchmark once
+        /// </summary>
+        private static bool ShouldExecuteOncePerIteration(Measurement jit, TimeInterval iterationTime)
+            => TimeInterval.FromNanoseconds(jit.GetAverageNanoseconds()) > iterationTime;
+
+        private static Measurement Jit(Engine engine, int unrollFactor)
+        {
+            DeadCodeEliminationHelper.KeepAliveWithoutBoxing(engine.RunIteration(new IterationData(IterationMode.IdleJit, index: -1, invokeCount: unrollFactor, unrollFactor: unrollFactor))); // don't forget to JIT idle
+
+            return engine.RunIteration(new IterationData(IterationMode.Jit, index: -1, invokeCount: unrollFactor, unrollFactor: unrollFactor));
+        }
+
+        private static Engine CreateEngine(EngineParameters engineParameters, IResolver resolver, Job job, Action<long> idle, Action<long> main)
+            => new Engine(
                 engineParameters.Host,
+                resolver,
                 engineParameters.Dummy1Action,
                 engineParameters.Dummy2Action,
                 engineParameters.Dummy3Action,
-                engineParameters.IdleAction,
-                engineParameters.MainAction,
-                engineParameters.TargetJob,
+                idle,
+                main,
+                job,
                 engineParameters.GlobalSetupAction,
                 engineParameters.GlobalCleanupAction,
                 engineParameters.IterationSetupAction,
                 engineParameters.IterationCleanupAction,
                 engineParameters.OperationsPerInvoke,
                 engineParameters.MeasureGcStats);
-        }
     }
 }

src/BenchmarkDotNet/Engines/EngineParameters.cs

@@ -7,18 +7,19 @@ namespace BenchmarkDotNet.Engines
     public class EngineParameters
     {
         public IHost Host { get; set; }
-        public Action<long> MainAction { get; set; }
+        public Action<long> MainSingleAction { get; set; }
+        public Action<long> MainMultiAction { get; set; }
         public Action Dummy1Action { get; set; }
         public Action Dummy2Action { get; set; }
         public Action Dummy3Action { get; set; }
-        public Action<long> IdleAction { get; set; }
+        public Action<long> IdleSingleAction { get; set; }
+        public Action<long> IdleMultiAction { get; set; }
         public Job TargetJob { get; set; } = Job.Default;
         public long OperationsPerInvoke { get; set; } = 1;
         public Action GlobalSetupAction { get; set; } = null;
         public Action GlobalCleanupAction { get; set; } = null;
         public Action IterationSetupAction { get; set; } = null;
         public Action IterationCleanupAction { get; set; } = null;
-        public IResolver Resolver { get; set; }
         public bool MeasureGcStats { get; set; }
     }
 }

src/BenchmarkDotNet/Engines/IEngine.cs

@@ -6,7 +6,7 @@
 
 namespace BenchmarkDotNet.Engines
 {
-    public interface IEngine
+    public interface IEngine : IDisposable
     {
         [NotNull]
         IHost Host { get; }
@@ -36,12 +36,6 @@ public interface IEngine
 
         Measurement RunIteration(IterationData data);
 
-        /// <summary>
-        /// must perform jitting via warmup calls
-        /// <remarks>is called after first call to GlobalSetup, from the auto-generated benchmark process</remarks>
-        /// </summary>
-        void Jitting();
-
         RunResults Run();
     }
 }

src/BenchmarkDotNet/Engines/IEngineFactory.cs

@@ -2,6 +2,6 @@ namespace BenchmarkDotNet.Engines
 {
     public interface IEngineFactory
     {
-        IEngine Create(EngineParameters engineParameters);
+        IEngine CreateReadyToRun(EngineParameters engineParameters);
     }
 }

src/BenchmarkDotNet/Engines/IterationMode.cs

@@ -35,6 +35,11 @@ public enum IterationMode
         /// <summary>
         /// Unknown 
         /// </summary>
-        Unknown
+        Unknown,
+
+        /// <summary>
+        /// executing benchmark for the purpose of JIT wamup
+        /// </summary>
+        Jit, IdleJit
     }
 }

src/BenchmarkDotNet/Engines/IterationModeExtensions.cs

@@ -3,6 +3,6 @@
     public static class IterationModeExtensions
     {
         public static bool IsIdle(this IterationMode mode) 
-            => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget;
+            => mode == IterationMode.IdleWarmup || mode == IterationMode.IdleTarget || mode == IterationMode.IdleJit;
     }
 }

src/BenchmarkDotNet/Templates/BenchmarkType.txt

@@ -20,11 +20,13 @@
             var engineParameters = new BenchmarkDotNet.Engines.EngineParameters()
             {
                 Host = host,
-                MainAction = instance.MainMultiAction,
+                MainMultiAction = instance.MainMultiAction,
+                MainSingleAction = instance.MainSingleAction,
                 Dummy1Action = instance.Dummy1,
                 Dummy2Action = instance.Dummy2,
                 Dummy3Action = instance.Dummy3,
-                IdleAction = instance.IdleMultiAction,
+                IdleSingleAction = instance.IdleSingleAction,
+                IdleMultiAction = instance.IdleMultiAction,
                 GlobalSetupAction = instance.globalSetupAction,
                 GlobalCleanupAction = instance.globalCleanupAction,
                 IterationSetupAction = instance.iterationSetupAction,
@@ -34,23 +36,14 @@
                 MeasureGcStats = $MeasureGcStats$
             };
 
-            var engine = new $EngineFactoryType$().Create(engineParameters);
-
-            instance?.globalSetupAction();
-            instance?.iterationSetupAction();
-
-            if (job.ResolveValue(RunMode.RunStrategyCharacteristic, EngineResolver.Instance).NeedsJitting())
-				engine.Jitting(); // does first call to main action, must be executed after globalSetup() and iterationSetup()!
-
-            instance?.iterationCleanupAction();
-
-            var results = engine.Run();
-
-            instance?.globalCleanupAction();
+            using (var engine = new $EngineFactoryType$().CreateReadyToRun(engineParameters))
+            {
+                var results = engine.Run();
 
-            host.ReportResults(results); // printing costs memory, do this after runs
+                host.ReportResults(results); // printing costs memory, do this after runs
 
-			instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;)
+                instance.__TrickTheJIT__(); // compile the method for disassembler, but without actual run of the benchmark ;)
+			}
         }
 
         public delegate $IdleMethodReturnTypeName$ IdleDelegate($ArgumentsDefinition$);
@@ -124,6 +117,12 @@
             }
         }
 
+        private void IdleSingleAction(long _)
+        {
+            $LoadArguments$
+            consumer.Consume(idleDelegate($PassArguments$));
+        }
+
         private void MainMultiAction(long invokeCount)
         {
             $LoadArguments$
@@ -133,6 +132,12 @@
             }
         }
 
+        private void MainSingleAction(long _)
+        {
+            $LoadArguments$
+            consumer.Consume(targetDelegate($PassArguments$)$ConsumeField$);
+        }
+
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
         public $TargetMethodReturnType$ $DiassemblerEntryMethodName$()
         {
@@ -158,6 +163,14 @@
             DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result);
         }
 
+        private void IdleSingleAction(long _)
+        {
+            $LoadArguments$
+            $IdleMethodReturnTypeName$ result = default($IdleMethodReturnTypeName$);
+            result = idleDelegate($PassArguments$);
+            DeadCodeEliminationHelper.KeepAliveWithoutBoxing(result);
+        }
+
         private void MainMultiAction(long invokeCount)
         {
             $LoadArguments$
@@ -169,6 +182,14 @@
             NonGenericKeepAliveWithoutBoxing(result);
         }
 
+        private void MainSingleAction(long _)
+        {
+            $LoadArguments$
+            $TargetMethodReturnType$ result = default($TargetMethodReturnType$);
+            result = targetDelegate($PassArguments$);
+            NonGenericKeepAliveWithoutBoxing(result);
+        }
+
         // we must not simply use DeadCodeEliminationHelper.KeepAliveWithoutBoxing<T> because it's generic method
         // and stack-only types like Span<T> can not be generic type arguments http://adamsitnik.com/Span/#span-must-not-be-a-generic-type-argument
         [MethodImpl(MethodImplOptions.NoInlining)]
@@ -198,6 +219,14 @@
             }
             DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value);
         }
+
+        private void IdleSingleAction(long _)
+        {
+            $LoadArguments$
+            $IdleMethodReturnTypeName$ value = default($IdleMethodReturnTypeName$);
+            value = idleDelegate($PassArguments$);
+            DeadCodeEliminationHelper.KeepAliveWithoutBoxing(value);
+        }
 
         private $TargetMethodReturnType$ mainDefaultValueHolder = default($TargetMethodReturnType$);
 
@@ -211,6 +240,14 @@
             }
             DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias);
         }
+
+        private void MainSingleAction(long _)
+        {
+            $LoadArguments$
+            ref $TargetMethodReturnType$ alias = ref mainDefaultValueHolder;
+            alias = targetDelegate($PassArguments$);
+            DeadCodeEliminationHelper.KeepAliveWithoutBoxing(ref alias);
+        }
 
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
         public ref $TargetMethodReturnType$ $DiassemblerEntryMethodName$()
@@ -234,6 +271,12 @@
             }
         }
 
+        private void IdleSingleAction(long _)
+        {
+            $LoadArguments$
+            idleDelegate($PassArguments$);
+        }
+
         private void MainMultiAction(long invokeCount)
         {
             $LoadArguments$
@@ -242,6 +285,12 @@
                 targetDelegate($PassArguments$);@Unroll@
             }
         }
+
+        private void MainSingleAction(long _)
+        {
+            $LoadArguments$
+            targetDelegate($PassArguments$);
+        }
 
         [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
         public void $DiassemblerEntryMethodName$()