Make MaxIterationCount configurable, keep current value as default, f…

…ixes dotnet#763

src/BenchmarkDotNet/Configs/DefaultConfig.cs

@@ -56,7 +56,7 @@ public IEnumerable<IValidator> GetValidators()
 #if !DEBUG
             yield return JitOptimizationsValidator.FailOnError;
 #endif
-            yield return UnrollFactorValidator.Default;
+            yield return RunModeValidator.FailOnError;
             yield return GenericBenchmarksValidator.DontFailOnError;
         }
 

src/BenchmarkDotNet/Engines/EngineResolver.cs

@@ -7,13 +7,19 @@ namespace BenchmarkDotNet.Engines
 {
     public class EngineResolver : Resolver
     {
+        internal const int DefaultMinTargetIterationCount = 15;
+        internal const int DefaultMaxTargetIterationCount = 100;
+
         public static readonly IResolver Instance = new EngineResolver();
 
         private EngineResolver()
         {
             Register(RunMode.RunStrategyCharacteristic, () => RunStrategy.Throughput);
             Register(RunMode.IterationTimeCharacteristic, () => TimeInterval.Millisecond * 500);
 
+            Register(RunMode.MinTargetIterationCountCharacteristic, () => DefaultMinTargetIterationCount);
+            Register(RunMode.MaxTargetIterationCountCharacteristic, () => DefaultMaxTargetIterationCount);
+
             Register(AccuracyMode.MaxRelativeErrorCharacteristic, () => 0.02);
             Register(AccuracyMode.MinIterationTimeCharacteristic, () => TimeInterval.Millisecond * 500);
             Register(AccuracyMode.MinInvokeCountCharacteristic, () => 4);

src/BenchmarkDotNet/Engines/EngineTargetStage.cs

@@ -9,8 +9,6 @@ namespace BenchmarkDotNet.Engines
 {
     public class EngineTargetStage : EngineStage
     {
-        internal const int MinIterationCount = 15;
-        internal const int MaxIterationCount = 100;
         internal const int MaxIdleIterationCount = 20;
         internal const double MaxIdleRelativeError = 0.05;
         internal const int DefaultTargetCount = 10;
@@ -19,13 +17,17 @@ public class EngineTargetStage : EngineStage
         private readonly double maxRelativeError;
         private readonly TimeInterval? maxAbsoluteError;
         private readonly bool removeOutliers;
+        private readonly int minIterationCount;
+        private readonly int maxIterationCount;
 
         public EngineTargetStage(IEngine engine) : base(engine)
         {
             targetCount = engine.TargetJob.ResolveValueAsNullable(RunMode.TargetCountCharacteristic);
             maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver);
             maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic);
             removeOutliers = engine.TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, engine.Resolver);
+            minIterationCount = engine.TargetJob.ResolveValue(RunMode.MinTargetIterationCountCharacteristic, engine.Resolver);
+            maxIterationCount = engine.TargetJob.ResolveValue(RunMode.MaxTargetIterationCountCharacteristic, engine.Resolver);
         }
 
         public IReadOnlyList<Measurement> RunIdle(long invokeCount, int unrollFactor) 
@@ -41,8 +43,8 @@ internal IReadOnlyList<Measurement> Run(long invokeCount, IterationMode iteratio
 
         private List<Measurement> RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor)
         {
-            var measurements = new List<Measurement>(MaxIterationCount);
-            var measurementsForStatistics = new List<Measurement>(MaxIterationCount);
+            var measurements = new List<Measurement>(maxIterationCount);
+            var measurementsForStatistics = new List<Measurement>(maxIterationCount);
 
             int iterationCounter = 0;
             bool isIdle = iterationMode.IsIdle();
@@ -61,10 +63,10 @@ private List<Measurement> RunAuto(long invokeCount, IterationMode iterationMode,
                 double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
                 double maxError = Math.Min(maxError1, maxError2);
 
-                if (iterationCounter >= MinIterationCount && actualError < maxError)
+                if (iterationCounter >= minIterationCount && actualError < maxError)
                     break;
 
-                if (iterationCounter >= MaxIterationCount || (isIdle && iterationCounter >= MaxIdleIterationCount))
+                if (iterationCounter >= maxIterationCount || (isIdle && iterationCounter >= MaxIdleIterationCount))
                     break;
             }
             WriteLine();
@@ -74,7 +76,7 @@ private List<Measurement> RunAuto(long invokeCount, IterationMode iterationMode,
 
         private List<Measurement> RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor)
         {
-            var measurements = new List<Measurement>(MaxIterationCount);
+            var measurements = new List<Measurement>(iterationCount);
 
             for (int i = 0; i < iterationCount; i++)
                 measurements.Add(RunIteration(iterationMode, i + 1, invokeCount, unrollFactor));

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -33,6 +33,8 @@ public static class JobExtensions
         public static Job WithIterationTime(this Job job, TimeInterval time) => job.WithCore(j => j.Run.IterationTime = time);
         public static Job WithInvocationCount(this Job job, int count) => job.WithCore(j => j.Run.InvocationCount = count);
         public static Job WithUnrollFactor(this Job job, int factor) => job.WithCore(j => j.Run.UnrollFactor = factor);
+        public static Job WithMinTargetIterationCount(this Job job, int count) => job.WithCore(j => j.Run.MinTargetIterationCount = count);
+        public static Job WithMaxTargetIterationCount(this Job job, int count) => job.WithCore(j => j.Run.MaxTargetIterationCount = count);
 
         // Infrastructure
         public static Job With(this Job job, IToolchain toolchain) => job.WithCore(j => j.Infrastructure.Toolchain = toolchain);

src/BenchmarkDotNet/Jobs/RunMode.cs

@@ -16,6 +16,8 @@ public sealed class RunMode : JobMode<RunMode>
         public static readonly Characteristic<TimeInterval> IterationTimeCharacteristic = CreateCharacteristic<TimeInterval>(nameof(IterationTime));
         public static readonly Characteristic<int> InvocationCountCharacteristic = CreateCharacteristic<int>(nameof(InvocationCount));
         public static readonly Characteristic<int> UnrollFactorCharacteristic = CreateCharacteristic<int>(nameof(UnrollFactor));
+        public static readonly Characteristic<int> MinTargetIterationCountCharacteristic = CreateCharacteristic<int>(nameof(MinTargetIterationCount));
+        public static readonly Characteristic<int> MaxTargetIterationCountCharacteristic = CreateCharacteristic<int>(nameof(MaxTargetIterationCount));
 
         public static readonly RunMode Dry = new RunMode(nameof(Dry))
         {
@@ -94,6 +96,8 @@ public int WarmupCount
 
         /// <summary>
         /// How many target iterations should be performed
+        /// If specified, <see cref="MinTargetIterationCount"/> will be ignored.
+        /// If specified, <see cref="MaxTargetIterationCount"/> will be ignored.
         /// </summary>
         public int TargetCount
         {
@@ -129,5 +133,25 @@ public int UnrollFactor
             get { return UnrollFactorCharacteristic[this]; }
             set { UnrollFactorCharacteristic[this] = value; }
         }
+
+        /// <summary>
+        /// Minimum count of target iterations that should be performed
+        /// The default is 15
+        /// </summary>
+        public int MinTargetIterationCount
+        {
+            get { return MinTargetIterationCountCharacteristic[this]; }
+            set { MinTargetIterationCountCharacteristic[this] = value; }
+        }
+
+        /// <summary>
+        /// Maximum count of target iterations that should be performed
+        /// The default is 100 
+        /// </summary>
+        public int MaxTargetIterationCount
+        {
+            get { return MaxTargetIterationCountCharacteristic[this]; }
+            set { MaxTargetIterationCountCharacteristic[this] = value; }
+        }
     }
 }

src/BenchmarkDotNet/Validators/CompositeValidator.cs

@@ -9,7 +9,7 @@ internal class CompositeValidator : IValidator
         {
             BaselineValidator.FailOnError,
             SetupCleanupValidator.FailOnError,
-            UnrollFactorValidator.Default,
+            RunModeValidator.FailOnError,
             DiagnosersValidator.Default,
             CompilationValidator.Default,
             ConfigValidator.Default,

src/BenchmarkDotNet/Validators/RunModeValidator.cs

@@ -0,0 +1,52 @@
+using System.Collections.Generic;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
+
+namespace BenchmarkDotNet.Validators
+{
+    public class RunModeValidator : IValidator
+    {
+        public static readonly IValidator FailOnError = new RunModeValidator();
+
+        private RunModeValidator() { }
+
+        public bool TreatsWarningsAsErrors => true;
+
+        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+        {
+            var resolver = new CompositeResolver(EnvResolver.Instance, EngineResolver.Instance); // TODO: use specified resolver.
+            foreach (var benchmark in validationParameters.Benchmarks)
+            {
+                var run = benchmark.Job.Run;
+                int unrollFactor = run.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver);
+                if (unrollFactor <= 0)
+                {
+                    yield return new ValidationError(true, $"Specified UnrollFactor ({unrollFactor}) must be greater than zero", benchmark);
+                }
+                else if (run.HasValue(RunMode.InvocationCountCharacteristic))
+                {
+                    int invocationCount = run.InvocationCount;
+                    if (invocationCount % unrollFactor != 0)
+                    {
+                        string message = $"Specified InvocationCount ({invocationCount}) must be a multiple of UnrollFactor ({unrollFactor})";
+                        yield return new ValidationError(true, message, benchmark);
+                    }
+                }
+
+                int minTargetCount = run.ResolveValue(RunMode.MinTargetIterationCountCharacteristic, resolver);
+                int maxTargetCount = run.ResolveValue(RunMode.MaxTargetIterationCountCharacteristic, resolver);
+
+                if (minTargetCount <= 0)
+                    yield return new ValidationError(true, $"{nameof(RunMode.MinTargetIterationCount)} must be greater than zero (was {minTargetCount})", benchmark);
+
+                if (maxTargetCount <= 0)
+                    yield return new ValidationError(true, $"{nameof(RunMode.MaxTargetIterationCount)} must be greater than zero (was {maxTargetCount})", benchmark);
+
+                if (minTargetCount >= maxTargetCount)
+                    yield return new ValidationError(true, $"{nameof(RunMode.MaxTargetIterationCount)} must be greater than {nameof(RunMode.MinTargetIterationCount)} (was {maxTargetCount} and {minTargetCount})", benchmark);
+            }
+        }
+    }
+}

tests/BenchmarkDotNet.Tests/Engine/EngineTargetStageTests.cs

@@ -10,8 +10,8 @@ namespace BenchmarkDotNet.Tests.Engine
 {
     public class EngineTargetStageTests
     {
-        private const int MinIterationCount = EngineTargetStage.MinIterationCount;
-        private const int MaxIterationCount = EngineTargetStage.MaxIterationCount;
+        private const int MinIterationCount = EngineResolver.DefaultMinTargetIterationCount;
+        private const int MaxIterationCount = EngineResolver.DefaultMaxTargetIterationCount;
         private const int MaxIdleIterationCount = EngineTargetStage.MaxIdleIterationCount;
 
         private readonly ITestOutputHelper output;

tests/BenchmarkDotNet.Tests/JobTests.cs

@@ -401,7 +401,7 @@ public static void Test07GetCharacteristics()
                 "MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;RemoveOutliers;Env;Affinity;" +
                 "Jit;Platform;Runtime;Gc;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;" +
                 "RetainVm;Server;Infrastructure;Arguments;BuildConfiguration;Clock;EngineFactory;EnvironmentVariables;Toolchain;Meta;IsBaseline;Run;InvocationCount;IterationTime;" +
-                "LaunchCount;RunStrategy;TargetCount;UnrollFactor;WarmupCount");
+                "LaunchCount;MaxTargetIterationCount;MinTargetIterationCount;RunStrategy;TargetCount;UnrollFactor;WarmupCount");
         }
     }
 }