obtain GC stats in separate iteration run, no overhead, support for i…

…teration setup&cleanup, fixes dotnet#606

src/BenchmarkDotNet.Core/Code/CodeGenerator.cs

@@ -7,17 +7,20 @@
 using System.Text;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Running;
+using RunMode = BenchmarkDotNet.Jobs.RunMode;
 
 namespace BenchmarkDotNet.Code
 {
     internal static class CodeGenerator
     {
-        internal static string Generate(Benchmark benchmark)
+        internal static string Generate(Benchmark benchmark, IConfig config)
         {
             var provider = GetDeclarationsProvider(benchmark.Target);
 
@@ -46,6 +49,7 @@ internal static string Generate(Benchmark benchmark)
                 Replace("$ShadowCopyDefines$", useShadowCopy ? "#define SHADOWCOPY" : null).
                 Replace("$ShadowCopyFolderPath$", shadowCopyFolderPath).
                 Replace("$Ref$", provider.UseRefKeyword ? "ref" : null).
+                Replace("$MeasureGcStats$", config.HasMemoryDiagnoser() ? "true" : "false").
                 ToString();
 
             text = Unroll(text, benchmark.Job.ResolveValue(RunMode.UnrollFactorCharacteristic, EnvResolver.Instance));

src/BenchmarkDotNet.Core/Configs/ConfigExtensions.cs

@@ -44,10 +44,12 @@ public static IDiagnoser GetCompositeDiagnoser(this IConfig config, Benchmark be
         public static IConfig RemoveBenchmarkFiles(this IConfig config) => config.KeepBenchmarkFiles(false);
 
         public static ReadOnlyConfig AsReadOnly(this IConfig config) =>
-            config is ReadOnlyConfig r
-                ? r
+            config is ReadOnlyConfig readOnly
+                ? readOnly
                 : new ReadOnlyConfig(config);
 
+        public static bool HasMemoryDiagnoser(this IConfig config) => config.GetDiagnosers().Any(diagnoser => diagnoser is MemoryDiagnoser);
+
         private static IConfig With(this IConfig config, Action<ManualConfig> addAction)
         {
             var manualConfig = ManualConfig.Create(config);

src/BenchmarkDotNet.Core/Diagnosers/MemoryDiagnoser.cs

@@ -9,7 +9,6 @@
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Extensions;
-using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Validators;
 
@@ -40,18 +39,7 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
 
         public void DisplayResults(ILogger logger) { }
 
-        public RunMode GetRunMode(Benchmark benchmark)
-        {
-            // for .NET Core we don't need to enable any kind of monitoring
-            // the allocated memory is available via GC's API
-            // so we don't need to perform any extra run
-            if (benchmark.Job.ResolveValue(EnvMode.RuntimeCharacteristic, EnvResolver.Instance) is CoreRuntime)
-                return RunMode.NoOverhead;
-
-            // for classic .NET we need to enable AppDomain.MonitoringIsEnabled
-            // which may cause overhead, so we perform an extra run to collect stats about allocated memory
-            return RunMode.ExtraRun; 
-        }
+        public RunMode GetRunMode(Benchmark benchmark) => RunMode.NoOverhead; 
 
         public void ProcessResults(DiagnoserResults results) 
             => this.results.Add(results.Benchmark, results.GcStats);
@@ -63,10 +51,7 @@ public class AllocationColumn : IColumn
         {
             private readonly Dictionary<Benchmark, GcStats> results;
 
-            public AllocationColumn(Dictionary<Benchmark, GcStats> results)
-            {
-                this.results = results;
-            }
+            public AllocationColumn(Dictionary<Benchmark, GcStats> results) => this.results = results;
 
             public string Id => nameof(AllocationColumn);
             public string ColumnName => "Allocated";

src/BenchmarkDotNet.Core/Engines/ConsoleHost.cs

@@ -9,15 +9,12 @@ public sealed class ConsoleHost : IHost
         private readonly TextWriter outWriter;
         private readonly TextReader inReader;
 
-        public ConsoleHost([NotNull]TextWriter outWriter, [NotNull]TextReader inReader, bool hasDiagnoserAttached)
+        public ConsoleHost([NotNull]TextWriter outWriter, [NotNull]TextReader inReader)
         {
             this.outWriter = outWriter ?? throw new ArgumentNullException(nameof(outWriter));
             this.inReader = inReader ?? throw new ArgumentNullException(nameof(inReader));
-            IsDiagnoserAttached = hasDiagnoserAttached;
         }
 
-        public bool IsDiagnoserAttached { get; }
-
         public void Write(string message) => outWriter.Write(message);
 
         public void WriteLine() => outWriter.WriteLine();
@@ -26,9 +23,6 @@ public ConsoleHost([NotNull]TextWriter outWriter, [NotNull]TextReader inReader,
 
         public void SendSignal(HostSignal hostSignal)
         {
-            if(!IsDiagnoserAttached) // no need to send the signal, nobody is listening for it
-                return;
-
             WriteLine(Engine.Signals.ToMessage(hostSignal));
 
             // read the response from Parent process, make the communication blocking

src/BenchmarkDotNet.Core/Engines/Engine.cs

@@ -17,7 +17,6 @@ public class Engine : IEngine
         public const int MinInvokeCount = 4;
 
         public IHost Host { get; }
-        public bool IsDiagnoserAttached { get; }
         public Action<long> MainAction { get; }
         public Action Dummy1Action { get; }
         public Action Dummy2Action { get; }
@@ -41,16 +40,17 @@ public class Engine : IEngine
         private readonly EnginePilotStage pilotStage;
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineTargetStage targetStage;
-        private bool isJitted, isPreAllocated;
-        private int forcedFullGarbageCollections;
+        private readonly bool includeMemoryStats;
+        private bool isJitted;
 
         internal Engine(
             IHost host,
             Action dummy1Action, Action dummy2Action, Action dummy3Action, Action<long> idleAction, Action<long> mainAction, Job targetJob,
-            Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke)
+            Action globalSetupAction, Action globalCleanupAction, Action iterationSetupAction, Action iterationCleanupAction, long operationsPerInvoke,
+            bool includeMemoryStats)
         {
+
             Host = host;
-            IsDiagnoserAttached = host.IsDiagnoserAttached;
             IdleAction = idleAction;
             Dummy1Action = dummy1Action;
             Dummy2Action = dummy2Action;
@@ -62,6 +62,7 @@ internal Engine(
             IterationSetupAction = iterationSetupAction;
             IterationCleanupAction = iterationCleanupAction;
             OperationsPerInvoke = operationsPerInvoke;
+            this.includeMemoryStats = includeMemoryStats;
 
             Resolver = new CompositeResolver(BenchmarkRunnerCore.DefaultResolver, EngineResolver.Instance);
 
@@ -77,17 +78,6 @@ internal Engine(
             targetStage = new EngineTargetStage(this);
         }
 
-        public void PreAllocate()
-        {
-            var list = new List<Measurement> { new Measurement(), new Measurement() };
-            list.Sort(); // provoke JIT, static ctors etc (was allocating 1740 bytes with first call)
-
-            // ReSharper disable once CompareOfFloatsByEqualityOperator
-            if (TimeUnit.All == null || list[0].Nanoseconds != default(double))
-                throw new Exception("just use this things here to provoke static ctor");
-            isPreAllocated = true;
-        }
-
         public void Jitting()
         {
             // first signal about jitting is raised from auto-generated Program.cs, look at BenchmarkProgram.txt
@@ -103,8 +93,6 @@ public RunResults Run()
         {
             if (Strategy.NeedsJitting() != isJitted)
                 throw new Exception($"You must{(Strategy.NeedsJitting() ? "" : " not")} call Jitting() first (Strategy = {Strategy})!");
-            if (!isPreAllocated)
-                throw new Exception("You must call PreAllocate() first!");
 
             long invokeCount = InvocationCount;
             IReadOnlyList<Measurement> idle = null;
@@ -125,22 +113,16 @@ public RunResults Run()
                 warmupStage.RunMain(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring);
             }
 
-            // we enable monitoring after pilot & warmup, just to ignore the memory allocated by these runs
-            EnableMonitoring();
-
             Host.BeforeMainRun();
 
-            forcedFullGarbageCollections = 0; // zero it in case the Engine instance is reused (InProcessToolchain)
-            var initialGcStats = GcStats.ReadInitial(IsDiagnoserAttached);
-
             var main = targetStage.RunMain(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring);
 
-            var finalGcStats = GcStats.ReadFinal(IsDiagnoserAttached);
-            var forcedCollections = GcStats.FromForced(forcedFullGarbageCollections);
-            var workGcHasDone = finalGcStats - forcedCollections - initialGcStats;
-
             Host.AfterMainRun();
 
+            var workGcHasDone = includeMemoryStats 
+                ? MeasureGcStats(new IterationData(IterationMode.MainTarget, 0, invokeCount, UnrollFactor)) 
+                : GcStats.Empty;
+
             bool removeOutliers = TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, Resolver);
 
             return new RunResults(idle, main, removeOutliers, workGcHasDone);
@@ -167,11 +149,31 @@ public Measurement RunIteration(IterationData data)
 
             // Results
             var measurement = new Measurement(0, data.IterationMode, data.Index, totalOperations, clockSpan.GetNanoseconds());
-            if (!IsDiagnoserAttached) WriteLine(measurement.ToOutputLine());
+            WriteLine(measurement.ToOutputLine());
 
             return measurement;
         }
 
+        private GcStats MeasureGcStats(IterationData data)
+        {
+            // we enable monitoring after main target run, for this single iteration which is executed at the end
+            // so even if we enable AppDomain monitoring in separate process
+            // it does not matter, because we have already obtained the results!
+            EnableMonitoring();
+
+            IterationSetupAction(); // we run iteration setup first, so even if it allocates, it is not included in the results
+
+            var initialGcStats = GcStats.ReadInitial();
+
+            MainAction(data.InvokeCount / data.UnrollFactor);
+
+            var finalGcStats = GcStats.ReadFinal();
+
+            IterationCleanupAction(); // we run iteration cleanup after collecting GC stats 
+
+            return (finalGcStats - initialGcStats).WithTotalOperations(data.InvokeCount);
+        }
+
         private void GcCollect()
         {
             if (!ForceAllocations)
@@ -185,45 +187,22 @@ private void ForceGcCollect()
             GC.Collect();
             GC.WaitForPendingFinalizers();
             GC.Collect();
-
-            forcedFullGarbageCollections += 2;
-        }
-
-        public void WriteLine(string text)
-        {
-            EnsureNothingIsPrintedWhenDiagnoserIsAttached();
-
-            Host.WriteLine(text);
         }
 
-        public void WriteLine()
-        {
-            EnsureNothingIsPrintedWhenDiagnoserIsAttached();
+        public void WriteLine(string text) => Host.WriteLine(text);
 
-            Host.WriteLine();
-        }
+        public void WriteLine() => Host.WriteLine();
 
         private void EnableMonitoring()
         {
-            if (!IsDiagnoserAttached) // it could affect the results, we do this in separate, diagnostics-only run
-                return;
 #if CLASSIC
-            if (RuntimeInformation.IsMono()
-            ) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono
+            if (RuntimeInformation.IsMono()) // Monitoring is not available in Mono, see http://stackoverflow.com/questions/40234948/how-to-get-the-number-of-allocated-bytes-in-mono
                 return;
 
             AppDomain.MonitoringIsEnabled = true;
 #endif
         }
 
-        private void EnsureNothingIsPrintedWhenDiagnoserIsAttached()
-        {
-            if (IsDiagnoserAttached)
-            {
-                throw new InvalidOperationException("to avoid memory allocations we must not print anything when diagnoser is still attached");
-            }
-        }
-
         [UsedImplicitly]
         public static class Signals
         {

src/BenchmarkDotNet.Core/Engines/EngineFactory.cs

@@ -32,7 +32,8 @@ public IEngine Create(EngineParameters engineParameters)
                 engineParameters.GlobalCleanupAction,
                 engineParameters.IterationSetupAction,
                 engineParameters.IterationCleanupAction,
-                engineParameters.OperationsPerInvoke);
+                engineParameters.OperationsPerInvoke,
+                engineParameters.MeasureGcStats);
         }
     }
 }

src/BenchmarkDotNet.Core/Engines/EngineParameters.cs

@@ -19,5 +19,6 @@ public class EngineParameters
         public Action IterationSetupAction { get; set; } = null;
         public Action IterationCleanupAction { get; set; } = null;
         public IResolver Resolver { get; set; }
+        public bool MeasureGcStats { get; set; }
     }
 }

src/BenchmarkDotNet.Core/Engines/EnginePilotStage.cs

@@ -70,7 +70,7 @@ private long RunAuto()
 
                 invokeCount *= 2;
             }
-            if (!IsDiagnoserAttached) WriteLine();
+            WriteLine();
 
             return invokeCount;
         }
@@ -100,7 +100,7 @@ private long RunSpecific()
 
                 invokeCount = newInvokeCount;
             }
-            if (!IsDiagnoserAttached) WriteLine();
+            WriteLine();
 
             return invokeCount;
         }

src/BenchmarkDotNet.Core/Engines/EngineStage.cs

@@ -9,13 +9,9 @@ public class EngineStage
     {
         private readonly IEngine engine;
 
-        protected EngineStage(IEngine engine)
-        {
-            this.engine = engine;
-        }
+        protected EngineStage(IEngine engine) => this.engine = engine;
 
         protected Job TargetJob => engine.TargetJob;
-        protected bool IsDiagnoserAttached => engine.IsDiagnoserAttached;
 
         protected Measurement RunIteration(IterationMode mode, int index, long invokeCount, int unrollFactor)
         {

src/BenchmarkDotNet.Core/Engines/EngineTargetStage.cs

@@ -19,15 +19,13 @@ public class EngineTargetStage : EngineStage
         private readonly double maxRelativeError;
         private readonly TimeInterval? maxAbsoluteError;
         private readonly bool removeOutliers;
-        private readonly MeasurementsPool measurementsPool;
 
         public EngineTargetStage(IEngine engine) : base(engine)
         {
             targetCount = engine.TargetJob.ResolveValueAsNullable(RunMode.TargetCountCharacteristic);
             maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver);
             maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic);
             removeOutliers = engine.TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, engine.Resolver);
-            measurementsPool = MeasurementsPool.PreAllocate(10, MaxIterationCount, targetCount);
         }
 
         public IReadOnlyList<Measurement> RunIdle(long invokeCount, int unrollFactor) 
@@ -43,8 +41,8 @@ internal IReadOnlyList<Measurement> Run(long invokeCount, IterationMode iteratio
 
         private List<Measurement> RunAuto(long invokeCount, IterationMode iterationMode, int unrollFactor)
         {
-            var measurements = measurementsPool.Next();
-            var measurementsForStatistics = measurementsPool.Next();
+            var measurements = new List<Measurement>(MaxIterationCount);
+            var measurementsForStatistics = new List<Measurement>(MaxIterationCount);
 
             int iterationCounter = 0;
             bool isIdle = iterationMode.IsIdle();
@@ -69,18 +67,19 @@ private List<Measurement> RunAuto(long invokeCount, IterationMode iterationMode,
                 if (iterationCounter >= MaxIterationCount || (isIdle && iterationCounter >= MaxIdleIterationCount))
                     break;
             }
-            if (!IsDiagnoserAttached) WriteLine();
+            WriteLine();
+
             return measurements;
         }
 
         private List<Measurement> RunSpecific(long invokeCount, IterationMode iterationMode, int iterationCount, int unrollFactor)
         {
-            var measurements = measurementsPool.Next();
+            var measurements = new List<Measurement>(MaxIterationCount);
 
             for (int i = 0; i < iterationCount; i++)
                 measurements.Add(RunIteration(iterationMode, i + 1, invokeCount, unrollFactor));
 
-            if (!IsDiagnoserAttached) WriteLine();
+            WriteLine();
 
             return measurements;
         }