Refactor initialization logic to allow for enabling Memory Randomizat…

…ion (#1587)

* there is no need for the warmup workaround anymore, dotnet/BenchmarkDotNet#1573 has solved the problem

* allocate the array first to try to take advantage of memory randomization as it's usually the first thing called from GlobalSetup method

which with MemoryRandomization enabled is the first method called right after allocation of random-sized memory by BDN engine

* avoid having big global setup methods and try to have Targeted setups that allocate less

as Global Setup methods might get called right after random-size memory allocation

* avoid readonly fields, initialize them in [GlobalSetup] instead of ctors (to allow for re-allocation with different alignment)

* this code can be executed only once

* make sure that every setup creates a brand new delegate instead of combining with existing one

* the "WriteDeepUtf16" benchmark can report up to x4 more time when MemoryRandmization is enabled

this is due to having new _arrayBufferWriter every time and allocating a lot of memory
so we don't always allocate a new instance

* allocate the array in GlobalSetup, not in field initializer

src/benchmarks/micro/Serializers/Binary_FromStream.cs

@@ -45,10 +45,13 @@ public T BinaryFormatter_()
         public void SetupProtoBuffNet()
         {
             value = DataGenerator.Generate<T>();
+            if (memoryStream is null) // to ensure it's done only once
+            {
+                ProtoBuf.Meta.RuntimeTypeModel.Default.Add(typeof(DateTimeOffset), false).SetSurrogate(typeof(DateTimeOffsetSurrogate)); // https://stackoverflow.com/a/7046868
+            }
             // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
             memoryStream = new MemoryStream(capacity: short.MaxValue);
             memoryStream.Position = 0;
-            ProtoBuf.Meta.RuntimeTypeModel.Default.Add(typeof(DateTimeOffset), false).SetSurrogate(typeof(DateTimeOffsetSurrogate)); // https://stackoverflow.com/a/7046868
             ProtoBuf.Serializer.Serialize(memoryStream, value);
         }
 

src/benchmarks/micro/Serializers/Json_FromStream.cs

@@ -16,74 +16,55 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     public class Json_FromStream<T>
     {
-        private readonly T value;
-
-        private readonly MemoryStream memoryStream;
-
+        private T value;
+        private MemoryStream memoryStream;
         private DataContractJsonSerializer dataContractJsonSerializer;
         private Newtonsoft.Json.JsonSerializer newtonSoftJsonSerializer;
 
-        public Json_FromStream()
+        [GlobalSetup(Target = nameof(Jil_))]
+        public void SetupJil_()
         {
             value = DataGenerator.Generate<T>();
 
             // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
             memoryStream = new MemoryStream(capacity: short.MaxValue);
-
-            dataContractJsonSerializer = new DataContractJsonSerializer(typeof(T));
-            newtonSoftJsonSerializer = new Newtonsoft.Json.JsonSerializer();
-        }
-
-        [GlobalSetup(Target = nameof(Jil_))]
-        public void SetupJil_()
-        {
             memoryStream.Position = 0;
 
             using (var writer = new StreamWriter(memoryStream, Encoding.UTF8, short.MaxValue, leaveOpen: true))
             {
                 Jil.JSON.Serialize<T>(value, writer, Jil.Options.ISO8601);
                 writer.Flush();
             }
+        }
+
+        [BenchmarkCategory(Categories.ThirdParty)]
+        [Benchmark(Description = "Jil")]
+        public T Jil_()
+        {
+            memoryStream.Position = 0;
 
-            Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
+            using (var reader = CreateNonClosingReaderWithDefaultSizes())
+                return Jil.JSON.Deserialize<T>(reader, Jil.Options.ISO8601);
         }
 
         [GlobalSetup(Target = nameof(JsonNet_))]
         public void SetupJsonNet_()
         {
+            value = DataGenerator.Generate<T>();
+
+            // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
             memoryStream.Position = 0;
 
+            newtonSoftJsonSerializer = new Newtonsoft.Json.JsonSerializer();
+
             using (var writer = new StreamWriter(memoryStream, Encoding.UTF8, short.MaxValue, leaveOpen: true))
             {
                 newtonSoftJsonSerializer.Serialize(writer, value);
                 writer.Flush();
             }
         }
 
-        [GlobalSetup(Target = nameof(Utf8Json_))]
-        public void SetupUtf8Json_()
-        {
-            memoryStream.Position = 0;
-            Utf8Json.JsonSerializer.Serialize<T>(memoryStream, value);
-        }
-
-        [GlobalSetup(Target = nameof(DataContractJsonSerializer_))]
-        public void SetupDataContractJsonSerializer_()
-        {
-            memoryStream.Position = 0;
-            dataContractJsonSerializer.WriteObject(memoryStream, value);
-        }
-
-        [BenchmarkCategory(Categories.ThirdParty)]
-        [Benchmark(Description = "Jil")]
-        public T Jil_()
-        {
-            memoryStream.Position = 0;
-
-            using (var reader = CreateNonClosingReaderWithDefaultSizes())
-                return Jil.JSON.Deserialize<T>(reader, Jil.Options.ISO8601);
-        }
-
         [BenchmarkCategory(Categories.Runtime, Categories.Libraries, Categories.ThirdParty)] // JSON.NET is so popular that despite being 3rd Party lib we run the benchmarks for CoreFX and CoreCLR CI
         [Benchmark(Description = "JSON.NET")]
         public T JsonNet_()
@@ -94,6 +75,17 @@ public T JsonNet_()
                 return (T)newtonSoftJsonSerializer.Deserialize(reader, typeof(T));
         }
 
+        [GlobalSetup(Target = nameof(Utf8Json_))]
+        public void SetupUtf8Json_()
+        {
+            value = DataGenerator.Generate<T>();
+
+            // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
+            memoryStream.Position = 0;
+            Utf8Json.JsonSerializer.Serialize<T>(memoryStream, value);
+        }
+
         [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Utf8Json")]
         public T Utf8Json_()
@@ -102,6 +94,18 @@ public T Utf8Json_()
             return Utf8Json.JsonSerializer.Deserialize<T>(memoryStream);
         }
 
+        [GlobalSetup(Target = nameof(DataContractJsonSerializer_))]
+        public void SetupDataContractJsonSerializer_()
+        {
+            value = DataGenerator.Generate<T>();
+
+            // the stream is pre-allocated, we don't want the benchmarks to include stream allocaton cost
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
+            memoryStream.Position = 0;
+            dataContractJsonSerializer = new DataContractJsonSerializer(typeof(T));
+            dataContractJsonSerializer.WriteObject(memoryStream, value);
+        }
+
         [BenchmarkCategory(Categories.Runtime, Categories.Libraries)]
         [Benchmark(Description = "DataContractJsonSerializer")]
         public T DataContractJsonSerializer_()
@@ -110,15 +114,15 @@ public T DataContractJsonSerializer_()
             return (T)dataContractJsonSerializer.ReadObject(memoryStream);
         }
 
+        [GlobalCleanup]
+        public void Cleanup() => memoryStream.Dispose();
+
         private StreamReader CreateNonClosingReaderWithDefaultSizes()
             => new StreamReader(
                 memoryStream, 
                 Encoding.UTF8, 
                 true, // default is true https://github.com/dotnet/corefx/blob/708e4537d8944199af7d580def0d97a030be98c7/src/Common/src/CoreLib/System/IO/StreamReader.cs#L98
                 1024, // default buffer size from CoreFX https://github.com/dotnet/corefx/blob/708e4537d8944199af7d580def0d97a030be98c7/src/Common/src/CoreLib/System/IO/StreamReader.cs#L27 
                 leaveOpen: true); // we want to reuse the same string in the benchmarks to make sure that cost of allocating stream is not included in the benchmarks
-
-        [GlobalCleanup]
-        public void Cleanup() => memoryStream.Dispose();
     }
 }

src/benchmarks/micro/Serializers/Json_FromString.cs

@@ -13,33 +13,25 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     public class Json_FromString<T>
     {
-        private readonly T value;
         private string serialized;
 
-        public Json_FromString() => value = DataGenerator.Generate<T>();
-
         [GlobalSetup(Target = nameof(Jil_))]
-        public void SerializeJil()
-        {
-            serialized = Jil.JSON.Serialize<T>(value, Jil.Options.ISO8601);
-
-            Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
-        }
-
-        [GlobalSetup(Target = nameof(JsonNet_))]
-        public void SerializeJsonNet() => serialized = Newtonsoft.Json.JsonConvert.SerializeObject(value);
-
-        [GlobalSetup(Target = nameof(Utf8Json_))]
-        public void SerializeUtf8Json_() => serialized = Utf8Json.JsonSerializer.ToJsonString(value);
+        public void SetupJil() => serialized = Jil.JSON.Serialize<T>(DataGenerator.Generate<T>(), Jil.Options.ISO8601);
 
         [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]
         public T Jil_() => Jil.JSON.Deserialize<T>(serialized, Jil.Options.ISO8601);
 
+        [GlobalSetup(Target = nameof(JsonNet_))]
+        public void SerializeJsonNet() => serialized = Newtonsoft.Json.JsonConvert.SerializeObject(DataGenerator.Generate<T>());
+
         [BenchmarkCategory(Categories.Runtime, Categories.Libraries, Categories.ThirdParty)]
         [Benchmark(Description = "JSON.NET")]
         public T JsonNet_() => Newtonsoft.Json.JsonConvert.DeserializeObject<T>(serialized);
 
+        [GlobalSetup(Target = nameof(Utf8Json_))]
+        public void SerializeUtf8Json_() => serialized = Utf8Json.JsonSerializer.ToJsonString(DataGenerator.Generate<T>());
+
         [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Utf8Json")]
         public T Utf8Json_() => Utf8Json.JsonSerializer.Deserialize<T>(serialized);

src/benchmarks/micro/Serializers/Json_ToStream.cs

@@ -16,15 +16,16 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     public class Json_ToStream<T>
     {
-        private readonly T value;
+        private T value;
 
-        private readonly MemoryStream memoryStream;
-        private readonly StreamWriter streamWriter;
+        private MemoryStream memoryStream;
+        private StreamWriter streamWriter;
 
         private DataContractJsonSerializer dataContractJsonSerializer;
         private Newtonsoft.Json.JsonSerializer newtonSoftJsonSerializer;
 
-        public Json_ToStream()
+        [GlobalSetup]
+        public void Setup()
         {
             value = DataGenerator.Generate<T>();
 
@@ -36,9 +37,6 @@ public Json_ToStream()
             newtonSoftJsonSerializer = new Newtonsoft.Json.JsonSerializer();
         }
 
-        [GlobalSetup(Target = nameof(Jil_))]
-        public void WarmupJil() => Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
-
         [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]
         public void Jil_()

src/benchmarks/micro/Serializers/Json_ToString.cs

@@ -13,12 +13,10 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(CollectionsOfPrimitives))]
     public class Json_ToString<T>
     {
-        private readonly T value;
+        private T value;
 
-        public Json_ToString() => value = DataGenerator.Generate<T>();
-
-        [GlobalSetup(Target = nameof(Jil_))]
-        public void WarmupJil() => Jil_(); // workaround for https://github.com/dotnet/BenchmarkDotNet/issues/837
+        [GlobalSetup]
+        public void Setup() => value = DataGenerator.Generate<T>();
 
         [BenchmarkCategory(Categories.ThirdParty)]
         [Benchmark(Description = "Jil")]

src/benchmarks/micro/Serializers/Xml_FromStream.cs

@@ -20,41 +20,39 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
     public class Xml_FromStream<T>
     {
-        private readonly T value;
-        private readonly XmlSerializer xmlSerializer;
-        private readonly DataContractSerializer dataContractSerializer;
-        private readonly MemoryStream memoryStream;
+        private T value;
+        private XmlSerializer xmlSerializer;
+        private DataContractSerializer dataContractSerializer;
+        private MemoryStream memoryStream;
 
-        public Xml_FromStream()
+        [GlobalSetup(Target = nameof(XmlSerializer_))]
+        public void SetupXmlSerializer()
         {
             value = DataGenerator.Generate<T>();
-            xmlSerializer = new XmlSerializer(typeof(T));
-            dataContractSerializer = new DataContractSerializer(typeof(T));
             memoryStream = new MemoryStream(capacity: short.MaxValue);
+            memoryStream.Position = 0;
+            xmlSerializer = new XmlSerializer(typeof(T));
+            xmlSerializer.Serialize(memoryStream, value);
         }
 
-        [GlobalSetup(Target = nameof(XmlSerializer_))]
-        public void SetupXmlSerializer()
+        [BenchmarkCategory(Categories.Libraries, Categories.Runtime)]
+        [Benchmark(Description = nameof(XmlSerializer))]
+        public T XmlSerializer_()
         {
             memoryStream.Position = 0;
-            xmlSerializer.Serialize(memoryStream, value);
+            return (T)xmlSerializer.Deserialize(memoryStream);
         }
 
         [GlobalSetup(Target = nameof(DataContractSerializer_))]
         public void SetupDataContractSerializer()
         {
+            value = DataGenerator.Generate<T>();
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
             memoryStream.Position = 0;
+            dataContractSerializer = new DataContractSerializer(typeof(T));
             dataContractSerializer.WriteObject(memoryStream, value);
         }
 
-        [BenchmarkCategory(Categories.Libraries, Categories.Runtime)]
-        [Benchmark(Description = nameof(XmlSerializer))]
-        public T XmlSerializer_()
-        {
-            memoryStream.Position = 0;
-            return (T)xmlSerializer.Deserialize(memoryStream);
-        }
-
         [BenchmarkCategory(Categories.Libraries)]
         [Benchmark(Description = nameof(DataContractSerializer))]
         public T DataContractSerializer_()

src/benchmarks/micro/Serializers/Xml_ToStream.cs

@@ -20,17 +20,18 @@ namespace MicroBenchmarks.Serializers
     [GenericTypeArguments(typeof(ClassImplementingIXmlSerialiable))]
     public class Xml_ToStream<T>
     {
-        private readonly T value;
-        private readonly XmlSerializer xmlSerializer;
-        private readonly DataContractSerializer dataContractSerializer;
-        private readonly MemoryStream memoryStream;
+        private T value;
+        private XmlSerializer xmlSerializer;
+        private DataContractSerializer dataContractSerializer;
+        private MemoryStream memoryStream;
 
-        public Xml_ToStream()
+        [GlobalSetup]
+        public void Setup()
         {
             value = DataGenerator.Generate<T>();
+            memoryStream = new MemoryStream(capacity: short.MaxValue);
             xmlSerializer = new XmlSerializer(typeof(T));
             dataContractSerializer = new DataContractSerializer(typeof(T));
-            memoryStream = new MemoryStream(capacity: short.MaxValue);
         }
 
         [BenchmarkCategory(Categories.Libraries, Categories.Runtime)]

src/benchmarks/micro/libraries/Microsoft.Extensions.Logging/EventSourceBenchmark.cs

@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System.Collections.Generic;
 using System.Diagnostics.Tracing;
 using BenchmarkDotNet.Attributes;
 using Microsoft.Extensions.DependencyInjection;
@@ -59,19 +58,13 @@ public void Setup()
         }
 
         [GlobalCleanup]
-        public void Cleanup()
-        {
-            _listener?.Dispose();
-        }
+        public void Cleanup() => _listener?.Dispose();
 
         private class TestEventListener : EventListener
         {
             private readonly EventKeywords _keywords;
 
-            public TestEventListener(EventKeywords keywords)
-            {
-                _keywords = keywords;
-            }
+            public TestEventListener(EventKeywords keywords) => _keywords = keywords;
 
             protected override void OnEventSourceCreated(System.Diagnostics.Tracing.EventSource eventSource)
             {

src/benchmarks/micro/libraries/Microsoft.Extensions.Logging/LoggingOverheadBenchmark.cs

@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using BenchmarkDotNet.Attributes;
 using Microsoft.Extensions.DependencyInjection;
 using MicroBenchmarks;