Logical group support, fixes #617

docs/guide/Advanced/Baseline.md

@@ -1,47 +1,117 @@
 # Baseline
 
-In order to scale your results, you need to mark one of your benchmark methods as a baseline. Only one method in class can have `Baseline = true` applied.
+In order to scale your results, you can mark a benchmark method or a job as a baseline.
+Let's learn this feature by examples.
 
-## Example
+## Example 1: Methods
+
+You can mark a method as a baseline with the help of `[Benchmark(Baseline = true)]`. 
 
 ```cs
 public class Sleeps
 {
     [Benchmark]
-    public void Time50()
-    {
-        Thread.Sleep(50);
-    }
+    public void Time50() => Thread.Sleep(50);
 
     [Benchmark(Baseline = true)]
-    public void Time100()
-    {
-        Thread.Sleep(100);
-    }
+    public void Time100() => Thread.Sleep(100);
 
     [Benchmark]
-    public void Time150()
-    {
-        Thread.Sleep(150);
-    }
+    public void Time150() => Thread.Sleep(150);
 }
 ```
 
-As a result, you will have additional column in the summary table:
+As a result, you will have additional `Scaled` column in the summary table:
 
 ```ini
-BenchmarkDotNet=v0.9.0.0
-OS=Microsoft Windows NT 6.2.9200.0
-Processor=Intel(R) Core(TM) i7-4810MQ CPU @ 2.80GHz, ProcessorCount=8
-Frequency=2728067 ticks, Resolution=366.5599 ns
-HostCLR=MS.NET 4.0.30319.42000, Arch=64-bit RELEASE [RyuJIT]
+BenchmarkDotNet=v0.10.12, OS=Windows 10 Redstone 3 [1709, Fall Creators Update] (10.0.16299.192)
+Processor=Intel Core i7-6700HQ CPU 2.60GHz (Skylake), ProcessorCount=8
+Frequency=2531249 Hz, Resolution=395.0619 ns, Timer=TSC
+.NET Core SDK=2.0.3
+  [Host]     : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT
+  DefaultJob : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT
+```
+
+|  Method |      Mean |     Error |    StdDev | Scaled |
+|-------- |----------:|----------:|----------:|-------:|
+|  Time50 |  50.46 ms | 0.0779 ms | 0.0729 ms |   0.50 |
+| Time100 | 100.39 ms | 0.0762 ms | 0.0713 ms |   1.00 |
+| Time150 | 150.48 ms | 0.0986 ms | 0.0922 ms |   1.50 |
+
+
+## Example 2: Methods with categories
 
-Type=Sleeps  Mode=Throughput
+The only way to have several baselines in the same class is to separate them by categories
+  and mark the class with `[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]`.
+
+```cs
+[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+[CategoriesColumn]
+public class Sleeps
+{
+    [BenchmarkCategory("Fast"), Benchmark(Baseline = true)]        
+    public void Time50() => Thread.Sleep(50);
+
+    [BenchmarkCategory("Fast"), Benchmark]
+    public void Time100() => Thread.Sleep(100);
+
+    [BenchmarkCategory("Slow"), Benchmark(Baseline = true)]        
+    public void Time550() => Thread.Sleep(550);
+
+    [BenchmarkCategory("Slow"), Benchmark]
+    public void Time600() => Thread.Sleep(600);
+}
+```
+
+```ini
+BenchmarkDotNet=v0.10.12, OS=Windows 10 Redstone 3 [1709, Fall Creators Update] (10.0.16299.192)
+Processor=Intel Core i7-6700HQ CPU 2.60GHz (Skylake), ProcessorCount=8
+Frequency=2531249 Hz, Resolution=395.0619 ns, Timer=TSC
+.NET Core SDK=2.0.3
+  [Host]     : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT
+  DefaultJob : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT
 ```
 
-  Method |      Median |    StdDev | Scaled
--------- |------------ |---------- |-------
- Time100 | 100.2640 ms | 0.1238 ms |   1.00
- Time150 | 150.2093 ms | 0.1034 ms |   1.50
-  Time50 |  50.2509 ms | 0.1153 ms |   0.50
+|  Method | Categories |      Mean |     Error |    StdDev | Scaled |
+|-------- |----------- |----------:|----------:|----------:|-------:|
+|  Time50 |       Fast |  50.46 ms | 0.0745 ms | 0.0697 ms |   1.00 |
+| Time100 |       Fast | 100.47 ms | 0.0955 ms | 0.0893 ms |   1.99 |
+|         |            |           |           |           |        |
+| Time550 |       Slow | 550.48 ms | 0.0525 ms | 0.0492 ms |   1.00 |
+| Time600 |       Slow | 600.45 ms | 0.0396 ms | 0.0331 ms |   1.09 |
+
+
+## Example 3: Jobs
+
+If you want to compare several runtime configuration,
+  you can mark one of your jobs with `isBaseline = true`.
+
+```cs
+[ClrJob(isBaseline: true)]
+[MonoJob]
+[CoreJob]
+public class RuntimeCompetition
+{
+    [Benchmark]
+    public int SplitJoin() => string.Join(",", new string[1000]).Split(',').Length;
+}
+```
+
+```ini
+BenchmarkDotNet=v0.10.12, OS=Windows 10 Redstone 3 [1709, Fall Creators Update] (10.0.16299.192)
+Processor=Intel Core i7-6700HQ CPU 2.60GHz (Skylake), ProcessorCount=8
+Frequency=2531249 Hz, Resolution=395.0619 ns, Timer=TSC
+.NET Core SDK=2.0.3
+  [Host]     : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT
+  Job-MXFYPZ : .NET Framework 4.7 (CLR 4.0.30319.42000), 64bit RyuJIT-v4.7.2600.0
+  Core       : .NET Core 2.0.3 (Framework 4.6.25815.02), 64bit RyuJIT
+  Mono       : Mono 5.4.0 (Visual Studio), 64bit 
+```
 
+    Method | Runtime |     Mean |     Error |    StdDev | Scaled | ScaledSD |
+---------- |-------- |---------:|----------:|----------:|-------:|---------:|
+ SplitJoin |     Clr | 19.42 us | 0.2447 us | 0.1910 us |   1.00 |     0.00 |
+ SplitJoin |    Core | 13.00 us | 0.2183 us | 0.1935 us |   0.67 |     0.01 |
+ SplitJoin |    Mono | 39.14 us | 0.7763 us | 1.3596 us |   2.02 |     0.07 |
+
+

samples/BenchmarkDotNet.Samples/Intro/IntroOrderManual.cs

@@ -29,15 +29,22 @@ private class FastestToSlowestOrderProvider : IOrderProvider
                 public IEnumerable<Benchmark> GetExecutionOrder(Benchmark[] benchmarks) =>
                     from benchmark in benchmarks
                     orderby benchmark.Parameters["X"] descending,
-                            benchmark.Target.MethodDisplayInfo
+                        benchmark.Target.MethodDisplayInfo
                     select benchmark;
 
                 public IEnumerable<Benchmark> GetSummaryOrder(Benchmark[] benchmarks, Summary summary) =>
                     from benchmark in benchmarks
                     orderby summary[benchmark].ResultStatistics.Mean
                     select benchmark;
 
-                public string GetGroupKey(Benchmark benchmark, Summary summary) => null;
+                public string GetHighlightGroupKey(Benchmark benchmark) => null;
+
+                public string GetLogicalGroupKey(IConfig config, Benchmark[] allBenchmarks, Benchmark benchmark) =>
+                    benchmark.Job.DisplayInfo + "_" + benchmark.Parameters.DisplayInfo;
+
+                public IEnumerable<string> GetLogicalGroupOrder(IEnumerable<string> logicalGroups) => logicalGroups.OrderBy(s => s);
+
+                public bool SeparateLogicalGroups => true;
             }
         }
 

src/BenchmarkDotNet.Core/Attributes/Columns/IsBaselineColumnAttribute.cs

@@ -0,0 +1,12 @@
+using System;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Mathematics;
+
+namespace BenchmarkDotNet.Attributes.Columns
+{
+    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)]
+    public class IsBaselineColumnAttribute : ColumnConfigBaseAttribute
+    {
+        public IsBaselineColumnAttribute() : base(IsBaselineColumn.Default) { }
+    }
+}

src/BenchmarkDotNet.Core/Attributes/Columns/LogicalGroupColumnAttribute.cs

@@ -0,0 +1,12 @@
+using System;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Mathematics;
+
+namespace BenchmarkDotNet.Attributes.Columns
+{
+    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)]
+    public class LogicalGroupColumnAttribute : ColumnConfigBaseAttribute
+    {
+        public LogicalGroupColumnAttribute() : base(LogicalGroupColumn.Default) { }
+    }
+}

src/BenchmarkDotNet.Core/Attributes/GroupBenchmarksByAttribute.cs

@@ -0,0 +1,23 @@
+using System;
+using BenchmarkDotNet.Configs;
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [PublicAPI]
+    public class GroupBenchmarksByAttribute: Attribute, IConfigSource
+    {
+        public IConfig Config { get; }
+
+        // CLS-Compliant Code requires a constuctor without an array in the argument list
+        protected GroupBenchmarksByAttribute()
+        {
+            Config = ManualConfig.CreateEmpty();
+        }
+
+        public GroupBenchmarksByAttribute(params BenchmarkLogicalGroupRule[] rules)
+        {
+            Config = ManualConfig.CreateEmpty().With(rules);
+        }
+    }
+}

src/BenchmarkDotNet.Core/Attributes/Jobs/ClrJobAttribute.cs

@@ -4,7 +4,7 @@ namespace BenchmarkDotNet.Attributes.Jobs
 {
     public class ClrJobAttribute : JobConfigBaseAttribute
     {
-        public ClrJobAttribute() : base(Job.Clr)
+        public ClrJobAttribute(bool isBaseline = false) : base(Job.Clr.WithIsBaseline(isBaseline))
         {
         }
     }

src/BenchmarkDotNet.Core/Attributes/Jobs/CoreJobAttribute.cs

@@ -4,7 +4,7 @@ namespace BenchmarkDotNet.Attributes.Jobs
 {
     public class CoreJobAttribute : JobConfigBaseAttribute
     {
-        public CoreJobAttribute() : base(Job.Core)
+        public CoreJobAttribute(bool isBaseline = false) : base(Job.Core.WithIsBaseline(isBaseline))
         {
         }
     }

src/BenchmarkDotNet.Core/Attributes/Jobs/MonoJobAttribute.cs

@@ -7,12 +7,12 @@ namespace BenchmarkDotNet.Attributes.Jobs
     [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)]
     public class MonoJobAttribute : JobConfigBaseAttribute
     {
-        public MonoJobAttribute() : base(Job.Mono)
+        public MonoJobAttribute(bool isBaseline = false) : base(Job.Mono.WithIsBaseline(isBaseline))
         {
         }
 
-        public MonoJobAttribute(string name, string path) 
-            : base(new Job(name, new EnvMode(new MonoRuntime(name, path)).Freeze()).Freeze())
+        public MonoJobAttribute(string name, string path, bool isBaseline = false) 
+            : base(new Job(name, new EnvMode(new MonoRuntime(name, path)).Freeze()).WithIsBaseline(isBaseline).Freeze())
         {
         }
     }

src/BenchmarkDotNet.Core/Attributes/Jobs/SimpleJobAttribute.cs

@@ -17,10 +17,9 @@ public SimpleJobAttribute(
             int warmupCount = DefaultValue,
             int targetCount = DefaultValue,
             int invocationCount = DefaultValue,
-            string id = null
-        ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, null))
-        {
-        }
+            string id = null,
+            bool isBaseline = false
+        ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, null, isBaseline)) { }
 
         [PublicAPI]
         public SimpleJobAttribute(
@@ -29,12 +28,12 @@ public SimpleJobAttribute(
             int warmupCount = DefaultValue,
             int targetCount = DefaultValue,
             int invocationCount = DefaultValue,
-            string id = null
-        ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, runStrategy))
-        {
-        }
+            string id = null,
+            bool isBaseline = false
+        ) : base(CreateJob(id, launchCount, warmupCount, targetCount, invocationCount, runStrategy, isBaseline)) { }
 
-        private static Job CreateJob(string id, int launchCount, int warmupCount, int targetCount, int invocationCount, RunStrategy? runStrategy)
+        private static Job CreateJob(string id, int launchCount, int warmupCount, int targetCount, int invocationCount, RunStrategy? runStrategy,
+            bool isBaseline)
         {
             var job = new Job(id);
             if (launchCount != DefaultValue)
@@ -50,8 +49,11 @@ private static Job CreateJob(string id, int launchCount, int warmupCount, int ta
                 if (invocationCount % unrollFactor != 0)
                     job.Run.UnrollFactor = 1;
             }
+
             if (runStrategy != null)
                 job.Run.RunStrategy = runStrategy.Value;
+            if (isBaseline)
+                job.Meta.IsBaseline = true;
 
             return job.Freeze();
         }

src/BenchmarkDotNet.Core/BenchmarkDotNet.Core.csproj

@@ -20,10 +20,10 @@
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
     <ProjectReference Include="..\BenchmarkDotNet.Disassembler.x64\BenchmarkDotNet.Disassembler.x64.csproj">
-        <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
     </ProjectReference>
     <ProjectReference Include="..\BenchmarkDotNet.Disassembler.x86\BenchmarkDotNet.Disassembler.x86.csproj">
-        <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
     </ProjectReference>
     <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.3.0" />
     <Reference Include="System.Management" />

src/BenchmarkDotNet.Core/Columns/BaselineScaledColumn.cs

@@ -49,10 +49,10 @@ public string ColumnName
 
         public string GetValue(Summary summary, Benchmark benchmark)
         {
-            var baseline = summary.Benchmarks.
-                Where(b => b.Job.DisplayInfo == benchmark.Job.DisplayInfo).
-                Where(b => b.Parameters.DisplayInfo == benchmark.Parameters.DisplayInfo).
-                FirstOrDefault(b => b.Target.Baseline);
+            string logicalGroupKey = summary.GetLogicalGroupKey(benchmark);
+            var baseline = summary.Benchmarks
+                .Where(b => summary.GetLogicalGroupKey(b) == logicalGroupKey)
+                .FirstOrDefault(b => b.IsBaseline());
             bool invalidResults = baseline == null ||
                                  summary[baseline] == null ||
                                  summary[baseline].ResultStatistics == null ||
@@ -66,8 +66,8 @@ public string GetValue(Summary summary, Benchmark benchmark)
             var baselineStat = summary[baseline].ResultStatistics;
             var targetStat = summary[benchmark].ResultStatistics;
 
-            double mean = benchmark.Target.Baseline ? 1 : Statistics.DivMean(targetStat, baselineStat);
-            double stdDev = benchmark.Target.Baseline ? 0 : Math.Sqrt(Statistics.DivVariance(targetStat, baselineStat));
+            double mean = benchmark.IsBaseline() ? 1 : Statistics.DivMean(targetStat, baselineStat);
+            double stdDev = benchmark.IsBaseline() ? 0 : Math.Sqrt(Statistics.DivVariance(targetStat, baselineStat));
 
             switch (Kind)
             {
@@ -87,17 +87,17 @@ public string GetValue(Summary summary, Benchmark benchmark)
             }
         }
 
-        public bool IsNonBaselinesPrecise(Summary summary, Statistics baselineStat, Benchmark benchmark)
-        {
-            var nonBaselines = summary.Benchmarks.
-                        Where(b => b.Job.DisplayInfo == benchmark.Job.DisplayInfo).
-                        Where(b => b.Parameters.DisplayInfo == benchmark.Parameters.DisplayInfo).
-                        Where(b => !b.Target.Baseline);
-
-            return nonBaselines.Any(x => Statistics.DivMean(summary[x].ResultStatistics, baselineStat) < 0.01);
+        public bool IsNonBaselinesPrecise(Summary summary, Statistics baselineStat, Benchmark benchmark)
+        {
+            string logicalGroupKey = summary.GetLogicalGroupKey(benchmark);
+            var nonBaselines = summary.Benchmarks
+                        .Where(b => summary.GetLogicalGroupKey(b) == logicalGroupKey)
+                        .Where(b => !b.IsBaseline());
+
+            return nonBaselines.Any(x => Statistics.DivMean(summary[x].ResultStatistics, baselineStat) < 0.01);
         }
 
-        public bool IsAvailable(Summary summary) => summary.Benchmarks.Any(b => b.Target.Baseline);
+        public bool IsAvailable(Summary summary) => summary.Benchmarks.Any(b => b.IsBaseline());
         public bool AlwaysShow => true;
         public ColumnCategory Category => ColumnCategory.Baseline;
         public int PriorityInCategory => (int) Kind;

src/BenchmarkDotNet.Core/Columns/ColumnCategory.cs

@@ -2,6 +2,6 @@
 {
     public enum ColumnCategory
     {
-        Job, Params, Statistics, Baseline, Custom, Diagnoser
+        Job, Params, Statistics, Baseline, Custom, Meta, Diagnoser
     }
 }

src/BenchmarkDotNet.Core/Columns/DefaultColumnProvider.cs

@@ -47,7 +47,7 @@ public IEnumerable<IColumn> GetColumns(Summary summary)
                 if (NeedToShow(summary, s => s.StandardDeviation > 1e-9))
                     yield return StatisticColumn.StdDev;
 
-                if (summary.Reports != null && summary.Benchmarks.Any(b => b.Target.Baseline))
+                if (summary.Reports != null && summary.Benchmarks.Any(b => b.IsBaseline()))
                 {
                     yield return BaselineScaledColumn.Scaled;
                     var stdDevColumn = BaselineScaledColumn.ScaledStdDev;