From 93e0302623cf400b746a28571c8a316daa6ff56e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Sun, 27 May 2018 19:50:18 +0200 Subject: [PATCH] warn the users (once!) that if they run less than 15 iterations, the MultimodalDistributionAnalyzer is not going to work, #763 --- src/BenchmarkDotNet/Analysers/Conclusion.cs | 34 +++++++++++++++++-- .../MultimodalDistributionAnalyzer.cs | 16 ++++++--- src/BenchmarkDotNet/Jobs/RunMode.cs | 9 +++-- .../Running/BenchmarkRunner.cs | 2 +- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/BenchmarkDotNet/Analysers/Conclusion.cs b/src/BenchmarkDotNet/Analysers/Conclusion.cs index aa1d7e00c1..b8e76eda91 100644 --- a/src/BenchmarkDotNet/Analysers/Conclusion.cs +++ b/src/BenchmarkDotNet/Analysers/Conclusion.cs @@ -1,10 +1,11 @@ -using BenchmarkDotNet.Reports; +using System; +using BenchmarkDotNet.Reports; using JetBrains.Annotations; namespace BenchmarkDotNet.Analysers { // TODO: Find a better name - public sealed class Conclusion + public sealed class Conclusion : IEquatable { [NotNull] public string AnalyserId { get; } @@ -33,5 +34,34 @@ public static Conclusion CreateWarning(string analyserId, string message, [CanBe public static Conclusion CreateError(string analyserId, string message, [CanBeNull] BenchmarkReport report = null) => new Conclusion(analyserId, ConclusionKind.Error, message, report); + + public bool Equals(Conclusion other) + { + if (ReferenceEquals(null, other)) + return false; + if (ReferenceEquals(this, other)) + return true; + return string.Equals(AnalyserId, other.AnalyserId) && Kind == other.Kind && string.Equals(Message, other.Message); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) + return false; + if (ReferenceEquals(this, obj)) + return true; + return obj is Conclusion && Equals((Conclusion) obj); + } + + public override int GetHashCode() + { + unchecked + { + int hashCode = AnalyserId.GetHashCode(); + hashCode = (hashCode * 397) ^ (int) Kind; + hashCode = (hashCode * 397) ^ Message.GetHashCode(); + return hashCode; + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs b/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs index d2e1b57b76..7d76a95b81 100644 --- a/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs +++ b/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; using JetBrains.Annotations; @@ -7,20 +8,21 @@ namespace BenchmarkDotNet.Analysers { public class MultimodalDistributionAnalyzer : AnalyserBase { - public override string Id => "MultimodalDistribution"; public static readonly IAnalyser Default = new MultimodalDistributionAnalyzer(); private MultimodalDistributionAnalyzer() { } - [NotNull] - private Conclusion Create([NotNull] string kind, double mValue, [CanBeNull] BenchmarkReport report) - => CreateWarning($"It seems that the distribution {kind} (mValue = {mValue})", report); + public override string Id => "MultimodalDistribution"; public override IEnumerable AnalyseReport(BenchmarkReport report, Summary summary) { var statistics = report.ResultStatistics; - if (statistics == null || statistics.N < 15) + if (statistics == null) yield break; + if (statistics.N < EngineResolver.DefaultMinTargetIterationCount) + yield return CreateHint($"The number of iterations was set to less than {EngineResolver.DefaultMinTargetIterationCount}, " + + $"{nameof(MultimodalDistributionAnalyzer)} needs at least {EngineResolver.DefaultMinTargetIterationCount} iterations to work."); + double mValue = MathHelper.CalculateMValue(statistics); if (mValue > 4.2) yield return Create("is multimodal", mValue, report); @@ -29,5 +31,9 @@ public override IEnumerable AnalyseReport(BenchmarkReport report, Su else if (mValue > 2.8) yield return Create("can have several modes", mValue, report); } + + [NotNull] + private Conclusion Create([NotNull] string kind, double mValue, [CanBeNull] BenchmarkReport report) + => CreateWarning($"It seems that the distribution {kind} (mValue = {mValue})", report); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/RunMode.cs b/src/BenchmarkDotNet/Jobs/RunMode.cs index 56c8412635..9aa2024a89 100644 --- a/src/BenchmarkDotNet/Jobs/RunMode.cs +++ b/src/BenchmarkDotNet/Jobs/RunMode.cs @@ -1,4 +1,5 @@ -using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Horology; @@ -136,7 +137,8 @@ public int UnrollFactor /// /// Minimum count of target iterations that should be performed - /// The default is 15 + /// The default value is 15 + /// If you set this value to below 15, then is not going to work /// public int MinTargetIterationCount { @@ -146,7 +148,8 @@ public int MinTargetIterationCount /// /// Maximum count of target iterations that should be performed - /// The default is 100 + /// The default value is 100 + /// If you set this value to below 15, then is not going to work /// public int MaxTargetIterationCount { diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunner.cs b/src/BenchmarkDotNet/Running/BenchmarkRunner.cs index 0d3af90123..f7389de29f 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunner.cs @@ -208,7 +208,7 @@ private static Summary Run( MarkdownExporter.Console.ExportToLog(summary, logger); // TODO: make exporter - ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).ToList()); + ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).Distinct().ToList()); // TODO: move to conclusions var columnWithLegends = summary.Table.Columns.Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList();