diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux-Container.csv b/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux-Container.csv new file mode 100644 index 00000000000..2d5bfc90e25 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux-Container.csv @@ -0,0 +1,43 @@ +Legend,CommandLine +FortunesAspNet-intel-lin-4cores-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-4cores-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-4cores-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1core-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-1core-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1core-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-Unlimited-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --load.options.reuseBuild true +FortunesAspNet-intel-lin-Unlimited-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-Unlimited-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-16000m-16gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-16000m-16gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-16000m-16gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-4000m-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-4000m-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-4000m-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1000m-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-1000m-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1000m-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-500m-256mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-500m-256mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-500m-256mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4cores-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-4cores-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4cores-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1core-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-1core-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1core-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-Unlimited-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --load.options.reuseBuild true +JsonAspNet-intel-lin-Unlimited-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-Unlimited-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-16000m-16gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-16000m-16gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-16000m-16gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4000m-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-4000m-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4000m-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1000m-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-1000m-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1000m-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-500m-256mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-500m-256mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-500m-256mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs index 5b0e14e902e..89e46b10813 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs @@ -17,7 +17,7 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration // Environment Variables. // Add the environment variables from the configuration. Dictionary environmentVariables = new(); - foreach (var env in configuration.Environment.environment_variables) + foreach (var env in configuration.Environment!.environment_variables) { environmentVariables[env.Key] = env.Value; } @@ -41,8 +41,7 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration { string fileNameOfLog = Path.GetFileName(env.Value); commandStringBuilder.Append( $" --application.options.downloadFiles \"*{fileNameOfLog}.log\" " ); - string fileName = Path.GetFileNameWithoutExtension(env.Value); - commandStringBuilder.Append( $" --application.options.downloadFilesOutput \"{Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_GCLog")}\" " ); + commandStringBuilder.Append( $" --application.options.downloadFilesOutput \"{Path.Combine(configuration.Output!.Path, run.Key, $"{benchmarkNameToCommand.Key}_GCLog")}\" " ); } commandStringBuilder.Append($" --application.environmentVariables {env.Key}={variable} "); @@ -84,12 +83,6 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration commandStringBuilder.Append($" --application.options.traceOutput {Path.Combine(configuration.Output.Path, run.Key, (benchmarkNameToCommand.Key + "." + collectType)) + traceFileSuffix}"); } - // Add any additional arguments specified. - if (!string.IsNullOrEmpty(configuration.benchmark_settings.additional_arguments)) - { - commandStringBuilder.Append($" {configuration.benchmark_settings.additional_arguments} "); - } - string frameworkVersion = configuration.Environment.framework_version; // Override the framework version if it's specified at the level of the run. if (!string.IsNullOrEmpty(run.Value.framework_version)) @@ -120,6 +113,12 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration // Add the extra metrics by including the configuration. commandStringBuilder.Append($" --config {Path.Combine("Commands", "RunCommand", "BaseSuite", "PercentileBasedMetricsConfiguration.yml")} "); + // Add any additional arguments specified. + if (!string.IsNullOrEmpty(configuration.benchmark_settings.additional_arguments)) + { + commandStringBuilder.Append($" {configuration.benchmark_settings.additional_arguments} "); + } + string commandString = commandStringBuilder.ToString(); // Apply overrides. diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs index 989c251d05a..35a1d3eb45c 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs @@ -3,9 +3,9 @@ public sealed class ASPNetBenchmarksConfiguration : ConfigurationBase { public Dictionary? Runs { get; set; } - public Environment? Environment { get; set; } - public BenchmarkSettings? benchmark_settings { get; set; } - public Output? Output { get; set; } + public Environment Environment { get; set; } + public BenchmarkSettings benchmark_settings { get; set; } + public Output Output { get; set; } } public sealed class Run : RunBase @@ -35,10 +35,7 @@ public static class ASPNetBenchmarksConfigurationParser public static ASPNetBenchmarksConfiguration Parse(string path) { // Preconditions. - if (string.IsNullOrEmpty(path) || !File.Exists(path)) - { - throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(path)} is null/empty or doesn't exist. You must specify a valid path."); - } + ConfigurationChecker.VerifyFile(path, nameof(ASPNetBenchmarksConfigurationParser)); string serializedConfiguration = File.ReadAllText(path); @@ -59,6 +56,22 @@ public static ASPNetBenchmarksConfiguration Parse(string path) throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration)} is null. Check the syntax of the configuration."); } + // Checks if mandatory arguments are specified in the configuration. + if (configuration.Output == null) + { + throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration.Output)} is null. Check the syntax of the configuration."); + } + + if (string.IsNullOrEmpty(configuration.Output.Path)) + { + throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration.Output.Path)} is null or empty. Please specify an output path."); + } + + if (configuration.Environment == null) + { + throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration.Environment)} is null. Please add the environment item in the configuration."); + } + // Check for any COMPlus_ environment variables. if (configuration.Environment != null) { diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 3ace859eba2..0c6afe62e3b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -1,12 +1,12 @@ using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.CommandBuilders; -using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.ASPNetBenchmarks; using Spectre.Console; using Spectre.Console.Cli; using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Net.NetworkInformation; using System.Text; using System.Text.RegularExpressions; @@ -41,30 +41,47 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe AnsiConsole.Write(new Rule("ASP.NET Benchmarks Orchestrator")); AnsiConsole.WriteLine(); - ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(AspNetBenchmarksCommand)); ASPNetBenchmarksConfiguration configuration = ASPNetBenchmarksConfigurationParser.Parse(settings.ConfigurationPath); // Before running the ASP.NET benchmarks, execute a few checks: - // Check 1. If you are connected to corpnet. These tests only run if you are connected to corp-net and therefore, preemptively, check this. + // Check 1. If the ASP.NET Machines are pingable. // Check 2. The host machines reboot between 12:00 AM - 12:08 AM PST. Check if we are running during that time, and if so, delay running the infrastructure. // Check 1. - if (!Directory.Exists(@"\\clrmain\tools\")) + const string machineName = "asp-citrine-win"; + bool success = false; + Ping ping = new Ping(); + try { - // Not connected to corpnet if that folder is unavailable. - AnsiConsole.MarkupLine($"[red bold] Not connected to corpnet. Ensure you are connected before proceeding with the ASP.NET Benchmarks [/]"); + PingReply reply = ping.Send(machineName); + + if (reply.Status == IPStatus.Success) + { + success = true; + } + } + + catch (PingException exp) + { + // DO NOTHING but catch. We log this later. + } + + if (!success) + { + AnsiConsole.MarkupLine($"[red bold]Cannot ping the ASP.NET Machines. Ensure you are connected to corpnet or check if the machines are down before proceeding to run with the ASP.NET Benchmarks [/]"); return -1; } // Check 2. - SleepUntilHostsHaveRestarted(); + // We don't care about the output from the following method here since this is at a point before the tests have even started to run. + bool _ = TrySleepUntilHostsHaveRestarted(); RunASPNetBenchmarks(configuration); - AnsiConsole.MarkupLine($"[bold green] Report generated at: {configuration.Output.Path} [/]"); + AnsiConsole.MarkupLine($"[bold green] Report generated at: {configuration.Output!.Path} [/]"); return 0; } - private static void SleepUntilHostsHaveRestarted() + private static bool TrySleepUntilHostsHaveRestarted() { DateTime now = DateTime.UtcNow; @@ -74,25 +91,147 @@ private static void SleepUntilHostsHaveRestarted() // Check if the current time is between 12:00 AM and 12:09 AM. DateTime start = pstNow.Date; // 12:00 AM today. - DateTime end = start.AddMinutes(9); // 12:09 AM today. + DateTime end = start.AddMinutes(10); // 12:10 AM today. if (pstNow >= start && pstNow < end) { TimeSpan timeUntilEnd = end - pstNow; int secondsLeft = (int)timeUntilEnd.TotalSeconds; - // If we are between 12:00 AM and 12:09 AM PST, sleep for + // If we are between 12:00 AM and 12:09 AM PST, sleep for the seconds left. AnsiConsole.MarkupLine($"[yellow bold] ({DateTime.Now}) ASP.NET Benchmarks Sleeping for {secondsLeft} seconds since the host machines are rebooting. [/]"); Thread.Sleep((secondsLeft) * 1000); + return true; + } + + return false; + } + + + public static AspNetBenchmarkResults RunASPNetBenchmarks(ASPNetBenchmarksConfiguration configuration) + { + List<(string run, string benchmark, string reason)> retryMessages = new(); + Dictionary executionDetails = new(); + Core.Utilities.TryCreateDirectory(configuration.Output!.Path); + + // Parse the CSV file for the information. + string[] lines = File.ReadAllLines(configuration.benchmark_settings!.benchmark_file!); + Dictionary benchmarkNameToCommand = new(StringComparer.OrdinalIgnoreCase); + + for (int lineIdx = 0; lineIdx < lines.Length; lineIdx++) + { + if (lineIdx == 0) + { + continue; + } + + string[] line = lines[lineIdx].Split(',', StringSplitOptions.TrimEntries); + Debug.Assert(line.Length == 2); + + string benchmarkName = line[0]; + string benchmarkCommands = line[1]; + + benchmarkNameToCommand[benchmarkName] = benchmarkCommands; + } + + List> benchmarkToNameCommandAsKvpList = new(); + bool noBenchmarkFilters = + (configuration.benchmark_settings.benchmarkFilters == null || configuration.benchmark_settings.benchmarkFilters.Count == 0); + + // If the user has specified benchmark filters, retrieve them in that order. + if (!noBenchmarkFilters) + { + foreach (var filter in configuration.benchmark_settings.benchmarkFilters!) + { + foreach (var kvp in benchmarkNameToCommand) + { + // Check if we simply end with a "*", if so, match. + if (filter.EndsWith("*") && kvp.Key.StartsWith(filter.Replace("*", ""))) + { + benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } + + // Regular Regex check. + else if (Regex.IsMatch(kvp.Key, $"^{filter}$")) + { + benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } + } + } + + if (benchmarkToNameCommandAsKvpList.Count == 0) + { + throw new ArgumentException($"{nameof(AspNetBenchmarksCommand)}: No benchmark filters found. Please ensure you have added the wildcard character to do the regex matching. Benchmark Filter: {configuration.benchmark_settings.benchmarkFilters}"); + } + } + + // Else, add all the benchmarks. + else + { + foreach (var kvp in benchmarkNameToCommand) + { + benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } } + + // Overwrite the dictionary with the most up to date results of what will be run. + benchmarkNameToCommand = benchmarkToNameCommandAsKvpList.ToDictionary(pair => pair.Key, pair => pair.Value); + + // For each benchmark, iterate over all specified runs. + foreach (var benchmarkToCommand in benchmarkToNameCommandAsKvpList) + { + ExecuteBenchmarkForRuns(configuration, benchmarkToCommand, executionDetails, retryMessages); + } + + Dictionary> results = AspNetBenchmarksAnalyzeCommand.ExecuteAnalysis(configuration, benchmarkNameToCommand, executionDetails, retryMessages); + return new AspNetBenchmarkResults(executionDetails, results); } - private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksConfiguration configuration, KeyValuePair run, KeyValuePair benchmarkToCommand) + internal static void ExecuteBenchmarkForRuns(ASPNetBenchmarksConfiguration configuration, + KeyValuePair benchmarkToCommand, + Dictionary executionDetails, + List<(string, string, string)> retryMessages) { - // At the start of a run, if we are at a point in time where we are between the time where we deterministically know the host machines need to restart, - // sleep for the remaining time until the machines are back up. - SleepUntilHostsHaveRestarted(); + foreach (var run in configuration.Runs!) + { + const string NON_RESPONSIVE = @"for 'application' is invalid or not responsive: ""No such host is known"; + const string TIME_OUT = "[Time Out]"; + // If the machines fall asleep, re-run all the runs for the specific benchmark. + if (TrySleepUntilHostsHaveRestarted()) + { + ExecuteBenchmarkForRuns(configuration, benchmarkToCommand, executionDetails, retryMessages); + return; // Return here to prevent infinite looping. + } + + ProcessExecutionDetails result = ExecuteBenchmarkForRun(configuration, run, benchmarkToCommand); + string key = GetKey(benchmarkToCommand.Key, run.Key); + + bool timeout = result.StandardError.Contains(TIME_OUT); + bool nonResponsive = result.StandardOut.Contains(NON_RESPONSIVE) || result.StandardError.Contains(NON_RESPONSIVE); + bool timeoutOrNonResponsive = timeout || nonResponsive; + + // Wait 2 minutes and then retry if the run timed out or the host was non-responsive (post corp-net connection and check). + if (result.HasFailed && timeoutOrNonResponsive) + { + string retryReason = timeout ? "the run timed out" : "the server was non-responsive"; + string retryDetails = $"{run.Key} for {benchmarkToCommand.Key} failed as {retryReason}. Sleeping for 2 minutes and retrying"; + AnsiConsole.MarkupLine($"[red bold] {Markup.Escape(retryDetails)} [/]"); + retryMessages.Add((run.Key, benchmarkToCommand.Key, retryReason)); + Thread.Sleep(60 * 2 * 1000); + result = ExecuteBenchmarkForRun(configuration, run, benchmarkToCommand); + executionDetails[key] = result; + } + + else + { + executionDetails[key] = result; + } + } + } + + internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksConfiguration configuration, KeyValuePair run, KeyValuePair benchmarkToCommand) + { OS os = !benchmarkToCommand.Key.Contains("Win") ? OS.Linux : OS.Windows; (string, string) commandLine = ASPNetBenchmarksCommandBuilder.Build(configuration, run, benchmarkToCommand, os); @@ -105,13 +244,13 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo // There are 3 main ASP.NET errors: // 1. The server is unavailable - this could be because you aren't connected to CorpNet or the machine is down. // 2. The crank commands are incorrect. - // 3. Test fails because of a test error. + // 3. Test fails because of a test errors. // Launch new crank process. int exitCode = -1; string logfileOutput = Path.Combine(outputPath, $"{GetKey(benchmarkToCommand.Key, run.Key)}.log"); StringBuilder output = new(); - StringBuilder error = new(); + StringBuilder errors = new(); using (Process crankProcess = new()) { @@ -124,13 +263,16 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo AnsiConsole.MarkupLine($"[green bold] ({DateTime.Now}) Running ASP.NET Benchmark for Configuration {configuration.Name} {run.Key} {benchmarkToCommand.Key} [/]"); + // Add the command line to the top of the output file. + output.AppendLine($"Command: {commandLine.Item1} {commandLine.Item2}"); + crankProcess.OutputDataReceived += (s, d) => { output.AppendLine(d?.Data); }; crankProcess.ErrorDataReceived += (s, d) => { - error.AppendLine(d?.Data); + errors.AppendLine(d?.Data); }; crankProcess.Start(); @@ -143,11 +285,12 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo if (!crankProcess.HasExited) { AnsiConsole.MarkupLine($"[red bold] ASP.NET Benchmark timed out for: {configuration.Name} {run.Key} {benchmarkToCommand.Key} - skipping the results but writing stdout and stderror to {logfileOutput} [/]"); - File.WriteAllText(logfileOutput, "Output: \n" + output.ToString() + "\n Errors: \n" + error.ToString()); + errors.AppendLine($"Run: {configuration.Name} {run.Key} {benchmarkToCommand.Key} timed out."); + File.WriteAllText(logfileOutput, "Output: \n" + output.ToString() + "\n Errors: \n" + errors.ToString()); return new ProcessExecutionDetails(key: GetKey(benchmarkToCommand.Key, run.Key), commandlineArgs: commandLine.Item1 + " " + commandLine.Item2, environmentVariables: new(), - standardError: "[Time Out]: " + error.ToString(), + standardError: "[Time Out]: " + errors.ToString(), standardOut: output.ToString(), exitCode: exitCode); } @@ -155,12 +298,12 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo exitCode = crankProcess.ExitCode; } - string outputFile = Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkToCommand.Key}_{run.Key}.json"); + string outputJson = Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkToCommand.Key}_{run.Key}.json"); string outputDetails = output.ToString(); - if (File.Exists(outputFile)) + if (File.Exists(outputJson)) { - string[] outputLines = File.ReadAllLines(outputFile); + string[] outputLines = File.ReadAllLines(outputJson); // In a quick and dirty way, check the returnCode from the file that'll tell us if the test failed. foreach (var o in outputLines) @@ -177,10 +320,10 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo else { - // For the case where the output file doesn't exist implies that was an issue connecting to the asp.net machines or error number 1. - // This case also applies for incorrect crank arguments or error number 2. - // Move the standard out to the standard error as the process failed. - error.AppendLine(outputDetails); + // For the case where the output file doesn't exist implies that was an issue connecting to the asp.net machines or errors number 1. + // This case also applies for incorrect crank arguments or errors number 2. + // Move the standard out to the standard errors as the process failed. + errors.AppendLine(outputDetails); } if (exitCode != 0) @@ -188,127 +331,110 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo string[] outputLines = outputDetails.Split("\n"); foreach (var o in outputLines) { - // Crank provides the standard error from the test itself by this mechanism. + // Crank provides the standard errors from the test itself by this mechanism. // Error #3: Issues with test run. if (o.StartsWith("[STDERR]")) { - error.AppendLine(o.Replace("[STDERR]", "")); + errors.AppendLine(o.Replace("[STDERR]", "")); + } + + // Highlight case where: Configuration 'https://github.com/aspnet/Benchmarks/blob/main/scenarios/aspnet.profiles.yml?raw=true' could not be loaded. + else if (o.Contains("Configuration '") && o.Contains("could not be loaded")) + { + errors.AppendLine(o); } } - AnsiConsole.Markup($"[red bold] Failed with the following errors:\n {Markup.Escape(error.ToString())} Check the log file for more information: {logfileOutput} \n[/]"); + AnsiConsole.Markup($"[red bold] Failed with the following errors:\n {Markup.Escape(errors.ToString())}. Check the log file for more information: {logfileOutput} \n[/]"); } - File.WriteAllText(logfileOutput, "Output: \n" + outputDetails + "\n Errors: \n" + error.ToString()); + // Check to see if we got back all the files regardless of the exit code. + HashSet missingOutputs = CheckForMissingOutputsAndReturnNames(configuration, run.Key, benchmarkToCommand.Key); + + // If traces are missing, consider the run failed so that users can re-run it via the Failed yaml. + if (missingOutputs.Contains("Traces")) + { + exitCode = -1; + } + + File.WriteAllText(logfileOutput, "Output: \n" + outputDetails + "\n Errors: \n" + errors.ToString()); return new ProcessExecutionDetails(key: GetKey(benchmarkToCommand.Key, run.Key), commandlineArgs: commandLine.Item1 + " " + commandLine.Item2, environmentVariables: new(), - standardError: error.ToString(), + standardError: errors.ToString(), standardOut: output.ToString(), exitCode: exitCode); } - public static AspNetBenchmarkResults RunASPNetBenchmarks(ASPNetBenchmarksConfiguration configuration) + internal static HashSet CheckForMissingOutputsAndReturnNames(ASPNetBenchmarksConfiguration configuration, string runName, string benchmarkName) { - List<(string run, string benchmark, string reason)> retryMessages = new(); - Dictionary executionDetails = new(); - Core.Utilities.TryCreateDirectory(configuration.Output.Path); - - // Parse the CSV file for the information. - string[] lines = File.ReadAllLines(configuration.benchmark_settings.benchmark_file); - Dictionary benchmarkNameToCommand = new(StringComparer.OrdinalIgnoreCase); + HashSet missingOutputs = new HashSet(StringComparer.OrdinalIgnoreCase); + string basePath = Path.Combine(configuration.Output!.Path, runName); - for (int lineIdx = 0; lineIdx < lines.Length; lineIdx++) + // Files to expect always from crank: + // 1. Build Output: BenchmarkName_RunName.build.log + string buildOutput = Path.Combine(basePath, $"{benchmarkName}_{runName}.build.log"); + if (!File.Exists(buildOutput)) { - if (lineIdx == 0) - { - continue; - } - - string[] line = lines[lineIdx].Split(',', StringSplitOptions.TrimEntries); - Debug.Assert(line.Length == 2); - - string benchmarkName = line[0]; - string benchmarkCommands = line[1]; - - benchmarkNameToCommand[benchmarkName] = benchmarkCommands; + missingOutputs.Add("Build Output"); } - List> benchmarkToNameCommandAsKvpList = new(); - bool noBenchmarkFilters = - (configuration.benchmark_settings.benchmarkFilters == null || configuration.benchmark_settings.benchmarkFilters.Count == 0); - - // If the user has specified benchmark filters, retrieve them in that order. - if (!noBenchmarkFilters) + // 2. Application Output: BenchmarkName_RunName.output.log + string applicationOutput = Path.Combine(basePath, $"{benchmarkName}_{runName}.output.log"); + if (!File.Exists(applicationOutput)) { - foreach (var filter in configuration.benchmark_settings.benchmarkFilters!) - { - foreach (var kvp in benchmarkNameToCommand) - { - // Check if we simply end with a "*", if so, match. - if (filter.EndsWith("*") && kvp.Key.StartsWith(filter.Replace("*", ""))) - { - benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); - } - - // Regular Regex check. - else if (Regex.IsMatch(kvp.Key, $"^{filter}$")) - { - benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); - } - } - } + missingOutputs.Add("Application Output"); + } - if (benchmarkToNameCommandAsKvpList.Count == 0) - { - throw new ArgumentException($"{nameof(AspNetBenchmarksCommand)}: No benchmark filters found. Please ensure you have added the wildcard character to do the regex matching. Benchmark Filter: {configuration.benchmark_settings.benchmarkFilters}"); - } + // 3. Json Output: BenchmarkName_RunName.json + string outputJson = Path.Combine(basePath, $"{benchmarkName}_{runName}.json"); + if (!File.Exists(outputJson)) + { + missingOutputs.Add("Output Json"); } - // Else, add all the benchmarks. - else + // Optionally Requested Files: + // 1. Trace: BenchmarkName..etl.zip or BenchmarkName..nettrace + if (configuration.TraceConfigurations?.Type != "none") { - foreach (var kvp in benchmarkNameToCommand) + string etlFileName = Path.Combine(basePath, $"{benchmarkName}.{configuration.TraceConfigurations!.Type}.etl.zip"); + string nettraceFileName = Path.Combine(basePath, $"{benchmarkName}.{configuration.TraceConfigurations!.Type}.nettrace"); + if (!File.Exists(etlFileName) && !File.Exists(nettraceFileName)) { - benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + missingOutputs.Add("Traces"); } } - // For each benchmark, iterate over all specified runs. - foreach (var c in benchmarkToNameCommandAsKvpList) + // 2. GCLog: BenchmarkName_GCLog/.log + if (configuration.Environment!.environment_variables!.ContainsKey("DOTNET_GCLog") || + configuration.Environment!.environment_variables!.ContainsKey("COMPlus_GCLog") || + configuration.Runs!.Any(r => r.Value.environment_variables?.ContainsKey("DOTNET_GCLog") ?? false) || + configuration.Runs!.Any(r => r.Value.environment_variables?.ContainsKey("COMPlus_GCLog") ?? false) ) { - foreach (var run in configuration.Runs!) - { - const string NON_RESPONSIVE = @"for 'application' is invalid or not responsive: ""No such host is known"; - const string TIME_OUT = "[Time Out]"; - ProcessExecutionDetails result = ExecuteBenchmarkForRun(configuration, run, c); - string key = GetKey(c.Key, run.Key); - - bool timeout = result.StandardError.Contains(TIME_OUT); - bool nonResponsive = result.StandardOut.Contains(NON_RESPONSIVE) || result.StandardError.Contains(NON_RESPONSIVE); - bool timeoutOrNonResponsive = timeout || nonResponsive; + string basePathForGCLog = Path.Combine(configuration.Output.Path, runName, $"{benchmarkName}_GCLog"); - // Wait 2 minutes and then retry if the run timed out or the host was non-responsive (post corp-net connection and check). - if (result.HasFailed && timeoutOrNonResponsive) - { - string retryReason = timeout ? "the run timed out" : "the server was non-responsive"; - string retryDetails = $"{run.Key} for {c.Key} failed as {retryReason}. Sleeping for 2 minutes and retrying"; - AnsiConsole.MarkupLine($"[red bold] {Markup.Escape(retryDetails)} [/]"); - retryMessages.Add((run.Key, c.Key, retryReason)); - Thread.Sleep(60 * 2 * 1000); - result = ExecuteBenchmarkForRun(configuration, run, c); - executionDetails[key] = result; - } + // If the directory is entirely missing. + if (!Directory.Exists(basePathForGCLog)) + { + missingOutputs.Add("GCLog"); + } - else + else // If the directory exists but somehow we didn't get back the gclog file. + { + IEnumerable gcLogFiles = Directory.EnumerateFiles(basePathForGCLog, "*.log"); + if (gcLogFiles.Count() == 0) { - executionDetails[key] = result; + missingOutputs.Add("GCLog"); } } } - Dictionary> results = AspNetBenchmarksAnalyzeCommand.ExecuteAnalysis(configuration, benchmarkNameToCommand, executionDetails, retryMessages); - return new AspNetBenchmarkResults(executionDetails, results); + if (missingOutputs.Any()) + { + AnsiConsole.Markup($"[yellow bold] Missing the following files from the run: \n\t-{string.Join("\n\t-", missingOutputs)} \n[/]"); + } + + return missingOutputs; } } }