From bfd32ed2e35c209277909a3a833c143cababa0af Mon Sep 17 00:00:00 2001 From: mrsharm Date: Tue, 19 Mar 2024 13:05:57 -0700 Subject: [PATCH 01/12] Added ping check rather than dir existence check --- .../AspNetBenchmarksCommand.cs | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 3ace859eba2..16faba0e7a6 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -7,6 +7,8 @@ using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Net.NetworkInformation; +using System.Security.Policy; using System.Text; using System.Text.RegularExpressions; @@ -45,14 +47,30 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe ASPNetBenchmarksConfiguration configuration = ASPNetBenchmarksConfigurationParser.Parse(settings.ConfigurationPath); // Before running the ASP.NET benchmarks, execute a few checks: - // Check 1. If you are connected to corpnet. These tests only run if you are connected to corp-net and therefore, preemptively, check this. + // Check 1. If the ASP.NET Machines are pingable. // Check 2. The host machines reboot between 12:00 AM - 12:08 AM PST. Check if we are running during that time, and if so, delay running the infrastructure. // Check 1. - if (!Directory.Exists(@"\\clrmain\tools\")) + const string machineName = "aspnet-citrine-win"; + bool success = false; + Ping ping = new Ping(); + try { - // Not connected to corpnet if that folder is unavailable. - AnsiConsole.MarkupLine($"[red bold] Not connected to corpnet. Ensure you are connected before proceeding with the ASP.NET Benchmarks [/]"); + PingReply reply = ping.Send(machineName); + + if (reply.Status == IPStatus.Success) + { + success = true; + } + } + + catch (PingException _) + { + } + + if (!success) + { + AnsiConsole.MarkupLine($"[red bold] Cannot ping the ASP.NET Machines. Ensure you are connected to corpnet or check if the machines are down before proceeding to run with the ASP.NET Benchmarks [/]"); return -1; } From ec905db15626801188f9afb33a2617d865a4c200 Mon Sep 17 00:00:00 2001 From: mrsharm Date: Tue, 19 Mar 2024 13:20:56 -0700 Subject: [PATCH 02/12] Added fix for filters --- .../Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 16faba0e7a6..32deae71761 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -8,7 +8,6 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Net.NetworkInformation; -using System.Security.Policy; using System.Text; using System.Text.RegularExpressions; @@ -292,6 +291,9 @@ public static AspNetBenchmarkResults RunASPNetBenchmarks(ASPNetBenchmarksConfigu } } + // Overwrite the dictionary with the most up to date results of what will be run. + benchmarkNameToCommand = benchmarkToNameCommandAsKvpList.ToDictionary(pair => pair.Key, pair => pair.Value); + // For each benchmark, iterate over all specified runs. foreach (var c in benchmarkToNameCommandAsKvpList) { From ee880a2979c00aa74d267fa18d6325a7d9af556f Mon Sep 17 00:00:00 2001 From: mrsharm Date: Tue, 19 Mar 2024 14:49:56 -0700 Subject: [PATCH 03/12] Added the ability to restart a run in case the ASP.NET machines have gone to sleep --- .../AspNetBenchmarksCommand.cs | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 32deae71761..cccc3d80f59 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -63,7 +63,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe } } - catch (PingException _) + catch (PingException exp) { } @@ -74,14 +74,15 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe } // Check 2. - SleepUntilHostsHaveRestarted(); + // We don't care about the output from the following method here since this is at a point before the tests have even started to run. + bool _ = TrySleepUntilHostsHaveRestarted(); RunASPNetBenchmarks(configuration); AnsiConsole.MarkupLine($"[bold green] Report generated at: {configuration.Output.Path} [/]"); return 0; } - private static void SleepUntilHostsHaveRestarted() + private static bool TrySleepUntilHostsHaveRestarted() { DateTime now = DateTime.UtcNow; @@ -98,18 +99,17 @@ private static void SleepUntilHostsHaveRestarted() TimeSpan timeUntilEnd = end - pstNow; int secondsLeft = (int)timeUntilEnd.TotalSeconds; - // If we are between 12:00 AM and 12:09 AM PST, sleep for + // If we are between 12:00 AM and 12:09 AM PST, sleep for the seconds left. AnsiConsole.MarkupLine($"[yellow bold] ({DateTime.Now}) ASP.NET Benchmarks Sleeping for {secondsLeft} seconds since the host machines are rebooting. [/]"); Thread.Sleep((secondsLeft) * 1000); + return true; } + + return false; } private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksConfiguration configuration, KeyValuePair run, KeyValuePair benchmarkToCommand) { - // At the start of a run, if we are at a point in time where we are between the time where we deterministically know the host machines need to restart, - // sleep for the remaining time until the machines are back up. - SleepUntilHostsHaveRestarted(); - OS os = !benchmarkToCommand.Key.Contains("Win") ? OS.Linux : OS.Windows; (string, string) commandLine = ASPNetBenchmarksCommandBuilder.Build(configuration, run, benchmarkToCommand, os); @@ -141,6 +141,9 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo AnsiConsole.MarkupLine($"[green bold] ({DateTime.Now}) Running ASP.NET Benchmark for Configuration {configuration.Name} {run.Key} {benchmarkToCommand.Key} [/]"); + // Add the command line to the top of the output file. + output.AppendLine($"Command: {commandLine.Item1} {commandLine.Item2}"); + crankProcess.OutputDataReceived += (s, d) => { output.AppendLine(d?.Data); @@ -301,6 +304,15 @@ public static AspNetBenchmarkResults RunASPNetBenchmarks(ASPNetBenchmarksConfigu { const string NON_RESPONSIVE = @"for 'application' is invalid or not responsive: ""No such host is known"; const string TIME_OUT = "[Time Out]"; + + // At the start of a run, if we are at a point in time where we are between the time where we deterministically know the host machines need to restart, + // sleep for the remaining time until the machines are back up. This may seem like an infinite recursion but we only restart if the hosts have been restarted. + if (TrySleepUntilHostsHaveRestarted()) + { + // If the machine went to sleep - restart the entire run. + return RunASPNetBenchmarks(configuration); + } + ProcessExecutionDetails result = ExecuteBenchmarkForRun(configuration, run, c); string key = GetKey(c.Key, run.Key); From 8cd7733712272b0852d042dfc341ce89400272c6 Mon Sep 17 00:00:00 2001 From: mrsharm Date: Tue, 19 Mar 2024 14:55:34 -0700 Subject: [PATCH 04/12] Added the Linux Container scenario --- .../ASPNetBenchmarks-Linux-Container.csv | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux-Container.csv diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux-Container.csv b/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux-Container.csv new file mode 100644 index 00000000000..2d5bfc90e25 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux-Container.csv @@ -0,0 +1,43 @@ +Legend,CommandLine +FortunesAspNet-intel-lin-4cores-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-4cores-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-4cores-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1core-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-1core-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1core-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-Unlimited-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --load.options.reuseBuild true +FortunesAspNet-intel-lin-Unlimited-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-Unlimited-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-16000m-16gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-16000m-16gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-16000m-16gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-4000m-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-4000m-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-4000m-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1000m-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-1000m-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-1000m-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-500m-256mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.options.reuseBuild true +FortunesAspNet-intel-lin-500m-256mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true +FortunesAspNet-intel-lin-500m-256mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4cores-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-4cores-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4cores-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0-3 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1core-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-1core-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1core-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuSet 0 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-Unlimited-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --load.options.reuseBuild true +JsonAspNet-intel-lin-Unlimited-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-Unlimited-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-16000m-16gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-16000m-16gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-16000m-16gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 16 --application.memoryLimitInBytes 16000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4000m-1gb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-4000m-1gb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-4000m-1gb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 4 --application.memoryLimitInBytes 1000000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1000m-512mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-1000m-512mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-1000m-512mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 1 --application.memoryLimitInBytes 512000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-500m-256mb-0rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.options.reuseBuild true +JsonAspNet-intel-lin-500m-256mb-10000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=10000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true +JsonAspNet-intel-lin-500m-256mb-1000rps, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/containers.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json_aspnet --profile intel-lin-app --profile intel-load-load --profile intel-db-db --variable rate=1000 --application.cpuLimitRatio 0.5 --application.memoryLimitInBytes 256000000 --load.job bombardier --load.options.reuseBuild true \ No newline at end of file From 3d1b4156afc479eb497326b9adf69f3308cce1ca Mon Sep 17 00:00:00 2001 From: mrsharm Date: Wed, 20 Mar 2024 12:02:51 -0700 Subject: [PATCH 05/12] Added check to see if we are missing outputs --- .../ASPNetBenchmarks.CommandBuilder.cs | 2 +- .../ASPNetBenchmark.Configuration.cs | 2 +- .../AspNetBenchmarksCommand.cs | 280 ++++++++++++------ 3 files changed, 187 insertions(+), 97 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs index 0d5bbdc3d32..ec80e7a2b70 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs @@ -17,7 +17,7 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration // Environment Variables. // Add the environment variables from the configuration. Dictionary environmentVariables = new(); - foreach (var env in configuration.Environment.environment_variables) + foreach (var env in configuration.Environment!.environment_variables) { environmentVariables[env.Key] = env.Value; } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs index 2cb86f3acc4..10f2bcddfe8 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs @@ -3,7 +3,7 @@ public sealed class ASPNetBenchmarksConfiguration : ConfigurationBase { public Dictionary? Runs { get; set; } - public Environment? Environment { get; set; } + public Environment Environment { get; set; } public BenchmarkSettings? benchmark_settings { get; set; } public Output? Output { get; set; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index cccc3d80f59..3abeac6449d 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -8,8 +8,11 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Net.NetworkInformation; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; using System.Text; using System.Text.RegularExpressions; +using XPlot.Plotly; namespace GC.Infrastructure.Commands.ASPNetBenchmarks { @@ -108,7 +111,130 @@ private static bool TrySleepUntilHostsHaveRestarted() return false; } - private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksConfiguration configuration, KeyValuePair run, KeyValuePair benchmarkToCommand) + + public static AspNetBenchmarkResults RunASPNetBenchmarks(ASPNetBenchmarksConfiguration configuration) + { + List<(string run, string benchmark, string reason)> retryMessages = new(); + Dictionary executionDetails = new(); + Core.Utilities.TryCreateDirectory(configuration.Output.Path); + + // Parse the CSV file for the information. + string[] lines = File.ReadAllLines(configuration.benchmark_settings!.benchmark_file!); + Dictionary benchmarkNameToCommand = new(StringComparer.OrdinalIgnoreCase); + + for (int lineIdx = 0; lineIdx < lines.Length; lineIdx++) + { + if (lineIdx == 0) + { + continue; + } + + string[] line = lines[lineIdx].Split(',', StringSplitOptions.TrimEntries); + Debug.Assert(line.Length == 2); + + string benchmarkName = line[0]; + string benchmarkCommands = line[1]; + + benchmarkNameToCommand[benchmarkName] = benchmarkCommands; + } + + List> benchmarkToNameCommandAsKvpList = new(); + bool noBenchmarkFilters = + (configuration.benchmark_settings.benchmarkFilters == null || configuration.benchmark_settings.benchmarkFilters.Count == 0); + + // If the user has specified benchmark filters, retrieve them in that order. + if (!noBenchmarkFilters) + { + foreach (var filter in configuration.benchmark_settings.benchmarkFilters!) + { + foreach (var kvp in benchmarkNameToCommand) + { + // Check if we simply end with a "*", if so, match. + if (filter.EndsWith("*") && kvp.Key.StartsWith(filter.Replace("*", ""))) + { + benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } + + // Regular Regex check. + else if (Regex.IsMatch(kvp.Key, $"^{filter}$")) + { + benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } + } + } + + if (benchmarkToNameCommandAsKvpList.Count == 0) + { + throw new ArgumentException($"{nameof(AspNetBenchmarksCommand)}: No benchmark filters found. Please ensure you have added the wildcard character to do the regex matching. Benchmark Filter: {configuration.benchmark_settings.benchmarkFilters}"); + } + } + + // Else, add all the benchmarks. + else + { + foreach (var kvp in benchmarkNameToCommand) + { + benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } + } + + // Overwrite the dictionary with the most up to date results of what will be run. + benchmarkNameToCommand = benchmarkToNameCommandAsKvpList.ToDictionary(pair => pair.Key, pair => pair.Value); + + // For each benchmark, iterate over all specified runs. + foreach (var benchmarkToCommand in benchmarkToNameCommandAsKvpList) + { + ExecuteBenchmarkForRuns(configuration, benchmarkToCommand, executionDetails, retryMessages); + } + + Dictionary> results = AspNetBenchmarksAnalyzeCommand.ExecuteAnalysis(configuration, benchmarkNameToCommand, executionDetails, retryMessages); + return new AspNetBenchmarkResults(executionDetails, results); + } + + internal static void ExecuteBenchmarkForRuns(ASPNetBenchmarksConfiguration configuration, + KeyValuePair benchmarkToCommand, + Dictionary executionDetails, + List<(string, string, string)> retryMessages) + { + foreach (var run in configuration.Runs!) + { + const string NON_RESPONSIVE = @"for 'application' is invalid or not responsive: ""No such host is known"; + const string TIME_OUT = "[Time Out]"; + + // If the machines fall asleep, re-run all the runs for the specific benchmark. + if (TrySleepUntilHostsHaveRestarted()) + { + ExecuteBenchmarkForRuns(configuration, benchmarkToCommand, executionDetails, retryMessages); + return; // Return here to prevent infinite looping. + } + + ProcessExecutionDetails result = ExecuteBenchmarkForRun(configuration, run, benchmarkToCommand); + string key = GetKey(benchmarkToCommand.Key, run.Key); + + bool timeout = result.StandardError.Contains(TIME_OUT); + bool nonResponsive = result.StandardOut.Contains(NON_RESPONSIVE) || result.StandardError.Contains(NON_RESPONSIVE); + bool timeoutOrNonResponsive = timeout || nonResponsive; + + // Wait 2 minutes and then retry if the run timed out or the host was non-responsive (post corp-net connection and check). + if (result.HasFailed && timeoutOrNonResponsive) + { + string retryReason = timeout ? "the run timed out" : "the server was non-responsive"; + string retryDetails = $"{run.Key} for {benchmarkToCommand.Key} failed as {retryReason}. Sleeping for 2 minutes and retrying"; + AnsiConsole.MarkupLine($"[red bold] {Markup.Escape(retryDetails)} [/]"); + retryMessages.Add((run.Key, benchmarkToCommand.Key, retryReason)); + Thread.Sleep(60 * 2 * 1000); + result = ExecuteBenchmarkForRun(configuration, run, benchmarkToCommand); + executionDetails[key] = result; + } + + else + { + executionDetails[key] = result; + } + } + } + + internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksConfiguration configuration, KeyValuePair run, KeyValuePair benchmarkToCommand) { OS os = !benchmarkToCommand.Key.Contains("Win") ? OS.Linux : OS.Windows; (string, string) commandLine = ASPNetBenchmarksCommandBuilder.Build(configuration, run, benchmarkToCommand, os); @@ -175,12 +301,12 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo exitCode = crankProcess.ExitCode; } - string outputFile = Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkToCommand.Key}_{run.Key}.json"); + string outputJson = Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkToCommand.Key}_{run.Key}.json"); string outputDetails = output.ToString(); - if (File.Exists(outputFile)) + if (File.Exists(outputJson)) { - string[] outputLines = File.ReadAllLines(outputFile); + string[] outputLines = File.ReadAllLines(outputJson); // In a quick and dirty way, check the returnCode from the file that'll tell us if the test failed. foreach (var o in outputLines) @@ -216,9 +342,12 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo } } - AnsiConsole.Markup($"[red bold] Failed with the following errors:\n {Markup.Escape(error.ToString())} Check the log file for more information: {logfileOutput} \n[/]"); + AnsiConsole.Markup($"[red bold] Failed with the following errors:\n {Markup.Escape(error.ToString())}. Check the log file for more information: {logfileOutput} \n[/]"); } + // Check to see if we got back all the files regardless of the exit code. + CheckForMissingOutputs(configuration, run.Key, benchmarkToCommand.Key); + File.WriteAllText(logfileOutput, "Output: \n" + outputDetails + "\n Errors: \n" + error.ToString()); return new ProcessExecutionDetails(key: GetKey(benchmarkToCommand.Key, run.Key), commandlineArgs: commandLine.Item1 + " " + commandLine.Item2, @@ -228,119 +357,80 @@ private static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksCo exitCode: exitCode); } - public static AspNetBenchmarkResults RunASPNetBenchmarks(ASPNetBenchmarksConfiguration configuration) + internal static void CheckForMissingOutputs(ASPNetBenchmarksConfiguration configuration, string runName, string benchmarkName) { - List<(string run, string benchmark, string reason)> retryMessages = new(); - Dictionary executionDetails = new(); - Core.Utilities.TryCreateDirectory(configuration.Output.Path); - - // Parse the CSV file for the information. - string[] lines = File.ReadAllLines(configuration.benchmark_settings.benchmark_file); - Dictionary benchmarkNameToCommand = new(StringComparer.OrdinalIgnoreCase); + HashSet missingOutputs = new HashSet(StringComparer.OrdinalIgnoreCase); + string basePath = Path.Combine(configuration.Output!.Path, runName); - for (int lineIdx = 0; lineIdx < lines.Length; lineIdx++) + // Files to expect always: + // 1. Run Output: BenchmarkName.RunName.log + string runOutput = Path.Combine(basePath, $"{benchmarkName}.{runName}.log"); + if (!File.Exists(runOutput)) { - if (lineIdx == 0) - { - continue; - } - - string[] line = lines[lineIdx].Split(',', StringSplitOptions.TrimEntries); - Debug.Assert(line.Length == 2); - - string benchmarkName = line[0]; - string benchmarkCommands = line[1]; - - benchmarkNameToCommand[benchmarkName] = benchmarkCommands; + missingOutputs.Add("Run Output"); } - List> benchmarkToNameCommandAsKvpList = new(); - bool noBenchmarkFilters = - (configuration.benchmark_settings.benchmarkFilters == null || configuration.benchmark_settings.benchmarkFilters.Count == 0); - - // If the user has specified benchmark filters, retrieve them in that order. - if (!noBenchmarkFilters) + // 2. Build Output: BenchmarkName_RunName.build.log + string buildOutput = Path.Combine(basePath, $"{benchmarkName}_{runName}.build.log"); + if (!File.Exists(buildOutput)) { - foreach (var filter in configuration.benchmark_settings.benchmarkFilters!) - { - foreach (var kvp in benchmarkNameToCommand) - { - // Check if we simply end with a "*", if so, match. - if (filter.EndsWith("*") && kvp.Key.StartsWith(filter.Replace("*", ""))) - { - benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); - } + missingOutputs.Add("Build Output"); + } - // Regular Regex check. - else if (Regex.IsMatch(kvp.Key, $"^{filter}$")) - { - benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); - } - } - } + // 3. Application Output: BenchmarkName_RunName.output.log + string applicationOutput = Path.Combine(basePath, $"{benchmarkName}_{runName}.output.log"); + if (!File.Exists(applicationOutput)) + { + missingOutputs.Add("Application Output"); + } - if (benchmarkToNameCommandAsKvpList.Count == 0) - { - throw new ArgumentException($"{nameof(AspNetBenchmarksCommand)}: No benchmark filters found. Please ensure you have added the wildcard character to do the regex matching. Benchmark Filter: {configuration.benchmark_settings.benchmarkFilters}"); - } + // 4. Json Output: BenchmarkName_RunName.json + string outputJson = Path.Combine(basePath, $"{benchmarkName}_{runName}.json"); + if (!File.Exists(outputJson)) + { + missingOutputs.Add("Output Json"); } - // Else, add all the benchmarks. - else + // Optionally Requested Files: + // 1. Trace: BenchmarkName..etl.zip or BenchmarkName..nettrace + if (configuration.TraceConfigurations?.Type != "none") { - foreach (var kvp in benchmarkNameToCommand) + string etlFileName = Path.Combine(basePath, $"{benchmarkName}.{configuration.TraceConfigurations!.Type}.etl.zip"); + string nettraceFileName = Path.Combine(basePath, $"{benchmarkName}.{configuration.TraceConfigurations!.Type}.nettrace"); + if (!File.Exists(etlFileName) && !File.Exists(nettraceFileName)) { - benchmarkToNameCommandAsKvpList.Add(new KeyValuePair(kvp.Key, kvp.Value)); + missingOutputs.Add("Traces"); } } - // Overwrite the dictionary with the most up to date results of what will be run. - benchmarkNameToCommand = benchmarkToNameCommandAsKvpList.ToDictionary(pair => pair.Key, pair => pair.Value); - - // For each benchmark, iterate over all specified runs. - foreach (var c in benchmarkToNameCommandAsKvpList) + // 2. GCLog: BenchmarkName_GCLog/.log + if (configuration.Environment!.environment_variables!.ContainsKey("DOTNET_GCLog") || + configuration.Environment!.environment_variables!.ContainsKey("COMPlus_GCLog") || + configuration.Runs!.Any(r => r.Value.environment_variables?.ContainsKey("DOTNET_GCLog") ?? false) || + configuration.Runs!.Any(r => r.Value.environment_variables?.ContainsKey("COMPlus_GCLog") ?? false) ) { - foreach (var run in configuration.Runs!) - { - const string NON_RESPONSIVE = @"for 'application' is invalid or not responsive: ""No such host is known"; - const string TIME_OUT = "[Time Out]"; - - // At the start of a run, if we are at a point in time where we are between the time where we deterministically know the host machines need to restart, - // sleep for the remaining time until the machines are back up. This may seem like an infinite recursion but we only restart if the hosts have been restarted. - if (TrySleepUntilHostsHaveRestarted()) - { - // If the machine went to sleep - restart the entire run. - return RunASPNetBenchmarks(configuration); - } - - ProcessExecutionDetails result = ExecuteBenchmarkForRun(configuration, run, c); - string key = GetKey(c.Key, run.Key); + string basePathForGCLog = Path.Combine(configuration.Output.Path, runName, $"{benchmarkName}_GCLog"); - bool timeout = result.StandardError.Contains(TIME_OUT); - bool nonResponsive = result.StandardOut.Contains(NON_RESPONSIVE) || result.StandardError.Contains(NON_RESPONSIVE); - bool timeoutOrNonResponsive = timeout || nonResponsive; - - // Wait 2 minutes and then retry if the run timed out or the host was non-responsive (post corp-net connection and check). - if (result.HasFailed && timeoutOrNonResponsive) - { - string retryReason = timeout ? "the run timed out" : "the server was non-responsive"; - string retryDetails = $"{run.Key} for {c.Key} failed as {retryReason}. Sleeping for 2 minutes and retrying"; - AnsiConsole.MarkupLine($"[red bold] {Markup.Escape(retryDetails)} [/]"); - retryMessages.Add((run.Key, c.Key, retryReason)); - Thread.Sleep(60 * 2 * 1000); - result = ExecuteBenchmarkForRun(configuration, run, c); - executionDetails[key] = result; - } + // If the directory is entirely missing. + if (!Directory.Exists(basePathForGCLog)) + { + missingOutputs.Add("GCLog"); + } - else + else // If the directory exists but somehow we didn't get back the gclog file. + { + IEnumerable gcLogFiles = Directory.EnumerateFiles(basePathForGCLog, "*.log"); + if (gcLogFiles.Count() == 0) { - executionDetails[key] = result; + missingOutputs.Add("GCLog"); } } } - Dictionary> results = AspNetBenchmarksAnalyzeCommand.ExecuteAnalysis(configuration, benchmarkNameToCommand, executionDetails, retryMessages); - return new AspNetBenchmarkResults(executionDetails, results); + if (missingOutputs.Any()) + { + AnsiConsole.Markup($"[yellow bold] Missing the following files from the run: \n\t-{string.Join("\n\t-", missingOutputs)} \n[/]"); + } } } } From e6046ca23caf9325b3f6815dbbad8f601c8ff826 Mon Sep 17 00:00:00 2001 From: mrsharm Date: Wed, 20 Mar 2024 12:31:35 -0700 Subject: [PATCH 06/12] Improved robustness and added the dump details --- .../ASPNetBenchmarks.CommandBuilder.cs | 26 +++++++++++++------ .../ASPNetBenchmark.Configuration.cs | 25 +++++++++++++----- .../AspNetBenchmarksCommand.cs | 8 +++--- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs index ec80e7a2b70..7db3f30ff65 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs @@ -42,8 +42,7 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration { string fileNameOfLog = Path.GetFileName(env.Value); commandStringBuilder.Append( $" --application.options.downloadFiles \"*{fileNameOfLog}.log\" " ); - string fileName = Path.GetFileNameWithoutExtension(env.Value); - commandStringBuilder.Append( $" --application.options.downloadFilesOutput \"{Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_GCLog")}\" " ); + commandStringBuilder.Append( $" --application.options.downloadFilesOutput \"{Path.Combine(configuration.Output!.Path, run.Key, $"{benchmarkNameToCommand.Key}_GCLog")}\" " ); } commandStringBuilder.Append($" --application.environmentVariables {env.Key}={variable} "); @@ -85,12 +84,6 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration commandStringBuilder.Append($" --application.options.traceOutput {Path.Combine(configuration.Output.Path, run.Key, (benchmarkNameToCommand.Key + "." + collectType)) + traceFileSuffix}"); } - // Add any additional arguments specified. - if (!string.IsNullOrEmpty(configuration.benchmark_settings.additional_arguments)) - { - commandStringBuilder.Append($" {configuration.benchmark_settings.additional_arguments} "); - } - string frameworkVersion = configuration.Environment.framework_version; // Override the framework version if it's specified at the level of the run. if (!string.IsNullOrEmpty(run.Value.framework_version)) @@ -121,6 +114,23 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration // Add the extra metrics by including the configuration. commandStringBuilder.Append($" --config {Path.Combine("Commands", "RunCommand", "BaseSuite", "PercentileBasedMetricsConfiguration.yml")} "); + // Add the dump collection mechanism on a crash. + // Add the 3 environment variables responsible for this. + string dumpFileName = $"{run.Key}_{benchmarkNameToCommand.Key}_Dump"; + commandStringBuilder.Append($" --application.environmentVariables DOTNET_DbgEnableMiniDump=1 "); + commandStringBuilder.Append($" --application.environmentVariables DOTNET_DbgMiniDumpType=4 "); // We always want the full Dump (4). + commandStringBuilder.Append($" --application.environmentVariables DOTNET_DbgMiniDumpName={dumpFileName} "); + commandStringBuilder.Append($" --application.options.downloadFiles \"*{dumpFileName}*\" "); + commandStringBuilder.Append($" --application.options.downloadFilesOutput \"{Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_Dump")}\" "); + + // Add the ability to download the dump file. + + // Add any additional arguments specified. + if (!string.IsNullOrEmpty(configuration.benchmark_settings.additional_arguments)) + { + commandStringBuilder.Append($" {configuration.benchmark_settings.additional_arguments} "); + } + return (processName, commandStringBuilder.ToString()); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs index 10f2bcddfe8..c09c3babd49 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/ASPNetBenchmark.Configuration.cs @@ -4,8 +4,8 @@ public sealed class ASPNetBenchmarksConfiguration : ConfigurationBase { public Dictionary? Runs { get; set; } public Environment Environment { get; set; } - public BenchmarkSettings? benchmark_settings { get; set; } - public Output? Output { get; set; } + public BenchmarkSettings benchmark_settings { get; set; } + public Output Output { get; set; } } public sealed class Run : RunBase @@ -34,10 +34,7 @@ public static class ASPNetBenchmarksConfigurationParser public static ASPNetBenchmarksConfiguration Parse(string path) { // Preconditions. - if (string.IsNullOrEmpty(path) || !File.Exists(path)) - { - throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(path)} is null/empty or doesn't exist. You must specify a valid path."); - } + ConfigurationChecker.VerifyFile(path, nameof(ASPNetBenchmarksConfigurationParser)); string serializedConfiguration = File.ReadAllText(path); @@ -58,6 +55,22 @@ public static ASPNetBenchmarksConfiguration Parse(string path) throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration)} is null. Check the syntax of the configuration."); } + // Checks if mandatory arguments are specified in the configuration. + if (configuration.Output == null) + { + throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration.Output)} is null. Check the syntax of the configuration."); + } + + if (string.IsNullOrEmpty(configuration.Output.Path)) + { + throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration.Output.Path)} is null or empty. Please specify an output path."); + } + + if (configuration.Environment == null) + { + throw new ArgumentNullException($"{nameof(ASPNetBenchmarksConfigurationParser)}: {nameof(configuration.Environment)} is null. Please add the environment item in the configuration."); + } + return configuration; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 3abeac6449d..7dcb5f68d39 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -45,7 +45,6 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe AnsiConsole.Write(new Rule("ASP.NET Benchmarks Orchestrator")); AnsiConsole.WriteLine(); - ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(AspNetBenchmarksCommand)); ASPNetBenchmarksConfiguration configuration = ASPNetBenchmarksConfigurationParser.Parse(settings.ConfigurationPath); // Before running the ASP.NET benchmarks, execute a few checks: @@ -68,11 +67,12 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe catch (PingException exp) { + // DO NOTHING but catch. We log this later. } if (!success) { - AnsiConsole.MarkupLine($"[red bold] Cannot ping the ASP.NET Machines. Ensure you are connected to corpnet or check if the machines are down before proceeding to run with the ASP.NET Benchmarks [/]"); + AnsiConsole.MarkupLine($"[red bold]Cannot ping the ASP.NET Machines. Ensure you are connected to corpnet or check if the machines are down before proceeding to run with the ASP.NET Benchmarks [/]"); return -1; } @@ -81,7 +81,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe bool _ = TrySleepUntilHostsHaveRestarted(); RunASPNetBenchmarks(configuration); - AnsiConsole.MarkupLine($"[bold green] Report generated at: {configuration.Output.Path} [/]"); + AnsiConsole.MarkupLine($"[bold green] Report generated at: {configuration.Output!.Path} [/]"); return 0; } @@ -116,7 +116,7 @@ public static AspNetBenchmarkResults RunASPNetBenchmarks(ASPNetBenchmarksConfigu { List<(string run, string benchmark, string reason)> retryMessages = new(); Dictionary executionDetails = new(); - Core.Utilities.TryCreateDirectory(configuration.Output.Path); + Core.Utilities.TryCreateDirectory(configuration.Output!.Path); // Parse the CSV file for the information. string[] lines = File.ReadAllLines(configuration.benchmark_settings!.benchmark_file!); From 3d91a172c9dfe8cd7486d9b98e265291290b5778 Mon Sep 17 00:00:00 2001 From: mrsharm Date: Wed, 20 Mar 2024 18:34:57 -0700 Subject: [PATCH 07/12] Clean up position of the additional args. --- .../ASPNetBenchmarks.CommandBuilder.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs index 7db3f30ff65..0cbf10c5109 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/CommandBuilders/ASPNetBenchmarks.CommandBuilder.cs @@ -114,17 +114,6 @@ public static (string, string) Build(ASPNetBenchmarksConfiguration configuration // Add the extra metrics by including the configuration. commandStringBuilder.Append($" --config {Path.Combine("Commands", "RunCommand", "BaseSuite", "PercentileBasedMetricsConfiguration.yml")} "); - // Add the dump collection mechanism on a crash. - // Add the 3 environment variables responsible for this. - string dumpFileName = $"{run.Key}_{benchmarkNameToCommand.Key}_Dump"; - commandStringBuilder.Append($" --application.environmentVariables DOTNET_DbgEnableMiniDump=1 "); - commandStringBuilder.Append($" --application.environmentVariables DOTNET_DbgMiniDumpType=4 "); // We always want the full Dump (4). - commandStringBuilder.Append($" --application.environmentVariables DOTNET_DbgMiniDumpName={dumpFileName} "); - commandStringBuilder.Append($" --application.options.downloadFiles \"*{dumpFileName}*\" "); - commandStringBuilder.Append($" --application.options.downloadFilesOutput \"{Path.Combine(configuration.Output.Path, run.Key, $"{benchmarkNameToCommand.Key}_Dump")}\" "); - - // Add the ability to download the dump file. - // Add any additional arguments specified. if (!string.IsNullOrEmpty(configuration.benchmark_settings.additional_arguments)) { From 9b257a5043353ad9178eb49251daac4f7af3bd8a Mon Sep 17 00:00:00 2001 From: mrsharm Date: Fri, 22 Mar 2024 13:07:20 -0700 Subject: [PATCH 08/12] Removed restriction for the Run output file that'll always be persisted --- .../AspNetBenchmarksCommand.cs | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 7dcb5f68d39..a34e1d8985b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -1,6 +1,5 @@ using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.CommandBuilders; -using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.ASPNetBenchmarks; using Spectre.Console; using Spectre.Console.Cli; @@ -8,11 +7,8 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Net.NetworkInformation; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics.X86; using System.Text; using System.Text.RegularExpressions; -using XPlot.Plotly; namespace GC.Infrastructure.Commands.ASPNetBenchmarks { @@ -362,29 +358,22 @@ internal static void CheckForMissingOutputs(ASPNetBenchmarksConfiguration config HashSet missingOutputs = new HashSet(StringComparer.OrdinalIgnoreCase); string basePath = Path.Combine(configuration.Output!.Path, runName); - // Files to expect always: - // 1. Run Output: BenchmarkName.RunName.log - string runOutput = Path.Combine(basePath, $"{benchmarkName}.{runName}.log"); - if (!File.Exists(runOutput)) - { - missingOutputs.Add("Run Output"); - } - - // 2. Build Output: BenchmarkName_RunName.build.log + // Files to expect always from crank: + // 1. Build Output: BenchmarkName_RunName.build.log string buildOutput = Path.Combine(basePath, $"{benchmarkName}_{runName}.build.log"); if (!File.Exists(buildOutput)) { missingOutputs.Add("Build Output"); } - // 3. Application Output: BenchmarkName_RunName.output.log + // 2. Application Output: BenchmarkName_RunName.output.log string applicationOutput = Path.Combine(basePath, $"{benchmarkName}_{runName}.output.log"); if (!File.Exists(applicationOutput)) - { + { missingOutputs.Add("Application Output"); } - // 4. Json Output: BenchmarkName_RunName.json + // 3. Json Output: BenchmarkName_RunName.json string outputJson = Path.Combine(basePath, $"{benchmarkName}_{runName}.json"); if (!File.Exists(outputJson)) { From b0b3a9d166f7d9253dc98b4454c97e32be402704 Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Wed, 10 Apr 2024 09:48:31 -0700 Subject: [PATCH 09/12] Added a few more places we handle errors from the crank agent --- .../AspNetBenchmarksCommand.cs | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index a34e1d8985b..7d8489aaa4e 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -244,13 +244,13 @@ internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksC // There are 3 main ASP.NET errors: // 1. The server is unavailable - this could be because you aren't connected to CorpNet or the machine is down. // 2. The crank commands are incorrect. - // 3. Test fails because of a test error. + // 3. Test fails because of a test errors. // Launch new crank process. int exitCode = -1; string logfileOutput = Path.Combine(outputPath, $"{GetKey(benchmarkToCommand.Key, run.Key)}.log"); StringBuilder output = new(); - StringBuilder error = new(); + StringBuilder errors = new(); using (Process crankProcess = new()) { @@ -272,7 +272,7 @@ internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksC }; crankProcess.ErrorDataReceived += (s, d) => { - error.AppendLine(d?.Data); + errors.AppendLine(d?.Data); }; crankProcess.Start(); @@ -285,11 +285,12 @@ internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksC if (!crankProcess.HasExited) { AnsiConsole.MarkupLine($"[red bold] ASP.NET Benchmark timed out for: {configuration.Name} {run.Key} {benchmarkToCommand.Key} - skipping the results but writing stdout and stderror to {logfileOutput} [/]"); - File.WriteAllText(logfileOutput, "Output: \n" + output.ToString() + "\n Errors: \n" + error.ToString()); + errors.AppendLine($"Run: {configuration.Name} {run.Key} {benchmarkToCommand.Key} timed out."); + File.WriteAllText(logfileOutput, "Output: \n" + output.ToString() + "\n Errors: \n" + errors.ToString()); return new ProcessExecutionDetails(key: GetKey(benchmarkToCommand.Key, run.Key), commandlineArgs: commandLine.Item1 + " " + commandLine.Item2, environmentVariables: new(), - standardError: "[Time Out]: " + error.ToString(), + standardError: "[Time Out]: " + errors.ToString(), standardOut: output.ToString(), exitCode: exitCode); } @@ -319,10 +320,10 @@ internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksC else { - // For the case where the output file doesn't exist implies that was an issue connecting to the asp.net machines or error number 1. - // This case also applies for incorrect crank arguments or error number 2. - // Move the standard out to the standard error as the process failed. - error.AppendLine(outputDetails); + // For the case where the output file doesn't exist implies that was an issue connecting to the asp.net machines or errors number 1. + // This case also applies for incorrect crank arguments or errors number 2. + // Move the standard out to the standard errors as the process failed. + errors.AppendLine(outputDetails); } if (exitCode != 0) @@ -330,25 +331,31 @@ internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksC string[] outputLines = outputDetails.Split("\n"); foreach (var o in outputLines) { - // Crank provides the standard error from the test itself by this mechanism. + // Crank provides the standard errors from the test itself by this mechanism. // Error #3: Issues with test run. if (o.StartsWith("[STDERR]")) { - error.AppendLine(o.Replace("[STDERR]", "")); + errors.AppendLine(o.Replace("[STDERR]", "")); + } + + // Highlight case where: Configuration 'https://github.com/aspnet/Benchmarks/blob/main/scenarios/aspnet.profiles.yml?raw=true' could not be loaded. + else if (o.Contains("Configuration '") && o.Contains("could not be loaded")) + { + errors.AppendLine(o); } } - AnsiConsole.Markup($"[red bold] Failed with the following errors:\n {Markup.Escape(error.ToString())}. Check the log file for more information: {logfileOutput} \n[/]"); + AnsiConsole.Markup($"[red bold] Failed with the following errors:\n {Markup.Escape(errors.ToString())}. Check the log file for more information: {logfileOutput} \n[/]"); } // Check to see if we got back all the files regardless of the exit code. CheckForMissingOutputs(configuration, run.Key, benchmarkToCommand.Key); - File.WriteAllText(logfileOutput, "Output: \n" + outputDetails + "\n Errors: \n" + error.ToString()); + File.WriteAllText(logfileOutput, "Output: \n" + outputDetails + "\n Errors: \n" + errors.ToString()); return new ProcessExecutionDetails(key: GetKey(benchmarkToCommand.Key, run.Key), commandlineArgs: commandLine.Item1 + " " + commandLine.Item2, environmentVariables: new(), - standardError: error.ToString(), + standardError: errors.ToString(), standardOut: output.ToString(), exitCode: exitCode); } From 4fc72b43f153389248801ec81bcce6d172a9fdd8 Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Wed, 24 Apr 2024 17:08:02 -0700 Subject: [PATCH 10/12] Replaced aspnet-citrine-win with asp-citrine-win as the ping check --- .../Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 7d8489aaa4e..60372ed0c12 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -48,7 +48,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe // Check 2. The host machines reboot between 12:00 AM - 12:08 AM PST. Check if we are running during that time, and if so, delay running the infrastructure. // Check 1. - const string machineName = "aspnet-citrine-win"; + const string machineName = "https://asp-citrine-win"; bool success = false; Ping ping = new Ping(); try From f05d3c899f7f2ecc0f8af846975b67370ced3f53 Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Wed, 24 Apr 2024 17:12:29 -0700 Subject: [PATCH 11/12] Fixed name of machine again --- .../Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index 60372ed0c12..e0076c2ba22 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -48,7 +48,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] AspNetBe // Check 2. The host machines reboot between 12:00 AM - 12:08 AM PST. Check if we are running during that time, and if so, delay running the infrastructure. // Check 1. - const string machineName = "https://asp-citrine-win"; + const string machineName = "asp-citrine-win"; bool success = false; Ping ping = new Ping(); try From e62af8ea0f2208d2318658a58a9c31f926294413 Mon Sep 17 00:00:00 2001 From: Mukund Raghav Sharma Date: Thu, 27 Jun 2024 13:05:35 -0700 Subject: [PATCH 12/12] Updated sleep time if we are during midnight and consider a run a fail if no requested traces are downloaded --- .../ASPNetBenchmarks/AspNetBenchmarksCommand.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs index e0076c2ba22..0c6afe62e3b 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/ASPNetBenchmarks/AspNetBenchmarksCommand.cs @@ -91,7 +91,7 @@ private static bool TrySleepUntilHostsHaveRestarted() // Check if the current time is between 12:00 AM and 12:09 AM. DateTime start = pstNow.Date; // 12:00 AM today. - DateTime end = start.AddMinutes(9); // 12:09 AM today. + DateTime end = start.AddMinutes(10); // 12:10 AM today. if (pstNow >= start && pstNow < end) { @@ -349,7 +349,13 @@ internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksC } // Check to see if we got back all the files regardless of the exit code. - CheckForMissingOutputs(configuration, run.Key, benchmarkToCommand.Key); + HashSet missingOutputs = CheckForMissingOutputsAndReturnNames(configuration, run.Key, benchmarkToCommand.Key); + + // If traces are missing, consider the run failed so that users can re-run it via the Failed yaml. + if (missingOutputs.Contains("Traces")) + { + exitCode = -1; + } File.WriteAllText(logfileOutput, "Output: \n" + outputDetails + "\n Errors: \n" + errors.ToString()); return new ProcessExecutionDetails(key: GetKey(benchmarkToCommand.Key, run.Key), @@ -360,7 +366,7 @@ internal static ProcessExecutionDetails ExecuteBenchmarkForRun(ASPNetBenchmarksC exitCode: exitCode); } - internal static void CheckForMissingOutputs(ASPNetBenchmarksConfiguration configuration, string runName, string benchmarkName) + internal static HashSet CheckForMissingOutputsAndReturnNames(ASPNetBenchmarksConfiguration configuration, string runName, string benchmarkName) { HashSet missingOutputs = new HashSet(StringComparer.OrdinalIgnoreCase); string basePath = Path.Combine(configuration.Output!.Path, runName); @@ -427,6 +433,8 @@ internal static void CheckForMissingOutputs(ASPNetBenchmarksConfiguration config { AnsiConsole.Markup($"[yellow bold] Missing the following files from the run: \n\t-{string.Join("\n\t-", missingOutputs)} \n[/]"); } + + return missingOutputs; } } }