Startup performance measure tool #851
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <TargetFramework>netcoreapp3.0</TargetFramework> | ||
| <OutputType>Exe</OutputType> | ||
| <AssemblyOriginatorKeyFile>dotvvmwizard.snk</AssemblyOriginatorKeyFile> | ||
| <SignAssembly>true</SignAssembly> | ||
| <PackAsTool>true</PackAsTool> | ||
| <ToolCommandName>dotvvm-startup-perf</ToolCommandName> | ||
| <PublicSign Condition=" '$(OS)' != 'Windows_NT' ">true</PublicSign> | ||
| <PackageId>DotVVM.Tools.StartupPerf</PackageId> | ||
| <PackageVersion>2.4.0.2-preview02-76679</PackageVersion> | ||
| <Authors>RIGANTI</Authors> | ||
| <Description>Command-line tool for measuring startup performance of DotVVM apps.</Description> | ||
| <PackageRequireLicenseAcceptance>false</PackageRequireLicenseAcceptance> | ||
| <PackageTags>dotvvm;asp.net;mvvm;owin;dotnetcore;dnx;cli</PackageTags> | ||
| <PackageIconUrl>https://dotvvm.com/Content/images/icons/icon-blue-64x64.png</PackageIconUrl> | ||
| <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <PackageReference Include="system.commandline" Version="2.0.0-beta1.20253.1" /> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| using System; | ||
| using System.IO; | ||
|
|
||
| namespace DotVVM.Framework.StartupPerfTests | ||
| { | ||
| static internal class FileSystemHelper | ||
| { | ||
| public static string CreateTempDir() | ||
| { | ||
| var tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); | ||
| Directory.CreateDirectory(tempDir); | ||
| return tempDir; | ||
| } | ||
|
|
||
| public static bool RemoveDir(string dir) | ||
| { | ||
| try | ||
| { | ||
| if (Directory.Exists(dir)) | ||
| { | ||
| Directory.Delete(dir, true); | ||
| } | ||
| return true; | ||
| } | ||
| catch (Exception) | ||
| { | ||
| return false; | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| using System; | ||
| using System.Net; | ||
| using System.Net.Sockets; | ||
|
|
||
| namespace DotVVM.Framework.StartupPerfTests | ||
| { | ||
| public static class NetworkingHelpers | ||
| { | ||
| public static int FindRandomPort() | ||
| { | ||
| int port; | ||
| var random = new Random(); | ||
| do | ||
| { | ||
| port = 60000 + random.Next(5000); | ||
| } while (!TestPort(port)); | ||
|
|
||
| return port; | ||
| } | ||
|
|
||
| private static bool TestPort(int port) | ||
| { | ||
| using (var client = new TcpClient()) | ||
| { | ||
| try | ||
| { | ||
| client.Connect(new IPEndPoint(IPAddress.Loopback, port)); | ||
| client.Close(); | ||
| return false; | ||
| } | ||
| catch (Exception) | ||
| { | ||
| return true; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Net; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
| using System.CommandLine; | ||
| using System.CommandLine.Invocation; | ||
| using System.Diagnostics; | ||
| using System.Net.Http; | ||
| using System.Net.Sockets; | ||
| using System.Threading; | ||
| using Process = System.Diagnostics.Process; | ||
|
|
||
| namespace DotVVM.Framework.StartupPerfTests | ||
| { | ||
| class Program | ||
| { | ||
| private static TextWriter logWriter; | ||
|
|
||
| static void Main(string[] args) | ||
| { | ||
| var rootCommand = new RootCommand | ||
| { | ||
| new Argument<FileInfo>( | ||
| "project", | ||
| "Path to the project file" | ||
| ) { Arity = ArgumentArity.ExactlyOne }, | ||
| new Option<TestTarget>( | ||
| new [] { "-t", "--type" }, | ||
| "Type of the project - use 'owin' or 'aspnetcore'" | ||
| ) { Required = true }, | ||
| new Option<int>( | ||
| new [] { "-r", "--repeat" }, | ||
| () => 1, | ||
| "How many times the operation should be repeated." | ||
| ), | ||
| new Option<string>( | ||
| new [] { "-u", "--url" }, | ||
| () => "", | ||
| "Relative URL in the app that should be tested." | ||
| ), | ||
| new Option<bool>( | ||
| new [] { "-v", "--verbose" }, | ||
| () => false, | ||
| "Diagnostics output" | ||
| ) | ||
| }; | ||
| rootCommand.Handler = CommandHandler.Create<FileInfo, TestTarget, int, string, bool>((project, type, repeat, url, verbose) => | ||
| { | ||
| new StartupPerformanceTest(project, type, repeat, url, verbose).HandleCommand(); | ||
| }); | ||
| rootCommand.Invoke(args); | ||
| } | ||
| } | ||
|
|
||
| public enum TestTarget | ||
| { | ||
| Owin, | ||
| AspNetCore | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| { | ||
| "profiles": { | ||
| "DotVVM.Framework.StartupPerfTests": { | ||
| "commandName": "Project", | ||
| "commandLineArgs": "-t owin -r 1" | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| using System; | ||
| using System.Diagnostics; | ||
| using System.IO; | ||
| using System.Net; | ||
| using System.Net.Http; | ||
| using System.Net.Sockets; | ||
| using System.Threading; | ||
|
|
||
| namespace DotVVM.Framework.StartupPerfTests | ||
| { | ||
| public class StartupPerformanceTest | ||
| { | ||
| private readonly FileInfo project; | ||
| private readonly TestTarget type; | ||
| private readonly int repeat; | ||
| private readonly string url; | ||
| private readonly bool verbose; | ||
|
|
||
| public StartupPerformanceTest(FileInfo project, TestTarget type, int repeat, string url, bool verbose) | ||
| { | ||
| this.project = project; | ||
| this.type = type; | ||
| this.repeat = repeat; | ||
| this.url = url; | ||
| this.verbose = verbose; | ||
| } | ||
|
|
||
|
|
||
| public void HandleCommand() | ||
| { | ||
| // test project existence | ||
| var projectPath = project.FullName; | ||
| if (!project.Exists) | ||
| { | ||
| throw new Exception($"The project {projectPath} doesn't exist!"); | ||
| } | ||
|
|
||
| // find a random port | ||
| var port = NetworkingHelpers.FindRandomPort(); | ||
| var urlToTest = $"http://localhost:{port}/{url.TrimStart('/')}"; | ||
|
|
||
| // prepare directories | ||
| var dir = Path.GetDirectoryName(projectPath); | ||
| TraceOutput($"Project dir: {dir}"); | ||
| var tempDir = FileSystemHelper.CreateTempDir(); | ||
| TraceOutput($"Temp dir: {tempDir}"); | ||
|
|
||
| long measuredTime = 0; | ||
| if (type == TestTarget.Owin) | ||
| { | ||
| // OWIN | ||
| TraceOutput($"Publishing..."); | ||
| RunProcessAndWait(@"c:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe", @$"""{Path.GetFileName(projectPath)}"" /p:DeployOnBuild=true /t:WebPublish /p:WebPublishMethod=FileSystem /p:publishUrl=""{tempDir}""", dir); | ||
|
|
||
| for (var i = 0; i < repeat; i++) | ||
| { | ||
| TraceOutput($"Attempt #{i + 1}"); | ||
| measuredTime += RunProcessAndWaitForHealthCheck(@"C:\Program Files (x86)\IIS Express\iisexpress.exe", $@"""/path:{dir}"" /port:{port}", dir, urlToTest); | ||
| } | ||
| } | ||
| else if (type == TestTarget.AspNetCore) | ||
| { | ||
| // ASP.NET Core | ||
| TraceOutput($"Publishing..."); | ||
| RunProcessAndWait(@"dotnet", $@"publish -c Release -o ""{tempDir}""", dir); | ||
|
|
||
| for (var i = 0; i < repeat; i++) | ||
| { | ||
| TraceOutput($"Attempt #{i + 1}"); | ||
| measuredTime += RunProcessAndWaitForHealthCheck(@"dotnet", $@"""./{Path.GetFileNameWithoutExtension(projectPath)}.dll"" --urls {urlToTest}", tempDir, urlToTest); | ||
|
There was a problem hiding this comment. Unfortunately, the .NET API for starting processes is terrible, I'd recommend using https://github.com/madelson/MedallionShell, it handles the argument escaping well. There was a problem hiding this comment. Thanks for the tip, it is a nice library. |
||
| } | ||
| } | ||
| else | ||
| { | ||
| throw new NotSupportedException(); | ||
| } | ||
|
|
||
| TraceOutput($"Average time: {measuredTime / repeat}"); | ||
|
|
||
| TraceOutput($"Removing temp dir..."); | ||
| FileSystemHelper.RemoveDir(tempDir); | ||
| TraceOutput($"Done"); | ||
| } | ||
|
|
||
| private void RunProcessAndWait(string path, string arguments, string workingDirectory) | ||
| { | ||
| TraceOutput($"Running {path} {arguments}"); | ||
|
|
||
| var psi = new ProcessStartInfo(path, arguments) { | ||
| CreateNoWindow = true, | ||
| UseShellExecute = false, | ||
| WindowStyle = ProcessWindowStyle.Hidden, | ||
| WorkingDirectory = workingDirectory | ||
| }; | ||
| var process = Process.Start(psi); | ||
| process.WaitForExit(); | ||
| if (process.ExitCode != 0) | ||
| { | ||
| throw new Exception($"Process exited with code {process.ExitCode}!"); | ||
| } | ||
| } | ||
|
|
||
| private long RunProcessAndWaitForHealthCheck(string path, string arguments, string workingDirectory, string urlToTest) | ||
| { | ||
| var psi = new ProcessStartInfo(path, arguments) { | ||
| CreateNoWindow = true, | ||
| UseShellExecute = false, | ||
| WindowStyle = ProcessWindowStyle.Hidden, | ||
| WorkingDirectory = workingDirectory | ||
| }; | ||
| var process = Process.Start(psi); | ||
|
|
||
| var sw = new Stopwatch(); | ||
| sw.Start(); | ||
|
|
||
| retry: | ||
| try | ||
| { | ||
| var wc = new WebClient(); | ||
| var response = wc.DownloadString(urlToTest); | ||
| } | ||
| catch (WebException ex) when (ex.InnerException is HttpRequestException hrex && (hrex.InnerException is IOException || hrex.InnerException is SocketException)) | ||
| { | ||
| Thread.Sleep(100); | ||
| goto retry; | ||
|
There was a problem hiding this comment. Added the timeout option (with a default value of 10 seconds). If the HTTP port doesn't open within the timeout, an exception is thrown. |
||
| } | ||
|
|
||
| if (process.HasExited) | ||
| { | ||
| throw new Exception("The process has exited!"); | ||
|
There was a problem hiding this comment. The check should be inside the |
||
| } | ||
|
|
||
| var time = sw.ElapsedMilliseconds; | ||
| ImportantOutput(time.ToString()); | ||
|
|
||
| process.Kill(); | ||
| return time; | ||
| } | ||
|
|
||
| public void ImportantOutput(string message) | ||
| { | ||
| Console.WriteLine(message); | ||
| } | ||
|
|
||
| public void TraceOutput(string message) | ||
| { | ||
| if (verbose) | ||
| { | ||
| Console.WriteLine(message); | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| # Prepare temp content directory | ||
| $temp = [System.IO.Path]::GetTempPath() | ||
| if (Test-Path $temp/DotVVM.Samples.Common) { | ||
| rmdir $temp/DotVVM.Samples.Common -Recurse -Force | ||
| } | ||
| mkdir $temp/DotVVM.Samples.Common | out-null | ||
| copy ../DotVVM.Samples.Common/Content -Destination $temp/DotVVM.Samples.Common -Recurse -Force | ||
| copy ../DotVVM.Samples.Common/Scripts -Destination $temp/DotVVM.Samples.Common -Recurse -Force | ||
| copy ../DotVVM.Samples.Common/Views -Destination $temp/DotVVM.Samples.Common -Recurse -Force | ||
| copy ../DotVVM.Samples.Common/sampleConfig.json -Destination $temp/DotVVM.Samples.Common -Force | ||
|
|
||
| # Run OWIN tests | ||
| ./bin/Debug/netcoreapp3.0/DotVVM.Framework.StartupPerfTests.exe ../DotVVM.Samples.BasicSamples.Owin/DotVVM.Samples.BasicSamples.Owin.csproj -t owin -v -r 5 | ||
|
|
||
| # Run ASP.NET Core tests | ||
| ./bin/Debug/netcoreapp3.0/DotVVM.Framework.StartupPerfTests.exe ../DotVVM.Samples.BasicSamples.AspNetCoreLatest/DotVVM.Samples.BasicSamples.AspNetCoreLatest.csproj -t aspnetcore -v -r 5 |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should only catch
DirectoryNotFoundException. If there is another problem (like, file with the same name exists, runtime error, ... it should still fail)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The catch was here actually for
IOExceptionandUnauthorizedAccessException- I am using it only to delete a temp folder and in general, it's not a big deal if it fails.I've added a retry logic - sometimes it failed to delete it because some files were still locked.