From a42b92d26e9359a485fb6ec234b93146e9ff5b30 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 6 Nov 2020 18:35:43 -0800 Subject: [PATCH] Perf Framework (#16671) --- .../Azure.Sample.Perf.csproj | 12 + common/Perf/Azure.Sample.Perf/DelayTest.cs | 52 ++ common/Perf/Azure.Sample.Perf/DisposeTest.cs | 39 ++ .../Perf/Azure.Sample.Perf/ExceptionTest.cs | 42 ++ .../Azure.Sample.Perf/HttpClientGetTest.cs | 52 ++ common/Perf/Azure.Sample.Perf/NoOpTest.cs | 26 + common/Perf/Azure.Sample.Perf/Program.cs | 16 + common/Perf/Azure.Sample.Perf/TimerRunTest.cs | 34 ++ common/Perf/Azure.Test.Perf.sln | 79 +++ .../Azure.Test.Perf/Azure.Test.Perf.csproj | 11 + .../Azure.Test.Perf/BenchmarkMeasurement.cs | 14 + .../Perf/Azure.Test.Perf/BenchmarkMetadata.cs | 14 + .../Perf/Azure.Test.Perf/BenchmarkOutput.cs | 14 + common/Perf/Azure.Test.Perf/CircularStream.cs | 99 ++++ common/Perf/Azure.Test.Perf/CountOptions.cs | 13 + common/Perf/Azure.Test.Perf/IPerfTest.cs | 19 + common/Perf/Azure.Test.Perf/PerfOptions.cs | 46 ++ common/Perf/Azure.Test.Perf/PerfProgram.cs | 517 ++++++++++++++++++ common/Perf/Azure.Test.Perf/PerfTest.cs | 77 +++ common/Perf/Azure.Test.Perf/PerfTransport.cs | 86 +++ common/Perf/Azure.Test.Perf/RandomStream.cs | 20 + common/Perf/Azure.Test.Perf/SizeOptions.cs | 13 + doc/ApiDocGeneration/Generate-Api-Docs.ps1 | 4 +- eng/.docsettings.yml | 1 + eng/Directory.Build.Data.props | 13 +- eng/Packages.Data.props | 4 +- .../templates/jobs/archetype-sdk-client.yml | 2 +- .../jobs/archetype-sdk-tests-jobs.yml | 2 +- eng/scripts/Export-API.ps1 | 2 +- eng/service.proj | 3 + .../perf/Azure.Template.Perf.csproj | 16 + .../perf/MiniSecretClientTest.cs | 48 ++ sdk/template/Azure.Template/perf/Program.cs | 16 + sdk/template/ci.yml | 1 + 34 files changed, 1396 insertions(+), 11 deletions(-) create mode 100644 common/Perf/Azure.Sample.Perf/Azure.Sample.Perf.csproj create mode 100644 common/Perf/Azure.Sample.Perf/DelayTest.cs create mode 100644 common/Perf/Azure.Sample.Perf/DisposeTest.cs create mode 100644 common/Perf/Azure.Sample.Perf/ExceptionTest.cs create mode 100644 common/Perf/Azure.Sample.Perf/HttpClientGetTest.cs create mode 100644 common/Perf/Azure.Sample.Perf/NoOpTest.cs create mode 100644 common/Perf/Azure.Sample.Perf/Program.cs create mode 100644 common/Perf/Azure.Sample.Perf/TimerRunTest.cs create mode 100644 common/Perf/Azure.Test.Perf.sln create mode 100644 common/Perf/Azure.Test.Perf/Azure.Test.Perf.csproj create mode 100644 common/Perf/Azure.Test.Perf/BenchmarkMeasurement.cs create mode 100644 common/Perf/Azure.Test.Perf/BenchmarkMetadata.cs create mode 100644 common/Perf/Azure.Test.Perf/BenchmarkOutput.cs create mode 100644 common/Perf/Azure.Test.Perf/CircularStream.cs create mode 100644 common/Perf/Azure.Test.Perf/CountOptions.cs create mode 100644 common/Perf/Azure.Test.Perf/IPerfTest.cs create mode 100644 common/Perf/Azure.Test.Perf/PerfOptions.cs create mode 100644 common/Perf/Azure.Test.Perf/PerfProgram.cs create mode 100644 common/Perf/Azure.Test.Perf/PerfTest.cs create mode 100644 common/Perf/Azure.Test.Perf/PerfTransport.cs create mode 100644 common/Perf/Azure.Test.Perf/RandomStream.cs create mode 100644 common/Perf/Azure.Test.Perf/SizeOptions.cs create mode 100644 sdk/template/Azure.Template/perf/Azure.Template.Perf.csproj create mode 100644 sdk/template/Azure.Template/perf/MiniSecretClientTest.cs create mode 100644 sdk/template/Azure.Template/perf/Program.cs diff --git a/common/Perf/Azure.Sample.Perf/Azure.Sample.Perf.csproj b/common/Perf/Azure.Sample.Perf/Azure.Sample.Perf.csproj new file mode 100644 index 0000000000000..4f84d19e9a51b --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/Azure.Sample.Perf.csproj @@ -0,0 +1,12 @@ + + + + Exe + + + + + + + + diff --git a/common/Perf/Azure.Sample.Perf/DelayTest.cs b/common/Perf/Azure.Sample.Perf/DelayTest.cs new file mode 100644 index 0000000000000..32d3088c85cfb --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/DelayTest.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Test.Perf; +using CommandLine; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Sample.Perf +{ + public class DelayTest : PerfTest + { + private static int _instanceCount = 0; + private TimeSpan _delay; + + public DelayTest(DelayTest.DelayOptions options) : base(options) + { + var instanceCount = Interlocked.Increment(ref _instanceCount) - 1; + + _delay = TimeSpan.FromMilliseconds(options.InitialDelayMs * Math.Pow(options.InstanceGrowthFactor, instanceCount)); + } + + public override void Run(CancellationToken cancellationToken) + { + Thread.Sleep(_delay); + _delay = TimeSpan.FromMilliseconds(_delay.TotalMilliseconds * Options.IterationGrowthFactor); + } + + public override async Task RunAsync(CancellationToken cancellationToken) + { + await Task.Delay(_delay, cancellationToken); + _delay = TimeSpan.FromMilliseconds(_delay.TotalMilliseconds * Options.IterationGrowthFactor); + } + + public class DelayOptions : PerfOptions + { + [Option("initialDelayMs", Default = 1000, HelpText = "Initial delay (in milliseconds)")] + public int InitialDelayMs { get; set; } + + // Used for verifying the perf framework correctly computes average throughput across parallel tests of different speed. + // Each instance of this test completes operations at a different rate, to allow for testing scenarios where + // some instances are still waiting when time expires. The first instance completes in 1 second per operation, + // the second instance in 2 seconds, the third instance in 4 seconds, and so on. + [Option("instanceGrowthFactor", Default = 1, HelpText = "Instance growth factor. The delay of instance N will be (InitialDelayMS * (InstanceGrowthFactor ^ InstanceCount)).")] + public double InstanceGrowthFactor { get; set; } + + [Option("iterationGrowthFactor", Default = 1, HelpText = "Iteration growth factor. The delay of iteration N will be (InitialDelayMS * (IterationGrowthFactor ^ IterationCount)).")] + public double IterationGrowthFactor { get; set; } + } + } +} diff --git a/common/Perf/Azure.Sample.Perf/DisposeTest.cs b/common/Perf/Azure.Sample.Perf/DisposeTest.cs new file mode 100644 index 0000000000000..481779d628790 --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/DisposeTest.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Test.Perf; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Sample.Perf +{ + // Used to verify framework calls DisposeAsync() + public class DisposeTest : PerfTest + { + public DisposeTest(PerfOptions options) : base(options) + { + } + + public override void Run(CancellationToken cancellationToken) + { + } + + public override Task RunAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public override void Dispose(bool disposing) + { + Console.WriteLine($"Dispose({disposing})"); + base.Dispose(disposing); + } + + public override ValueTask DisposeAsyncCore() + { + Console.WriteLine("DisposeAsyncCore()"); + return base.DisposeAsyncCore(); + } + } +} diff --git a/common/Perf/Azure.Sample.Perf/ExceptionTest.cs b/common/Perf/Azure.Sample.Perf/ExceptionTest.cs new file mode 100644 index 0000000000000..e4fa9de1d5044 --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/ExceptionTest.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Test.Perf; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Sample.Perf +{ + // Measures the overhead of creating, throwing, and catching an exception (compared to NoOpTest) + public class ExceptionTest : PerfTest + { + public ExceptionTest(PerfOptions options) : base(options) + { + } + + public override void Run(CancellationToken cancellationToken) + { + try + { + throw new InvalidOperationException(); + } + catch + { + } + } + + public override Task RunAsync(CancellationToken cancellationToken) + { + try + { + throw new InvalidOperationException(); + } + catch + { + } + + return Task.CompletedTask; + } + } +} diff --git a/common/Perf/Azure.Sample.Perf/HttpClientGetTest.cs b/common/Perf/Azure.Sample.Perf/HttpClientGetTest.cs new file mode 100644 index 0000000000000..c6d701cec4f06 --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/HttpClientGetTest.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Test.Perf; +using CommandLine; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Sample.Perf +{ + public class HttpClientGetTest : PerfTest + { + private static HttpClient _httpClient; + + public HttpClientGetTest(HttpClientGetOptions options) : base(options) + { + } + + public override Task GlobalSetupAsync() + { + if (Options.Insecure) + { + var httpClientHandler = new HttpClientHandler(); + httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true; + _httpClient = new HttpClient(httpClientHandler); + } + else + { + _httpClient = new HttpClient(); + } + + return Task.CompletedTask; + } + + public override void Run(CancellationToken cancellationToken) + { + _httpClient.GetStringAsync(Options.Url).Wait(); + } + + public override async Task RunAsync(CancellationToken cancellationToken) + { + await _httpClient.GetStringAsync(Options.Url); + } + + public class HttpClientGetOptions : PerfOptions + { + [Option('u', "url", Required = true)] + public string Url { get; set; } + } + } +} diff --git a/common/Perf/Azure.Sample.Perf/NoOpTest.cs b/common/Perf/Azure.Sample.Perf/NoOpTest.cs new file mode 100644 index 0000000000000..0500af96dcc0a --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/NoOpTest.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Test.Perf; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Sample.Perf +{ + // Used for measuring the overhead of the perf framework with the fastest possible test + public class NoOpTest : PerfTest + { + public NoOpTest(PerfOptions options) : base(options) + { + } + + public override void Run(CancellationToken cancellationToken) + { + } + + public override Task RunAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} diff --git a/common/Perf/Azure.Sample.Perf/Program.cs b/common/Perf/Azure.Sample.Perf/Program.cs new file mode 100644 index 0000000000000..3f87fa32cd032 --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/Program.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Test.Perf; +using System.Threading.Tasks; + +namespace Azure.Sample.Perf +{ + public class Program + { + public static async Task Main(string[] args) + { + await PerfProgram.Main(typeof(Program).Assembly, args); + } + } +} diff --git a/common/Perf/Azure.Sample.Perf/TimerRunTest.cs b/common/Perf/Azure.Sample.Perf/TimerRunTest.cs new file mode 100644 index 0000000000000..89f9b82ce2593 --- /dev/null +++ b/common/Perf/Azure.Sample.Perf/TimerRunTest.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Test.Perf; +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Sample.Perf +{ + public class TimerRunTest : PerfTest + { + private readonly SemaphoreSlim _semaphoreSlim; + + private readonly Timer _timer; + + public TimerRunTest(PerfOptions options) : base(options) + { + _semaphoreSlim = new SemaphoreSlim(0); + _timer = new Timer(_ => _semaphoreSlim.Release(), state: null, dueTime: TimeSpan.FromSeconds(1), period: TimeSpan.FromSeconds(1)); + } + + public override void Run(CancellationToken cancellationToken) + { + _semaphoreSlim.Wait(); + } + + public override Task RunAsync(CancellationToken cancellationToken) + { + return _semaphoreSlim.WaitAsync(); + } + } +} diff --git a/common/Perf/Azure.Test.Perf.sln b/common/Perf/Azure.Test.Perf.sln new file mode 100644 index 0000000000000..9f5af5c80d9b8 --- /dev/null +++ b/common/Perf/Azure.Test.Perf.sln @@ -0,0 +1,79 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30621.155 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "Azure.Test.Perf\Azure.Test.Perf.csproj", "{B868679E-AD3C-4367-935A-D290F163DA20}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sample.Perf", "Azure.Sample.Perf\Azure.Sample.Perf.csproj", "{24568380-3D5D-4068-A7E9-4C5C2730D3E8}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template.Perf", "..\..\sdk\template\Azure.Template\perf\Azure.Template.Perf.csproj", "{4A3CADC3-8B42-462B-8135-AEBA5487BE18}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template", "..\..\sdk\template\Azure.Template\src\Azure.Template.csproj", "{02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B868679E-AD3C-4367-935A-D290F163DA20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Debug|x64.ActiveCfg = Debug|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Debug|x64.Build.0 = Debug|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Debug|x86.ActiveCfg = Debug|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Debug|x86.Build.0 = Debug|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Release|Any CPU.Build.0 = Release|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Release|x64.ActiveCfg = Release|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Release|x64.Build.0 = Release|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Release|x86.ActiveCfg = Release|Any CPU + {B868679E-AD3C-4367-935A-D290F163DA20}.Release|x86.Build.0 = Release|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Debug|x64.ActiveCfg = Debug|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Debug|x64.Build.0 = Debug|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Debug|x86.ActiveCfg = Debug|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Debug|x86.Build.0 = Debug|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Release|Any CPU.Build.0 = Release|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Release|x64.ActiveCfg = Release|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Release|x64.Build.0 = Release|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Release|x86.ActiveCfg = Release|Any CPU + {24568380-3D5D-4068-A7E9-4C5C2730D3E8}.Release|x86.Build.0 = Release|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Debug|x64.ActiveCfg = Debug|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Debug|x64.Build.0 = Debug|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Debug|x86.ActiveCfg = Debug|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Debug|x86.Build.0 = Debug|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Release|Any CPU.Build.0 = Release|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Release|x64.ActiveCfg = Release|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Release|x64.Build.0 = Release|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Release|x86.ActiveCfg = Release|Any CPU + {4A3CADC3-8B42-462B-8135-AEBA5487BE18}.Release|x86.Build.0 = Release|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Debug|x64.ActiveCfg = Debug|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Debug|x64.Build.0 = Debug|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Debug|x86.ActiveCfg = Debug|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Debug|x86.Build.0 = Debug|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Release|Any CPU.Build.0 = Release|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Release|x64.ActiveCfg = Release|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Release|x64.Build.0 = Release|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Release|x86.ActiveCfg = Release|Any CPU + {02BFED5E-F4AE-471F-8B02-764E3D5FF5DC}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C874CFF4-58D8-4152-8B2C-36E96A07D974} + EndGlobalSection +EndGlobal diff --git a/common/Perf/Azure.Test.Perf/Azure.Test.Perf.csproj b/common/Perf/Azure.Test.Perf/Azure.Test.Perf.csproj new file mode 100644 index 0000000000000..6debc3b1af67d --- /dev/null +++ b/common/Perf/Azure.Test.Perf/Azure.Test.Perf.csproj @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/common/Perf/Azure.Test.Perf/BenchmarkMeasurement.cs b/common/Perf/Azure.Test.Perf/BenchmarkMeasurement.cs new file mode 100644 index 0000000000000..c5c1664f986e4 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/BenchmarkMeasurement.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +namespace Azure.Test.Perf +{ + internal class BenchmarkMeasurement + { + public DateTime Timestamp { get; internal set; } + public string Name { get; internal set; } + public object Value { get; internal set; } + } +} diff --git a/common/Perf/Azure.Test.Perf/BenchmarkMetadata.cs b/common/Perf/Azure.Test.Perf/BenchmarkMetadata.cs new file mode 100644 index 0000000000000..eed7c102faebc --- /dev/null +++ b/common/Perf/Azure.Test.Perf/BenchmarkMetadata.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Test.Perf +{ + internal class BenchmarkMetadata + { + public string Source { get; set; } + public string Name { get; set; } + public string ShortDescription { get; set; } + public string LongDescription { get; set; } + public string Format { get; set; } + } +} diff --git a/common/Perf/Azure.Test.Perf/BenchmarkOutput.cs b/common/Perf/Azure.Test.Perf/BenchmarkOutput.cs new file mode 100644 index 0000000000000..63b2e3c4beb29 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/BenchmarkOutput.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; + +namespace Azure.Test.Perf +{ + internal class BenchmarkOutput + { + public List Metadata { get; } = new List(); + + public List Measurements { get; } = new List(); + } +} diff --git a/common/Perf/Azure.Test.Perf/CircularStream.cs b/common/Perf/Azure.Test.Perf/CircularStream.cs new file mode 100644 index 0000000000000..a86b8ba24a6ca --- /dev/null +++ b/common/Perf/Azure.Test.Perf/CircularStream.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; + +namespace Azure.Test.Perf +{ + public class CircularStream : Stream + { + private readonly Stream _innerStream; + private readonly long _length; + + private long _position; + + public override bool CanRead => true; + + public override bool CanSeek => true; + + public override bool CanWrite => false; + + public override long Length => _length; + + public override long Position + { + get + { + return _position; + } + set + { + if (value < 0 || value > Length) + { + throw new ArgumentException("Position must be between 0 and Length inclusive"); + } + + _position = value; + _innerStream.Position = _position % _innerStream.Length; + } + } + + public CircularStream(Stream innerStream, long length) + { + _innerStream = innerStream; + _length = length; + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + var bytesAvailable = (Length - Position); + var bytesToRead = (int)Math.Min((long)count, bytesAvailable); + + var bytesRead = _innerStream.Read(buffer, offset, bytesToRead); + + if (_innerStream.Position == _innerStream.Length) + { + _innerStream.Seek(0, SeekOrigin.Begin); + } + + Position += bytesRead; + + return bytesRead; + } + + public override long Seek(long offset, SeekOrigin origin) + { + Position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.End => Length - offset, + SeekOrigin.Current => Position + offset, + _ => throw new InvalidOperationException() + }; + + return Position; + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + protected override void Dispose(bool disposing) + { + _innerStream.Dispose(); + base.Dispose(disposing); + } + } +} diff --git a/common/Perf/Azure.Test.Perf/CountOptions.cs b/common/Perf/Azure.Test.Perf/CountOptions.cs new file mode 100644 index 0000000000000..315e3376aa129 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/CountOptions.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using CommandLine; + +namespace Azure.Test.Perf +{ + public class CountOptions : PerfOptions + { + [Option('c', "count", Default = 10, HelpText = "Number of items")] + public int Count { get; set; } + } +} diff --git a/common/Perf/Azure.Test.Perf/IPerfTest.cs b/common/Perf/Azure.Test.Perf/IPerfTest.cs new file mode 100644 index 0000000000000..71ec05cbdc2f5 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/IPerfTest.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Test.Perf +{ + internal interface IPerfTest : IDisposable, IAsyncDisposable + { + Task GlobalSetupAsync(); + Task SetupAsync(); + void Run(CancellationToken cancellationToken); + Task RunAsync(CancellationToken cancellationToken); + Task CleanupAsync(); + Task GlobalCleanupAsync(); + } +} diff --git a/common/Perf/Azure.Test.Perf/PerfOptions.cs b/common/Perf/Azure.Test.Perf/PerfOptions.cs new file mode 100644 index 0000000000000..6fdd2fdf2eab5 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/PerfOptions.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using CommandLine; + +namespace Azure.Test.Perf +{ + public class PerfOptions + { + [Option('d', "duration", Default = 10, HelpText = "Duration of test in seconds")] + public int Duration { get; set; } + + [Option("host", HelpText = "Host to redirect HTTP requests")] + public string Host { get; set; } + + [Option("insecure", HelpText = "Allow untrusted SSL certs")] + public bool Insecure { get; set; } + + [Option('i', "iterations", Default = 1, HelpText = "Number of iterations of main test loop")] + public int Iterations { get; set; } + + [Option("job-statistics", HelpText = "Print job statistics (used by automation)")] + public bool JobStatistics { get; set; } + + [Option('l', "latency", HelpText = "Track and print per-operation latency statistics")] + public bool Latency { get; set; } + + [Option("no-cleanup", HelpText = "Disables test cleanup")] + public bool NoCleanup { get; set; } + + [Option('p', "parallel", Default = 1, HelpText = "Number of operations to execute in parallel")] + public int Parallel { get; set; } + + [Option("port", HelpText = "Port to redirect HTTP requests")] + public int? Port { get; set; } + + [Option('r', "rate", HelpText = "Target throughput (ops/sec)")] + public int? Rate { get; set; } + + [Option("sync", HelpText = "Runs sync version of test")] + public bool Sync { get; set; } + + [Option('w', "warmup", Default = 5, HelpText = "Duration of warmup in seconds")] + public int Warmup { get; set; } + } +} diff --git a/common/Perf/Azure.Test.Perf/PerfProgram.cs b/common/Perf/Azure.Test.Perf/PerfProgram.cs new file mode 100644 index 0000000000000..df379728f4371 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/PerfProgram.cs @@ -0,0 +1,517 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using CommandLine; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime; +using System.Text.Json; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Azure.Test.Perf +{ + public static class PerfProgram + { + private static int[] _completedOperations; + private static TimeSpan[] _lastCompletionTimes; + private static List[] _latencies; + private static List[] _correctedLatencies; + private static Channel<(TimeSpan, Stopwatch)> _pendingOperations; + + public static async Task Main(Assembly assembly, string[] args) + { + var testTypes = assembly.ExportedTypes + .Where(t => typeof(IPerfTest).IsAssignableFrom(t) && !t.IsAbstract); + + if (testTypes.Any()) + { + var optionTypes = GetOptionTypes(testTypes); + await Parser.Default.ParseArguments(args, optionTypes).MapResult( + async o => + { + var verbName = o.GetType().GetCustomAttribute().Name; + var testType = testTypes.Where(t => GetVerbName(t.Name) == verbName).Single(); + await Run(testType, o); + }, + errors => Task.CompletedTask + ); + } + else + { + Console.WriteLine($"Assembly '{assembly.GetName().Name}' does not contain any types implementing 'IPerfStressTest'"); + } + } + + private static async Task Run(Type testType, PerfOptions options) + { + // Require Server GC, since most performance-sensitive usage will be in ASP.NET apps which + // enable Server GC by default. Though Server GC is disabled on 1-core machines as of + // .NET Core 3.0 (https://github.com/dotnet/runtime/issues/12484). + if (Environment.ProcessorCount > 1 && !GCSettings.IsServerGC) + { + throw new InvalidOperationException("Requires server GC"); + } + + if (options.JobStatistics) + { + Console.WriteLine("Application started."); + } + + Console.WriteLine("=== Versions ==="); + Console.WriteLine($"Runtime: {Environment.Version}"); + var azureAssemblies = testType.Assembly.GetReferencedAssemblies() + .Where(a => a.Name.StartsWith("Azure", StringComparison.OrdinalIgnoreCase)) + .Where(a => !a.Name.Equals("Azure.Test.Perf", StringComparison.OrdinalIgnoreCase)) + .OrderBy(a => a.Name); + foreach (var a in azureAssemblies) + { + Console.WriteLine($"{a.Name}: {a.Version}"); + } + Console.WriteLine(); + + Console.WriteLine("=== Options ==="); + Console.WriteLine(JsonSerializer.Serialize(options, options.GetType(), new JsonSerializerOptions() + { + WriteIndented = true + })); + Console.WriteLine(); + + using var setupStatusCts = new CancellationTokenSource(); + var setupStatusThread = PrintStatus("=== Setup ===", () => ".", newLine: false, setupStatusCts.Token); + + using var cleanupStatusCts = new CancellationTokenSource(); + Thread cleanupStatusThread = null; + + var tests = new IPerfTest[options.Parallel]; + for (var i = 0; i < options.Parallel; i++) + { + tests[i] = (IPerfTest)Activator.CreateInstance(testType, options); + } + + try + { + + try + { + await tests[0].GlobalSetupAsync(); + + try + { + await Task.WhenAll(tests.Select(t => t.SetupAsync())); + setupStatusCts.Cancel(); + setupStatusThread.Join(); + + if (options.Warmup > 0) + { + await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Warmup, "Warmup"); + } + + for (var i = 0; i < options.Iterations; i++) + { + var title = "Test"; + if (options.Iterations > 1) + { + title += " " + (i + 1); + } + await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Duration, title, options.JobStatistics, options.Latency); + } + } + finally + { + if (!options.NoCleanup) + { + if (cleanupStatusThread == null) + { + cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token); + } + + await Task.WhenAll(tests.Select(t => t.CleanupAsync())); + } + } + } + finally + { + if (!options.NoCleanup) + { + if (cleanupStatusThread == null) + { + cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token); + } + + await tests[0].GlobalCleanupAsync(); + } + } + } + finally + { + await Task.WhenAll(tests.Select(t => t.DisposeAsync().AsTask())); + } + + cleanupStatusCts.Cancel(); + if (cleanupStatusThread != null) + { + cleanupStatusThread.Join(); + } + } + + private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parallel, int? rate, + int durationSeconds, string title, bool jobStatistics = false, bool latency = false) + { + _completedOperations = new int[parallel]; + _lastCompletionTimes = new TimeSpan[parallel]; + + if (latency) + { + _latencies = new List[parallel]; + for (var i = 0; i < parallel; i++) + { + _latencies[i] = new List(); + } + + if (rate.HasValue) + { + _correctedLatencies = new List[parallel]; + for (var i = 0; i < parallel; i++) + { + _correctedLatencies[i] = new List(); + } + } + } + + var duration = TimeSpan.FromSeconds(durationSeconds); + using var testCts = new CancellationTokenSource(duration); + var cancellationToken = testCts.Token; + + var lastCompleted = 0; + + using var progressStatusCts = new CancellationTokenSource(); + var progressStatusThread = PrintStatus( + $"=== {title} ===" + Environment.NewLine + + "Current\t\tTotal", + () => + { + var totalCompleted = _completedOperations.Sum(); + var currentCompleted = totalCompleted - lastCompleted; + lastCompleted = totalCompleted; + return currentCompleted + "\t\t" + totalCompleted; + }, + newLine: true, + progressStatusCts.Token); + + Thread pendingOperationsThread = null; + if (rate.HasValue) + { + _pendingOperations = Channel.CreateUnbounded>(); + pendingOperationsThread = WritePendingOperations(rate.Value, cancellationToken); + } + + if (sync) + { + var threads = new Thread[parallel]; + + for (var i = 0; i < parallel; i++) + { + var j = i; + threads[i] = new Thread(() => RunLoop(tests[j], j, latency, cancellationToken)); + threads[i].Start(); + } + for (var i = 0; i < parallel; i++) + { + threads[i].Join(); + } + } + else + { + var tasks = new Task[parallel]; + for (var i = 0; i < parallel; i++) + { + var j = i; + // Call Task.Run() instead of directly calling RunLoopAsync(), to ensure the requested + // level of parallelism is achieved even if the test RunAsync() completes synchronously. + tasks[j] = Task.Run(() => RunLoopAsync(tests[j], j, latency, cancellationToken)); + } + await Task.WhenAll(tasks); + } + + if (pendingOperationsThread != null) + { + pendingOperationsThread.Join(); + } + + progressStatusCts.Cancel(); + progressStatusThread.Join(); + + Console.WriteLine("=== Results ==="); + + var totalOperations = _completedOperations.Sum(); + var operationsPerSecond = _completedOperations.Zip(_lastCompletionTimes, (operations, time) => (operations / time.TotalSeconds)).Sum(); + var secondsPerOperation = 1 / operationsPerSecond; + var weightedAverageSeconds = totalOperations / operationsPerSecond; + + Console.WriteLine($"Completed {totalOperations} operations in a weighted-average of {weightedAverageSeconds:N2}s " + + $"({operationsPerSecond:N2} ops/s, {secondsPerOperation:N3} s/op)"); + Console.WriteLine(); + + if (latency) + { + PrintLatencies("Latency Distribution", _latencies); + + if (_correctedLatencies != null) + { + PrintLatencies("Corrected Latency Distribution", _correctedLatencies); + } + } + + if (jobStatistics) + { + var output = new BenchmarkOutput(); + + output.Metadata.Add(new BenchmarkMetadata + { + Source = "PerfStress", + Name = "perfstress/throughput", + ShortDescription = "Throughput (ops/sec)", + LongDescription = "Throughput (ops/sec)", + Format = "n2", + }); + + output.Measurements.Add(new BenchmarkMeasurement + { + Timestamp = DateTime.UtcNow, + Name = "perfstress/throughput", + Value = operationsPerSecond, + }); + + Console.WriteLine("#StartJobStatistics"); + Console.WriteLine(JsonSerializer.Serialize(output)); + Console.WriteLine("#EndJobStatistics"); + } + } + + private static void PrintLatencies(string header, List[] latencies) + { + Console.WriteLine($"=== {header} ==="); + var sortedLatencies = latencies.Aggregate>((list1, list2) => list1.Concat(list2)).ToArray(); + Array.Sort(sortedLatencies); + var percentiles = new double[] { 0.5, 0.75, 0.9, 0.99, 0.999, 0.9999, 0.99999, 1.0 }; + foreach (var percentile in percentiles) + { + Console.WriteLine($"{percentile,8:P3}\t{sortedLatencies[(int)(sortedLatencies.Length * percentile) - 1].TotalMilliseconds:N2}ms"); + } + Console.WriteLine(); + } + + private static void RunLoop(IPerfTest test, int index, bool latency, CancellationToken cancellationToken) + { + var sw = Stopwatch.StartNew(); + var latencySw = new Stopwatch(); + try + { + while (!cancellationToken.IsCancellationRequested) + { + if (latency) + { + latencySw.Restart(); + } + + test.Run(cancellationToken); + + if (latency) + { + _latencies[index].Add(latencySw.Elapsed); + } + + _completedOperations[index]++; + _lastCompletionTimes[index] = sw.Elapsed; + } + } + catch (OperationCanceledException) + { + } + } + + private static async Task RunLoopAsync(IPerfTest test, int index, bool latency, CancellationToken cancellationToken) + { + var sw = Stopwatch.StartNew(); + var latencySw = new Stopwatch(); + (TimeSpan Start, Stopwatch Stopwatch) operation = (TimeSpan.Zero, null); + + try + { + while (!cancellationToken.IsCancellationRequested) + { + if (_pendingOperations != null) + { + operation = await _pendingOperations.Reader.ReadAsync(cancellationToken); + } + + if (latency) + { + latencySw.Restart(); + } + + await test.RunAsync(cancellationToken); + + if (latency) + { + _latencies[index].Add(latencySw.Elapsed); + + if (_pendingOperations != null) + { + _correctedLatencies[index].Add(operation.Stopwatch.Elapsed - operation.Start); + } + } + + _completedOperations[index]++; + _lastCompletionTimes[index] = sw.Elapsed; + } + } + catch (Exception e) + { + // Ignore if any part of the exception chain is type OperationCanceledException + if (!ContainsOperationCanceledException(e)) + { + throw; + } + } + } + + private static Thread WritePendingOperations(int rate, CancellationToken token) + { + var thread = new Thread(() => + { + var sw = Stopwatch.StartNew(); + int writtenOperations = 0; + TimeSpan sleep = TimeSpan.FromSeconds(1.0 / rate); + + while (!token.IsCancellationRequested) + { + while (writtenOperations < (rate * sw.Elapsed.TotalSeconds)) + { + _pendingOperations.Writer.TryWrite(ValueTuple.Create(sw.Elapsed, sw)); + writtenOperations++; + } + + Thread.Sleep(sleep); + } + }); + + thread.Start(); + + return thread; + } + + // Run in dedicated thread instead of using async/await in ThreadPool, to ensure this thread has priority + // and never fails to run to due ThreadPool starvation. + private static Thread PrintStatus(string header, Func status, bool newLine, CancellationToken token) + { + var thread = new Thread(() => + { + Console.WriteLine(header); + + bool needsExtraNewline = false; + + while (!token.IsCancellationRequested) + { + try + { + Sleep(TimeSpan.FromSeconds(1), token); + } + catch (OperationCanceledException) + { + } + + var obj = status(); + + if (newLine) + { + Console.WriteLine(obj); + } + else + { + Console.Write(obj); + needsExtraNewline = true; + } + } + + if (needsExtraNewline) + { + Console.WriteLine(); + } + + Console.WriteLine(); + }); + + thread.Start(); + + return thread; + } + + private static void Sleep(TimeSpan timeout, CancellationToken token) + { + var sw = Stopwatch.StartNew(); + while (sw.Elapsed < timeout) + { + if (token.IsCancellationRequested) + { + // Simulate behavior of Task.Delay(TimeSpan, CancellationToken) + throw new OperationCanceledException(); + } + + Thread.Sleep(TimeSpan.FromMilliseconds(10)); + } + } + + private static bool ContainsOperationCanceledException(Exception e) + { + if (e is OperationCanceledException) + { + return true; + } + else if (e.InnerException != null) + { + return ContainsOperationCanceledException(e.InnerException); + } + else + { + return false; + } + } + + // Dynamically create option types with a "Verb" attribute + private static Type[] GetOptionTypes(IEnumerable testTypes) + { + var optionTypes = new List(); + + var ab = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName("Options"), AssemblyBuilderAccess.Run); + var mb = ab.DefineDynamicModule("Options"); + + foreach (var t in testTypes) + { + var baseOptionsType = t.GetConstructors().First().GetParameters()[0].ParameterType; + var tb = mb.DefineType(t.Name + "Options", TypeAttributes.Public, baseOptionsType); + + var attrCtor = typeof(VerbAttribute).GetConstructor(new Type[] { typeof(string), typeof(bool) }); + var verbName = GetVerbName(t.Name); + tb.SetCustomAttribute(new CustomAttributeBuilder(attrCtor, + new object[] { verbName, attrCtor.GetParameters()[1].DefaultValue })); + + optionTypes.Add(tb.CreateType()); + } + + return optionTypes.ToArray(); + } + + private static string GetVerbName(string testName) + { + var lower = testName.ToLowerInvariant(); + return lower.EndsWith("test") ? lower.Substring(0, lower.Length - 4) : lower; + } + } +} diff --git a/common/Perf/Azure.Test.Perf/PerfTest.cs b/common/Perf/Azure.Test.Perf/PerfTest.cs new file mode 100644 index 0000000000000..91ee9fad8cb57 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/PerfTest.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Test.Perf +{ + public abstract class PerfTest : IPerfTest where TOptions : PerfOptions + { + protected TOptions Options { get; private set; } + + public PerfTest(TOptions options) + { + Options = options; + } + + public virtual Task GlobalSetupAsync() + { + return Task.CompletedTask; + } + + public virtual Task SetupAsync() + { + return Task.CompletedTask; + } + + public abstract void Run(CancellationToken cancellationToken); + + public abstract Task RunAsync(CancellationToken cancellationToken); + + public virtual Task CleanupAsync() + { + return Task.CompletedTask; + } + + public virtual Task GlobalCleanupAsync() + { + return Task.CompletedTask; + } + + // https://docs.microsoft.com/en-us/dotnet/standard/garbage-collection/implementing-disposeasync#implement-both-dispose-and-async-dispose-patterns + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + public async ValueTask DisposeAsync() + { + await DisposeAsyncCore(); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + + public virtual void Dispose(bool disposing) + { + } + + public virtual ValueTask DisposeAsyncCore() + { + return default; + } + + protected static string GetEnvironmentVariable(string name) + { + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrEmpty(value)) + { + throw new InvalidOperationException($"Undefined environment variable {name}"); + } + return value; + } + } +} diff --git a/common/Perf/Azure.Test.Perf/PerfTransport.cs b/common/Perf/Azure.Test.Perf/PerfTransport.cs new file mode 100644 index 0000000000000..c84e1c2f75817 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/PerfTransport.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Core; +using Azure.Core.Pipeline; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Azure.Test.Perf +{ + public static class PerfTransport + { + public static HttpPipelineTransport Create(PerfOptions options) + { + HttpClient httpClient; + if (options.Insecure) + { + httpClient = new HttpClient(new HttpClientHandler() + { + ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true + }); + } + else + { + httpClient = new HttpClient(); + } + + var httpClientTransport = new HttpClientTransport(httpClient); + + if (!string.IsNullOrEmpty(options.Host)) + { + return new ChangeUriTransport(httpClientTransport, options.Host, options.Port); + } + else + { + return httpClientTransport; + } + } + + private class ChangeUriTransport : HttpPipelineTransport + { + private readonly HttpPipelineTransport _transport; + private readonly string _host; + private readonly int? _port; + + public ChangeUriTransport(HttpPipelineTransport transport, string host, int? port) + { + _transport = transport; + _host = host; + _port = port; + } + + public override Request CreateRequest() + { + return _transport.CreateRequest(); + } + + public override void Process(HttpMessage message) + { + ChangeUri(message); + _transport.Process(message); + } + + public override ValueTask ProcessAsync(HttpMessage message) + { + ChangeUri(message); + return _transport.ProcessAsync(message); + } + + private void ChangeUri(HttpMessage message) + { + // Ensure Host header is only set once, since the same HttpMessage will be reused on retries + if (!message.Request.Headers.Contains("Host")) + { + message.Request.Headers.Add("Host", message.Request.Uri.Host); + } + + message.Request.Uri.Host = _host; + if (_port.HasValue) + { + message.Request.Uri.Port = _port.Value; + } + } + } + } +} diff --git a/common/Perf/Azure.Test.Perf/RandomStream.cs b/common/Perf/Azure.Test.Perf/RandomStream.cs new file mode 100644 index 0000000000000..ad901bdb63fe2 --- /dev/null +++ b/common/Perf/Azure.Test.Perf/RandomStream.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; + +namespace Azure.Test.Perf +{ + public static class RandomStream + { + private static readonly Lazy _randomBytes = new Lazy(() => + { + var randomData = new byte[1024 * 1024]; + (new Random(0)).NextBytes(randomData); + return randomData; + }); + + public static Stream Create(long size) => new CircularStream(new MemoryStream(_randomBytes.Value, writable: false), size); + } +} diff --git a/common/Perf/Azure.Test.Perf/SizeOptions.cs b/common/Perf/Azure.Test.Perf/SizeOptions.cs new file mode 100644 index 0000000000000..468d029b38e4a --- /dev/null +++ b/common/Perf/Azure.Test.Perf/SizeOptions.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using CommandLine; + +namespace Azure.Test.Perf +{ + public class SizeOptions : PerfOptions + { + [Option('s', "size", Default = 1024, HelpText = "Size of payload (in bytes)")] + public long Size { get; set; } + } +} diff --git a/doc/ApiDocGeneration/Generate-Api-Docs.ps1 b/doc/ApiDocGeneration/Generate-Api-Docs.ps1 index c9fb3d39fb1da..657abf1efab30 100644 --- a/doc/ApiDocGeneration/Generate-Api-Docs.ps1 +++ b/doc/ApiDocGeneration/Generate-Api-Docs.ps1 @@ -78,10 +78,10 @@ mkdir $DocOutDir if ($LibType -eq 'client') { Write-Verbose "Build Packages for Doc Generation - Client" - dotnet build "${RepoRoot}/eng/service.proj" /p:ServiceDirectory=$PackageLocation /p:IncludeTests=false /p:IncludeSamples=false /p:IncludeStress=false /p:OutputPath=$ApiDir /p:TargetFramework=netstandard2.0 + dotnet build "${RepoRoot}/eng/service.proj" /p:ServiceDirectory=$PackageLocation /p:IncludeTests=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false /p:OutputPath=$ApiDir /p:TargetFramework=netstandard2.0 Write-Verbose "Include client Dependencies" - dotnet build "${RepoRoot}/eng/service.proj" /p:ServiceDirectory=$PackageLocation /p:IncludeTests=false /p:IncludeSamples=false /p:IncludeStress=false /p:OutputPath=$ApiDependenciesDir /p:TargetFramework=netstandard2.0 /p:CopyLocalLockFileAssemblies=true + dotnet build "${RepoRoot}/eng/service.proj" /p:ServiceDirectory=$PackageLocation /p:IncludeTests=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false /p:OutputPath=$ApiDependenciesDir /p:TargetFramework=netstandard2.0 /p:CopyLocalLockFileAssemblies=true } if ($LibType -eq 'management') { diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index bfc13c9385165..af8f7246249af 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -7,6 +7,7 @@ omitted_paths: - sdk/*mgmt*/* - sdk/*/*.Management.*/* - samples/* + - common/Perf/* - common/SmokeTests/* - common/Stress/* - sdk/*/samples/* diff --git a/eng/Directory.Build.Data.props b/eng/Directory.Build.Data.props index e741ebe6d9277..1ef1a2ac13863 100644 --- a/eng/Directory.Build.Data.props +++ b/eng/Directory.Build.Data.props @@ -39,10 +39,11 @@ true true true + true true true - true - true + true + true true true @@ -82,7 +83,7 @@ netstandard2.0 - + false netcoreapp2.1 netcoreapp2.1;net461 @@ -145,7 +146,7 @@ $(MSBuildThisFileDirectory)/../sdk/core/Azure.Core.TestFramework/src/Azure.Core.TestFramework.csproj - + false false @@ -154,6 +155,10 @@ + + true + + diff --git a/eng/Packages.Data.props b/eng/Packages.Data.props index bcf6472224a36..8a0db1e8a8b01 100644 --- a/eng/Packages.Data.props +++ b/eng/Packages.Data.props @@ -190,8 +190,8 @@ - - + + diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index b473086d087f5..d0f4a39110adb 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -143,7 +143,7 @@ jobs: dotnet test eng/service.proj --filter TestCategory!=Live --framework $(TestTargetFramework) --logger "trx;LogFileName=$(TestTargetFramework).trx" --logger:"console;verbosity=normal" /p:ServiceDirectory=${{parameters.ServiceToTest}} - /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludeStress=false + /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false /p:Configuration=$(BuildConfiguration) $(ConvertToProjectReferenceOption) /p:CollectCoverage=$(CollectCoverage) displayName: "Build & Test ($(TestTargetFramework))" env: diff --git a/eng/pipelines/templates/jobs/archetype-sdk-tests-jobs.yml b/eng/pipelines/templates/jobs/archetype-sdk-tests-jobs.yml index 6c3fe43f32598..29d8678684d01 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-tests-jobs.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-tests-jobs.yml @@ -79,7 +79,7 @@ jobs: --logger "trx" --logger:"console;verbosity=normal" /p:ServiceDirectory=${{ parameters.ServiceDirectory }} - /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludeStress=false + /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false /p:BuildInParallel=${{ parameters.BuildInParallel }} ${{ platform.AdditionalTestArguments }} diff --git a/eng/scripts/Export-API.ps1 b/eng/scripts/Export-API.ps1 index 60c08ecd72497..830b1776a6104 100644 --- a/eng/scripts/Export-API.ps1 +++ b/eng/scripts/Export-API.ps1 @@ -7,4 +7,4 @@ param ( $servicesProj = Resolve-Path "$PSScriptRoot/../service.proj" -dotnet build /p:GenerateApiListingOnBuild=true /p:RunApiCompat=false /p:GeneratePackageOnBuild=false /p:Configuration=Release /p:IncludeSamples=false /p:IncludeStress=false /p:IncludeTests=false /p:Scope="$ServiceDirectory" /restore $servicesProj +dotnet build /p:GenerateApiListingOnBuild=true /p:RunApiCompat=false /p:GeneratePackageOnBuild=false /p:Configuration=Release /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false /p:IncludeTests=false /p:Scope="$ServiceDirectory" /restore $servicesProj diff --git a/eng/service.proj b/eng/service.proj index 4d996c43dd53d..9e6ecfe5897a8 100644 --- a/eng/service.proj +++ b/eng/service.proj @@ -5,6 +5,7 @@ true true true + true true true false @@ -14,11 +15,13 @@ + + diff --git a/sdk/template/Azure.Template/perf/Azure.Template.Perf.csproj b/sdk/template/Azure.Template/perf/Azure.Template.Perf.csproj new file mode 100644 index 0000000000000..d35274819ae53 --- /dev/null +++ b/sdk/template/Azure.Template/perf/Azure.Template.Perf.csproj @@ -0,0 +1,16 @@ + + + + Exe + + + + + + + + + + + + diff --git a/sdk/template/Azure.Template/perf/MiniSecretClientTest.cs b/sdk/template/Azure.Template/perf/MiniSecretClientTest.cs new file mode 100644 index 0000000000000..522bfe869f312 --- /dev/null +++ b/sdk/template/Azure.Template/perf/MiniSecretClientTest.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Azure.Identity; +using Azure.Test.Perf; +using CommandLine; + +namespace Azure.Template.Perf +{ + public class MiniSecretClientTest : PerfTest + { + private readonly MiniSecretClient _miniSecretClient; + + public MiniSecretClientTest(MiniSecretClientOptions options) : base(options) + { + var keyVaultUri = GetEnvironmentVariable("KEYVAULT_URL"); + _miniSecretClient = new MiniSecretClient(new Uri(keyVaultUri), new DefaultAzureCredential()); + } + + public override void Run(CancellationToken cancellationToken) + { + // Throttle requests to avoid exceeding service limits + Thread.Sleep(TimeSpan.FromMilliseconds(Options.Delay)); + + _miniSecretClient.GetSecret(Options.SecretName, cancellationToken); + } + + public override async Task RunAsync(CancellationToken cancellationToken) + { + // Throttle requests to avoid exceeding service limits + await Task.Delay(TimeSpan.FromMilliseconds(Options.Delay), cancellationToken); + + await _miniSecretClient.GetSecretAsync(Options.SecretName, cancellationToken); + } + + public class MiniSecretClientOptions : PerfOptions + { + [Option("secret-name", Default = "TestSecret", HelpText = "Name of secret to get")] + public string SecretName { get; set; } + + [Option("delay", Default = 100, HelpText = "Delay between gets (milliseconds)")] + public int Delay { get; set; } + } + } +} diff --git a/sdk/template/Azure.Template/perf/Program.cs b/sdk/template/Azure.Template/perf/Program.cs new file mode 100644 index 0000000000000..754839a39f8f7 --- /dev/null +++ b/sdk/template/Azure.Template/perf/Program.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using Azure.Test.Perf; + +namespace Azure.Template.Perf +{ + public class Program + { + public static async Task Main(string[] args) + { + await PerfProgram.Main(typeof(Program).Assembly, args); + } + } +} diff --git a/sdk/template/ci.yml b/sdk/template/ci.yml index ecc6e75142e1c..bf40f4b238a1e 100644 --- a/sdk/template/ci.yml +++ b/sdk/template/ci.yml @@ -18,6 +18,7 @@ pr: - release/* paths: include: + - common/perf/ - common/stress/ - sdk/template/