Add retry support to testing framework (#33230)

src/ProjectTemplates/BlazorTemplates.Tests/BlazorServerTemplateTest.cs

@@ -16,6 +16,7 @@
 
 namespace Templates.Test
 {
+    [Retry]
     public class BlazorServerTemplateTest : BlazorTemplateTest
     {
         public BlazorServerTemplateTest(ProjectFactoryFixture projectFactory)

src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs

@@ -23,6 +23,7 @@
 
 namespace Templates.Test
 {
+    [Retry]
     public class BlazorWasmTemplateTest : BlazorTemplateTest
     {
         public BlazorWasmTemplateTest(ProjectFactoryFixture projectFactory)

src/ProjectTemplates/Shared/Project.cs

@@ -102,6 +102,13 @@ internal async Task<ProcessResult> RunDotNetNewAsync(
             try
             {
                 Output.WriteLine("Acquired DotNetNewLock");
+
+                if (Directory.Exists(TemplateOutputDir))
+                {
+                    Output.WriteLine($"Template directory already exists, deleting contents of {TemplateOutputDir}");
+                    Directory.Delete(TemplateOutputDir, recursive: true);
+                }
+
                 // Temporary while investigating why this process occasionally never runs or exits on Debian 9
                 environmentVariables.Add("COREHOST_TRACE", "1");
                 using var execution = ProcessEx.Run(Output, AppContext.BaseDirectory, DotNetMuxer.MuxerPathOrDefault(), argString, environmentVariables);

src/Testing/src/RetryAttribute.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.ComponentModel;
+
+namespace Microsoft.AspNetCore.Testing
+{
+    /// <summary>
+    /// Runs a test multiple times when it fails
+    /// This can be used on an assembly, class, or method name. Requires using the AspNetCore test framework.
+    /// </summary>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = false)]
+    public sealed class RetryAttribute : Attribute
+    {
+        public RetryAttribute(int maxRetries = 3)
+        {
+            MaxRetries = maxRetries;
+        }
+
+        /// <summary>
+        /// The maximum number of times to retry a failed test. Defaults to 3.
+        /// </summary>
+        public int MaxRetries { get; }
+    }
+}

src/Testing/src/xunit/AspNetTestInvoker.cs

@@ -45,7 +45,16 @@ await Aggregator.RunAsync(async () =>
                 }
             });
 
-            var time = await base.InvokeTestMethodAsync(testClassInstance);
+            var retryAttribute = GetRetryAttribute(TestMethod);
+            var time = 0.0M;
+            if (retryAttribute == null)
+            {
+                time = await base.InvokeTestMethodAsync(testClassInstance);
+            }
+            else
+            {
+                time = await RetryAsync(retryAttribute, testClassInstance);
+            }
 
             await Aggregator.RunAsync(async () =>
             {
@@ -59,6 +68,45 @@ await Aggregator.RunAsync(async () =>
             return time;
         }
 
+        protected async Task<decimal> RetryAsync(RetryAttribute retryAttribute, object testClassInstance)
+        {
+            var attempts = 0;
+            var timeTaken = 0.0M;
+            for (attempts = 0; attempts < retryAttribute.MaxRetries; attempts++)
+            {
+                timeTaken = await base.InvokeTestMethodAsync(testClassInstance);
+                if (!Aggregator.HasExceptions)
+                {
+                    return timeTaken;
+                }
+                else if (attempts < retryAttribute.MaxRetries - 1)
+                {
+                    _testOutputHelper.WriteLine($"Retrying test, attempt {attempts} of {retryAttribute.MaxRetries} failed.");
+                    await Task.Delay(5000);
+                    Aggregator.Clear();
+                }
+            }
+
+            return timeTaken;
+        }
+
+        private RetryAttribute GetRetryAttribute(MethodInfo methodInfo)
+        {
+            var attributeCandidate = methodInfo.GetCustomAttribute<RetryAttribute>();
+            if (attributeCandidate != null)
+            {
+                return attributeCandidate;
+            }
+
+            attributeCandidate = methodInfo.DeclaringType.GetCustomAttribute<RetryAttribute>();
+            if (attributeCandidate != null)
+            {
+                return attributeCandidate;
+            }
+
+            return methodInfo.DeclaringType.Assembly.GetCustomAttribute<RetryAttribute>();
+        }
+
         private static IEnumerable<ITestMethodLifecycle> GetLifecycleHooks(object testClassInstance, Type testClass, MethodInfo testMethod)
         {
             foreach (var attribute in testMethod.GetCustomAttributes(inherit: true).OfType<ITestMethodLifecycle>())

src/Testing/test/RetryTest.cs

@@ -0,0 +1,31 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Testing
+{
+    [Retry]
+    public class RetryTest
+    {
+        private static int _retryFailsUntil3 = 0;
+
+        [Fact]
+        public void RetryFailsUntil3()
+        {
+            _retryFailsUntil3++;
+            if (_retryFailsUntil3 != 2) throw new Exception("NOOOOOOOO");
+        }
+
+        private static int _canOverrideRetries = 0;
+
+        [Fact]
+        [Retry(5)]
+        public void CanOverrideRetries()
+        {
+            _canOverrideRetries++;
+            if (_canOverrideRetries != 5) throw new Exception("NOOOOOOOO");
+        }
+    }
+}