PoC to output all test information

src/Exercism.TestRunner.CSharp/CompilationTestRunner.cs

@@ -17,25 +17,32 @@ internal static class CompilationTestRunner
         private static readonly TestMessageSink ExecutionMessageSink = new TestMessageSink();
 
         public static async Task<TestRun> Run(Compilation compilation) =>
-            await Run(compilation.Rewrite().ToAssembly().ToAssemblyInfo());
+            await Run(compilation.Rewrite().ToAssembly().ToAssemblyInfo(), compilation);
 
-        private static async Task<TestRun> Run(IAssemblyInfo assemblyInfo)
+        private static async Task<TestRun> Run(IAssemblyInfo assemblyInfo, Compilation compilation)
         {
+            SyntaxNode GetSyntaxNode(ITestMethod testMethod)
+            {
+                var symbol = compilation.GetSymbolsWithName(testMethod.Method.Name, SymbolFilter.Member).FirstOrDefault();
+                var syntaxReference = symbol?.DeclaringSyntaxReferences.FirstOrDefault();
+                return syntaxReference?.GetSyntax();
+            }
+
             var testResults = new List<TestResult>();
-
+            
             ExecutionMessageSink.Execution.TestFailedEvent += args =>
-                testResults.Add(TestResult.FromFailed(args.Message));
+                testResults.Add(TestResult.FromFailed(args.Message, GetSyntaxNode(args.Message.TestMethod)));
 
             ExecutionMessageSink.Execution.TestPassedEvent += args =>
-                testResults.Add(TestResult.FromPassed(args.Message));
+                testResults.Add(TestResult.FromPassed(args.Message, GetSyntaxNode(args.Message.TestMethod)));
 
             var testCases = TestCases(assemblyInfo);
 
             using var assemblyRunner = CreateTestAssemblyRunner(testCases, assemblyInfo.ToTestAssembly());
             await assemblyRunner.RunAsync();
 
             var orderedTestNames = testCases.Select(testCase => testCase.DisplayName).ToArray();
-            var orderedTestResults = testResults.OrderBy(testResult => Array.IndexOf(orderedTestNames, testResult.Name)).ToArray();
+            var orderedTestResults = testResults.OrderBy(testResult => Array.IndexOf(orderedTestNames, testResult.Test)).ToArray();
 
             return TestRun.FromTests(orderedTestResults);
         }

src/Exercism.TestRunner.CSharp/TestResult.cs

@@ -1,21 +1,125 @@
+using System.Linq;
+using Humanizer;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
 using Xunit.Abstractions;
 
 namespace Exercism.TestRunner.CSharp
 {
     internal class TestResult
     {
         public string Name { get; }
+        public string Test { get; }
+        public string Expected { get; }
+        public string Command { get; }
         public string Message { get; }
         public string Output { get; }
         public TestStatus Status { get; }
 
-        private TestResult(string name, TestStatus status, string message, string output) =>
-            (Name, Message, Status, Output) = (name, message, status, output);
+        private TestResult(string name, string test, string expected, string command, TestStatus status, string message, string output) =>
+            (Name, Test, Expected, Command, Message, Status, Output) = (name, test, expected, command, message, status, output);
 
-        public static TestResult FromPassed(ITestPassed test) =>
-            new TestResult(test.TestCase.DisplayName, TestStatus.Pass, null, test.Output);
+        public static TestResult FromPassed(ITestPassed test, SyntaxNode syntaxNode) =>
+            new TestResult(ToName(test.TestCase), ToTest(test.TestCase), ToExpected(syntaxNode), ToCommand(syntaxNode), TestStatus.Pass, null, test.Output);
 
-        public static TestResult FromFailed(ITestFailed test) =>
-            new TestResult(test.TestCase.DisplayName, TestStatus.Fail, TestRunMessage.FromMessages(test.Messages), test.Output);
+        public static TestResult FromFailed(ITestFailed test, SyntaxNode syntaxNode) =>
+            new TestResult(ToName(test.TestCase), ToTest(test.TestCase), ToExpected(syntaxNode), ToCommand(syntaxNode), TestStatus.Fail, TestRunMessage.FromMessages(test.Messages), test.Output);
+
+        private static string ToName(ITestCase testCase) => testCase.TestMethod.Method.Name.Humanize();
+
+        private static string ToTest(ITestCase testCase) => testCase.DisplayName;
+
+        private static string ToExpected(SyntaxNode syntaxNode)
+        {
+            var assertExpression = syntaxNode.DescendantNodes()
+                .OfType<InvocationExpressionSyntax>()
+                .FirstOrDefault(invocationExpression => 
+                    invocationExpression.Expression is MemberAccessExpressionSyntax memberAccessExpression &&
+                    memberAccessExpression.Expression is IdentifierNameSyntax identifierName &&
+                    identifierName.Identifier.Text == "Assert");
+
+            if (assertExpression == null)
+                return null;
+
+            var memberAccessExpression = (MemberAccessExpressionSyntax)assertExpression.Expression;
+            var assertionIdentifierName = (IdentifierNameSyntax)memberAccessExpression.Name;
+
+            if (assertionIdentifierName.Identifier.Text == "True")
+                return "true";
+
+            if (assertionIdentifierName.Identifier.Text == "False")
+                return "false";
+
+            if (assertionIdentifierName.Identifier.Text == "Equal")
+                return assertExpression.ArgumentList.Arguments[0].ToString();
+
+            if (assertionIdentifierName.Identifier.Text == "InRange")
+                return $">= {assertExpression.ArgumentList.Arguments[1]} && <= {assertExpression.ArgumentList.Arguments[2]}";
+
+            return default;
+        }
+
+        private static string ToCommand(SyntaxNode syntaxNode)
+        {
+            var methodDeclaration = (MethodDeclarationSyntax)syntaxNode;
+
+            var assertExpression = methodDeclaration.DescendantNodes()
+                .OfType<InvocationExpressionSyntax>()
+                .FirstOrDefault(invocationExpression => 
+                    invocationExpression.Expression is MemberAccessExpressionSyntax memberAccessExpression &&
+                    memberAccessExpression.Expression is IdentifierNameSyntax identifierName &&
+                    identifierName.Identifier.Text == "Assert");
+
+            if (assertExpression == null)
+                return null;
+
+            var memberAccessExpression = (MemberAccessExpressionSyntax)assertExpression.Expression;
+            var assertionIdentifierName = (IdentifierNameSyntax)memberAccessExpression.Name;
+
+            if (assertionIdentifierName.Identifier.Text == "True" ||
+                assertionIdentifierName.Identifier.Text == "False")
+            {
+                if (methodDeclaration.ExpressionBody != null ||
+                    methodDeclaration.Body.Statements.Count == 1)
+                    return assertExpression.ArgumentList.Arguments[0].ToString();
+
+                return ToCommandStatement(0).ToString();
+            }
+
+            if (assertionIdentifierName.Identifier.Text == "Equal")
+            {
+                if (methodDeclaration.ExpressionBody != null ||
+                    methodDeclaration.Body.Statements.Count == 1)
+                    return assertExpression.ArgumentList.Arguments[1].ToString();
+
+                return ToCommandStatement(1).ToString();
+            }
+
+            if (assertionIdentifierName.Identifier.Text == "InRange")
+            {
+                if (methodDeclaration.ExpressionBody != null ||
+                    methodDeclaration.Body.Statements.Count == 1)
+                    return assertExpression.ArgumentList.Arguments[2].ToString();
+
+                return ToCommandStatement(0).ToString();
+            }
+
+            return default;
+
+            SyntaxList<StatementSyntax> ToCommandStatement(int argumentIndex)
+            {
+                var commandExpression = methodDeclaration.Body
+                    .ReplaceNode(assertExpression, assertExpression.ArgumentList.Arguments[argumentIndex].Expression);
+
+                for (var i = 0; i < commandExpression.Statements.Count; i++)
+                {
+                    commandExpression = commandExpression.ReplaceNode(
+                        commandExpression.Statements[i], commandExpression.Statements[i].WithoutLeadingTrivia());
+                }
+
+                return commandExpression
+                    .Statements;
+            }
+        }
     }
 }

src/Exercism.TestRunner.CSharp/TestRunWriter.cs

@@ -27,6 +27,9 @@ private static JsonTestResult ToJsonTestResult(this TestResult testResult) =>
             new JsonTestResult
             {
                 Name = testResult.Name,
+                Test = testResult.Test,
+                Expected = testResult.Expected,
+                Command = testResult.Command,
                 Status = testResult.Status.ToString().ToLower(),
                 Message = testResult.Message.ToNullIfEmptyOrWhiteSpace(),
                 Output = testResult.Output.ToNullIfEmptyOrWhiteSpace()
@@ -48,6 +51,15 @@ private class JsonTestResult
             [JsonPropertyName("name")]
             public string Name { get; set; }
 
+            [JsonPropertyName("test")]
+            public string Test { get; set; }
+
+            [JsonPropertyName("expected")]
+            public string Expected { get; set; }
+
+            [JsonPropertyName("command")]
+            public string Command { get; set; }
+
             [JsonPropertyName("status")]
             public string Status { get; set; }
 

test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/Expected/Fake.cs

@@ -0,0 +1,25 @@
+public class Fake
+{
+    private readonly int x; 
+
+    public Fake(int x)
+    {
+        this.x = x;
+    }
+
+    public int Invert() => -x;
+
+    public bool Positive() => x >= 0;
+
+    public string Describe() => $"Number: {x}";
+
+    public void Foo()
+    {
+    }
+
+    public static int Invert(int x) => -x;
+
+    public static bool Positive(int x) => x >= 0;
+
+    public static string Describe(int x) => $"Number: {x}";
+}

test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/Expected/Fake.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="Example.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
+  </ItemGroup>
+
+</Project>

test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/Expected/FakeTest.cs

@@ -0,0 +1,100 @@
+using Xunit;
+
+public class FakeExpressionBodies
+{
+    [Fact]
+    public void Test_Using_Assert_Equal_In_Expression_Body_With_Expected_Int() =>
+        Assert.Equal(-5, Fake.Invert(5));
+
+    [Fact]
+    public void Test_Using_Assert_Equal_In_Expression_Body_With_Expected_String() =>
+        Assert.Equal("Number: 7", Fake.Describe(7));
+
+    [Fact]
+    public void Test_Using_Assert_In_Range_In_Expression_Body_With_Expected_Int() =>
+        Assert.InRange(Fake.Invert(-5), 3, 7);
+
+    [Fact]
+    public void Test_Using_Assert_True_In_Expression_Body() =>
+        Assert.True(Fake.Positive(1));
+
+    [Fact]
+    public void Test_Using_Assert_False_In_Expression_Body() =>
+        Assert.False(Fake.Positive(-2));
+}
+
+public class FakeSingleLineStatements
+{
+    [Fact]
+    public void Test_Using_Assert_Equal_In_Single_Line_Statement_With_Expected_Int()
+    {
+        Assert.Equal(-5, Fake.Invert(5));
+    }
+
+    [Fact]
+    public void Test_Using_Assert_Equal_In_Single_Line_Statement_With_Expected_String()
+    {
+        Assert.Equal("Number: 7", Fake.Describe(7));
+    }
+
+    [Fact]
+    public void Test_Using_Assert_In_Range_In_Single_line_With_Expected_Int()
+    {
+        Assert.InRange(Fake.Invert(-5), 3, 7);
+    }
+
+    [Fact]
+    public void Test_Using_Assert_True_In_Single_Line_Statement()
+    {
+        Assert.True(Fake.Positive(1));
+    }
+
+    [Fact]
+    public void Test_Using_Assert_False_In_Single_Line_Statement()
+    {
+        Assert.False(Fake.Positive(-2));
+    }
+}
+
+public class FakeMultiLineStatements
+{
+    [Fact]
+    public void Test_Using_Assert_Equal_In_Multi_Line_Statement_With_Expected_Int()
+    {
+        var fake = new Fake(3);
+        fake.Foo();
+        Assert.Equal(-3, fake.Invert());
+    }
+
+    [Fact]
+    public void Test_Using_Assert_Equal_In_Multi_Line_Statement_With_Expected_String()
+    {
+        var fake = new Fake(9);
+        fake.Foo();
+        Assert.Equal("Number: 9", fake.Describe());
+    }
+
+    [Fact]
+    public void Test_Using_Assert_In_Range_In_Single_line_With_Expected_Int()
+    {
+        var fake = new Fake(9);
+        fake.Foo();
+        Assert.InRange(fake.Invert(), 3, 7);
+    }
+
+    [Fact]
+    public void Test_Using_Assert_True_In_Multi_Line_Statement()
+    {
+        var fake = new Fake(6);
+        fake.Foo();
+        Assert.True(fake.Positive());
+    }
+
+    [Fact]
+    public void Test_Using_Assert_False_In_Multi_Line_Statement()
+    {
+        var fake = new Fake(-8);
+        fake.Foo();
+        Assert.False(fake.Positive());
+    }
+}

test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/Expected/expected_results.json

@@ -0,0 +1,5 @@
+{
+  "status": "error",
+  "message": "FakeTest.cs:75: Member \u0027Fake.Add(int, int)\u0027 cannot be accessed with an instance reference; qualify it with a type name instead\nFakeTest.cs:83: Member \u0027Fake.Add(int, int)\u0027 cannot be accessed with an instance reference; qualify it with a type name instead\nFakeTest.cs:91: Member \u0027Fake.Describe(int)\u0027 cannot be accessed with an instance reference; qualify it with a type name instead\nFakeTest.cs:99: Member \u0027Fake.Describe(int)\u0027 cannot be accessed with an instance reference; qualify it with a type name instead\nFakeTest.cs:107: Member \u0027Fake.Positive(int)\u0027 cannot be accessed with an instance reference; qualify it with a type name instead\nFakeTest.cs:115: Member \u0027Fake.Positive(int)\u0027 cannot be accessed with an instance reference; qualify it with a type name instead",
+  "tests": []
+}

test/Exercism.TestRunner.CSharp.IntegrationTests/Solutions/MultipleTestClassesWithAllPasses/expected_results.json

@@ -2,23 +2,33 @@
   "status": "pass",
   "tests": [
     {
-      "name": "FakeTest.Add_should_add_numbers",
+      "name": "Add should add numbers",
+      "test": "FakeTest.Add_should_add_numbers",
+      "expected": "2",
       "status": "pass"
     },
     {
-      "name": "FakeTest.Sub_should_subtract_numbers",
+      "name": "Sub should subtract numbers",
+      "test": "FakeTest.Sub_should_subtract_numbers",
+      "expected": "4",
       "status": "pass"
     },
     {
-      "name": "FakeTest.Mul_should_multiply_numbers",
+      "name": "Mul should multiply numbers",
+      "test": "FakeTest.Mul_should_multiply_numbers",
+      "expected": "6",
       "status": "pass"
     },
     {
-      "name": "FooTest.Upper_should_uppercase_string",
+      "name": "Upper should uppercase string",
+      "test": "FooTest.Upper_should_uppercase_string",
+      "expected": "\u0022HELLO\u0022",
       "status": "pass"
     },
     {
-      "name": "FooTest.Lower_should_lowercase_string",
+      "name": "Lower should lowercase string",
+      "test": "FooTest.Lower_should_lowercase_string",
+      "expected": "\u0022hello\u0022",
       "status": "pass"
     }
   ]