Prototype: integrating with ILVerify and classifying differences

NuGet.Config

@@ -29,5 +29,8 @@
     <add key="dotnet.myget.org roslyn" value="https://dotnet.myget.org/F/roslyn/api/v3/index.json" />
     <add key="myget.org vs-editor" value="https://myget.org/F/vs-editor/api/v3/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+
+<add key="desktop" value="C:\Users\jcouv\Desktop\" />
+
   </packageSources>
 </configuration>

src/Compilers/CSharp/Test/Emit/Attributes/AttributeTests.cs

@@ -6296,7 +6296,7 @@ class X: Attribute
 {
 }
 ";
-            CompileAndVerify(source5, additionalRefs: new[] { comp1, comp2 });
+            CompileAndVerify(source5, additionalRefs: new[] { comp1, comp2 }, verify: Verification.ClassLoadGeneral);
 
             // Multiple from PE, multiple from Source
             var source6 = @"

src/Compilers/CSharp/Test/Emit/Attributes/AttributeTests_Assembly.cs

@@ -297,7 +297,7 @@ public void MismatchedSurrogateInAssemblyCultureAttribute()
             string s = @"[assembly: System.Reflection.AssemblyCultureAttribute(""\uD800"")]";
             var comp = CreateStandardCompilation(s, options: TestOptions.ReleaseDll);
 
-            CompileAndVerify(comp, verify: Verification.Fails, symbolValidator: m =>
+            CompileAndVerify(comp, verify: Verification.InvalidLocale, symbolValidator: m =>
             {
                 var utf8 = new System.Text.UTF8Encoding(false, false);
                 Assert.Equal(utf8.GetString(utf8.GetBytes("\uD800")), m.ContainingAssembly.Identity.CultureName);
@@ -655,7 +655,7 @@ void M(Test x) {}
 }
 ", options: TestOptions.ReleaseDll, references: new[] { MscorlibRef_v4_0_30316_17626, hash_module });
 
-            CompileAndVerify(compilation, verify: Verification.Fails,
+            CompileAndVerify(compilation, verify: Verification.Skipped,
                 manifestResources: hash_resources,
                 validator: (peAssembly) =>
                 {
@@ -684,7 +684,7 @@ void M(Test x) {}
 }
 ", options: TestOptions.ReleaseDll, references: new[] { MscorlibRef_v4_0_30316_17626, hash_module });
 
-            CompileAndVerify(compilation, verify: Verification.Fails,
+            CompileAndVerify(compilation, verify: Verification.Skipped,
                 manifestResources: hash_resources,
                 validator: (peAssembly) =>
                 {
@@ -717,7 +717,7 @@ void M(Test x) {}
 }
 ", options: TestOptions.ReleaseDll, references: new[] { MscorlibRef_v4_0_30316_17626, hash_module });
 
-            CompileAndVerify(compilation, verify: Verification.Fails,
+            CompileAndVerify(compilation, verify: Verification.Skipped,
                 manifestResources: hash_resources,
                 validator: (peAssembly) =>
                 {

src/Compilers/CSharp/Test/Emit/Attributes/AttributeTests_IsByRefLike.cs

@@ -959,7 +959,7 @@ public ref struct NotTypedReference { }
 }";
             var compilation1 = CreateCompilation(source1, assemblyName: GetUniqueName());
 
-            CompileAndVerify(compilation1, verify: Verification.Fails, symbolValidator: module =>
+            CompileAndVerify(compilation1, verify: Verification.FailsPeVerify, symbolValidator: module =>
             {
                 var type = module.ContainingAssembly.GetTypeByMetadataName("System.TypedReference");
                 AssertReferencedIsByRefLikeAttributes(Accessibility.Internal, type, module.ContainingAssembly.Name, hasObsolete: false);

src/Compilers/CSharp/Test/Emit/Attributes/AttributeTests_StructLayout.cs

@@ -211,10 +211,10 @@ class Structs
             };
 
             CompileAndVerify(verifiable, assemblyValidator: validator);
-            CompileAndVerify(unverifiable, assemblyValidator: validator, verify: Verification.Fails);
+            CompileAndVerify(unverifiable, assemblyValidator: validator, verify: Verification.ClassLayout);
 
             // CLR limitation on type size, not a RefEmit bug:
-            CompileAndVerify(unloadable, assemblyValidator: validator, verify: Verification.Fails);
+            CompileAndVerify(unloadable, assemblyValidator: validator, verify: Verification.ClassLayout);
         }
 
         [Fact]
@@ -332,7 +332,7 @@ public class C : B
 }
 ";
             // type C can't be loaded
-            CompileAndVerify(source, verify: Verification.Fails);
+            CompileAndVerify(source, verify: Verification.TypeLoadFailed);
         }
 
         [Fact]

src/Compilers/CSharp/Test/Emit/Attributes/AttributeTests_WellKnownAttributes.cs

@@ -3539,7 +3539,7 @@ public static int Main ()
 
             // the resulting code does not need to verify
             // This is consistent with Dev10 behavior
-            CompileAndVerify(source, options: TestOptions.ReleaseDll, verify: Verification.Fails, sourceSymbolValidator: sourceValidator, symbolValidator: metadataValidator);
+            CompileAndVerify(source, options: TestOptions.ReleaseDll, verify: Verification.UnableToResolveToken, sourceSymbolValidator: sourceValidator, symbolValidator: metadataValidator);
         }
 
         [Fact, WorkItem(544507, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/544507")]
@@ -5341,7 +5341,7 @@ public static void Main() {}
             // Dev10 Runtime Exception:
             // Unhandled Exception: System.TypeLoadException: Windows Runtime types can only be declared in Windows Runtime assemblies.
 
-            var verifier = CompileAndVerify(source, sourceSymbolValidator: sourceValidator, symbolValidator: metadataValidator, verify: Verification.Fails);
+            var verifier = CompileAndVerify(source, sourceSymbolValidator: sourceValidator, symbolValidator: metadataValidator, verify: Verification.TypeLoadFailed);
         }
 
         #endregion

src/Compilers/CSharp/Test/Emit/Attributes/InternalsVisibleToAndStrongNameTests.cs

@@ -1086,7 +1086,7 @@ public static void Main()
         .WithCryptoKeyFile(SigningTestHelpers.MaxSizeKeyFile)
         .WithStrongNameProvider(s_defaultDesktopProvider));
 
-            CompileAndVerify(comp2, expectedOutput: "Called M");
+            CompileAndVerify(comp2, expectedOutput: "Called M", verify: Verification.Passes);
             Assert.Equal(TestResources.General.snMaxSizePublicKey, comp2.Assembly.Identity.PublicKey);
             Assert.Equal<byte>(pubKeyTokenBytes, comp2.Assembly.Identity.PublicKeyToken);
 
@@ -1097,7 +1097,7 @@ public static void Main()
         .WithCryptoKeyFile(SigningTestHelpers.MaxSizeKeyFile)
         .WithStrongNameProvider(s_defaultDesktopProvider));
 
-            CompileAndVerify(comp3, expectedOutput: "Called M");
+            CompileAndVerify(comp3, expectedOutput: "Called M", verify: Verification.Passes);
             Assert.Equal(TestResources.General.snMaxSizePublicKey, comp3.Assembly.Identity.PublicKey);
             Assert.Equal<byte>(pubKeyTokenBytes, comp3.Assembly.Identity.PublicKeyToken);
         }
@@ -2391,7 +2391,7 @@ static void Main()
             CompileAndVerify(ca);
 
             var cb = CreateStandardCompilation(sourceB, options: TestOptions.ReleaseExe, assemblyName: "X", references: new[] { new CSharpCompilationReference(ca) });
-            CompileAndVerify(cb, expectedOutput: "42").Diagnostics.Verify();
+            CompileAndVerify(cb, expectedOutput: "42", verify: Verification.Passes).Diagnostics.Verify();
         }
 
         [Fact, WorkItem(1072339, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/1072339")]
@@ -2417,7 +2417,7 @@ static void Main()
             CompileAndVerify(ca);
 
             var cb = CreateStandardCompilation(sourceB, options: TestOptions.ReleaseExe, assemblyName: "X", references: new[] { new CSharpCompilationReference(ca) });
-            CompileAndVerify(cb, expectedOutput: "42").Diagnostics.Verify();
+            CompileAndVerify(cb, expectedOutput: "42", verify: Verification.IVT).Diagnostics.Verify();
         }
 
         [Fact, WorkItem(1095618, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/1095618")]

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenAsyncEHTests.cs

@@ -21,10 +21,10 @@ public CodeGenAsyncEHTests()
         {
         }
 
-        private CompilationVerifier CompileAndVerify(string source, string expectedOutput = null, IEnumerable<MetadataReference> references = null, CSharpCompilationOptions options = null)
+        private CompilationVerifier CompileAndVerify(string source, string expectedOutput = null, IEnumerable<MetadataReference> references = null, CSharpCompilationOptions options = null, Verification verify = Verification.Passes)
         {
             references = (references != null) ? references.Concat(s_asyncRefs) : s_asyncRefs;
-            return base.CompileAndVerify(source, expectedOutput: expectedOutput, additionalRefs: references, options: options);
+            return base.CompileAndVerify(source, expectedOutput: expectedOutput, additionalRefs: references, options: options, verify: verify);
         }
 
         [Fact]
@@ -936,7 +936,8 @@ public static void Main()
             var expected = @"
 2
 ";
-            var v = CompileAndVerify(source, s_asyncRefs, options: TestOptions.DebugExe.WithMetadataImportOptions(MetadataImportOptions.All), expectedOutput: expected, symbolValidator: module =>
+            var v = CompileAndVerify(source, s_asyncRefs, options: TestOptions.DebugExe.WithMetadataImportOptions(MetadataImportOptions.All),
+                expectedOutput: expected, symbolValidator: module =>
             {
                 Assert.Equal(new[]
                 {
@@ -1841,7 +1842,7 @@ public static void Main()
 hello
 2
 ";
-            CompileAndVerify(source, expectedOutput: expected);
+            CompileAndVerify(source, expectedOutput: expected, verify: Verification.LeaveIntoTry);
         }
 
         [Fact]

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenAsyncLocalsTests.cs

@@ -21,10 +21,10 @@ public CodeGenAsyncLocalsTests()
         {
         }
 
-        private CompilationVerifier CompileAndVerify(string source, string expectedOutput = null, IEnumerable<MetadataReference> references = null, CSharpCompilationOptions options = null)
+        private CompilationVerifier CompileAndVerify(string source, string expectedOutput = null, IEnumerable<MetadataReference> references = null, CSharpCompilationOptions options = null, Verification verify = Verification.Passes)
         {
             references = (references != null) ? references.Concat(s_asyncRefs) : s_asyncRefs;
-            return base.CompileAndVerify(source, expectedOutput: expectedOutput, additionalRefs: references, options: options);
+            return base.CompileAndVerify(source, expectedOutput: expectedOutput, additionalRefs: references, options: options, verify: verify);
         }
 
         private string GetFieldLoadsAndStores(CompilationVerifier c, string qualifiedMethodName)
@@ -231,7 +231,8 @@ public static async Task M(int x, int y, int z)
         y = 1;
     }
 }";
-            CompileAndVerify(source, additionalRefs: s_asyncRefs, options: TestOptions.ReleaseDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
+            CompileAndVerify(source, additionalRefs: s_asyncRefs,
+                options: TestOptions.ReleaseDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
             {
                 AssertEx.Equal(new[]
                 {
@@ -244,7 +245,8 @@ public static async Task M(int x, int y, int z)
                 }, module.GetFieldNames("C.<M>d__1"));
             });
 
-            CompileAndVerify(source, additionalRefs: s_asyncRefs, options: TestOptions.DebugDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
+            CompileAndVerify(source, additionalRefs: s_asyncRefs,
+                options: TestOptions.DebugDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
             {
                 AssertEx.Equal(new[]
                 {
@@ -283,7 +285,8 @@ public async Task M(IDisposable disposable)
         lock (this) { }
     }
 }";
-            CompileAndVerify(source, s_asyncRefs, options: TestOptions.ReleaseDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
+            CompileAndVerify(source, s_asyncRefs,
+                options: TestOptions.ReleaseDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
             {
                 AssertEx.Equal(new[]
                 {
@@ -301,7 +304,8 @@ public async Task M(IDisposable disposable)
                 }, module.GetFieldNames("C.<M>d__3"));
             });
 
-            var vd = CompileAndVerify(source, s_asyncRefs, options: TestOptions.DebugDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
+            var vd = CompileAndVerify(source, s_asyncRefs,
+                options: TestOptions.DebugDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
             {
                 AssertEx.Equal(new[]
                 {
@@ -1239,7 +1243,8 @@ public async Task F()
         var b8 = await Task.FromResult(default(Tuple<long, long, long>));
     }
 }";
-            CompileAndVerify(source, additionalRefs: s_asyncRefs, options: TestOptions.ReleaseDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
+            CompileAndVerify(source, additionalRefs: s_asyncRefs,
+                options: TestOptions.ReleaseDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
             {
                 AssertEx.Equal(new[]
                 {

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenAsyncSpillTests.cs

@@ -21,10 +21,10 @@ public CodeGenAsyncSpillTests()
         {
         }
 
-        private CompilationVerifier CompileAndVerify(string source, string expectedOutput = null, IEnumerable<MetadataReference> references = null, CSharpCompilationOptions options = null)
+        private CompilationVerifier CompileAndVerify(string source, string expectedOutput = null, IEnumerable<MetadataReference> references = null, CSharpCompilationOptions options = null, Verification verify = Verification.Passes)
         {
             references = (references != null) ? references.Concat(s_asyncRefs) : s_asyncRefs;
-            return base.CompileAndVerify(source, expectedOutput: expectedOutput, additionalRefs: references, options: options);
+            return base.CompileAndVerify(source, expectedOutput: expectedOutput, additionalRefs: references, options: options, verify: verify);
         }
 
         [Fact]
@@ -967,7 +967,7 @@ public static async Task<int> F(int[] array)
                 }, module.GetFieldNames("C.<F>d__3"));
             });
 
-            CompileAndVerify(source, additionalRefs: s_asyncRefs, verify: Verification.Passes, options: TestOptions.DebugDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
+            CompileAndVerify(source, additionalRefs: s_asyncRefs, options: TestOptions.DebugDll.WithMetadataImportOptions(MetadataImportOptions.All), symbolValidator: module =>
              {
                  AssertEx.Equal(new[]
                  {
@@ -2490,7 +2490,7 @@ public static void Main()
 2
 0
 ";
-            CompileAndVerify(source, expectedOutput: expected);
+            CompileAndVerify(source, expectedOutput: expected, verify: Verification.RuntimeArgumentHandle);
         }
 
         [Fact]

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenDeconstructTests.cs

@@ -92,11 +92,11 @@ public void Deconstruct(out int a, out string b)
                 var lhs = tree.GetRoot().DescendantNodes().OfType<TupleExpressionSyntax>().First();
                 Assert.Equal(@"(x, y)", lhs.ToString());
                 Assert.Equal("(System.Int64 x, System.String y)", model.GetTypeInfo(lhs).Type.ToTestDisplayString());
+                Assert.Equal("(System.Int64 x, System.String y)", model.GetTypeInfo(lhs).ConvertedType.ToTestDisplayString());
 
                 var right = tree.GetRoot().DescendantNodes().OfType<ObjectCreationExpressionSyntax>().Single();
                 Assert.Equal(@"new C()", right.ToString());
                 Assert.Equal("C", model.GetTypeInfo(right).Type.ToTestDisplayString());
-                Assert.Equal("C", model.GetTypeInfo(right).ConvertedType.ToTestDisplayString());
                 Assert.Equal(ConversionKind.Identity, model.GetConversion(right).Kind);
             };
 

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenDynamicTests.cs

@@ -28,16 +28,17 @@ private CompilationVerifier CompileAndVerifyIL(
             bool allowUnsafe = false,
             [CallerFilePath]string callerPath = null,
             [CallerLineNumber]int callerLine = 0,
-            CSharpParseOptions parseOptions = null)
+            CSharpParseOptions parseOptions = null,
+            Verification verify = Verification.Passes)
         {
             references = references ?? new[] { SystemCoreRef, CSharpRef };
 
             // verify that we emit correct optimized and unoptimized IL:
             var unoptimizedCompilation = CreateCompilationWithMscorlib45(source, references, parseOptions: parseOptions, options: TestOptions.DebugDll.WithMetadataImportOptions(MetadataImportOptions.All).WithAllowUnsafe(allowUnsafe));
             var optimizedCompilation = CreateCompilationWithMscorlib45(source, references, parseOptions: parseOptions, options: TestOptions.ReleaseDll.WithMetadataImportOptions(MetadataImportOptions.All).WithAllowUnsafe(allowUnsafe));
 
-            var unoptimizedVerifier = CompileAndVerify(unoptimizedCompilation);
-            var optimizedVerifier = CompileAndVerify(optimizedCompilation);
+            var unoptimizedVerifier = CompileAndVerify(unoptimizedCompilation, verify: verify);
+            var optimizedVerifier = CompileAndVerify(optimizedCompilation, verify: verify);
 
             // check what IL we emit exactly:
             if (expectedUnoptimizedIL != null)
@@ -9550,7 +9551,7 @@ .locals init (int V_0, //a
   IL_0066:  callvirt   ""void <>A{00000002}<System.Runtime.CompilerServices.CallSite, int, object>.Invoke(System.Runtime.CompilerServices.CallSite, ref int, object)""
   IL_006b:  ret
 }
-");
+", verify: Verification.TypedReference);
         }
 
         [Fact]

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenIncrementTests.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Globalization;
 using Microsoft.CodeAnalysis.CSharp.Test.Utilities;
+using Microsoft.CodeAnalysis.Test.Utilities;
 using Roslyn.Test.Utilities;
 using Xunit;
 
@@ -1028,7 +1029,7 @@ public static void Main()
     }
 }
 ";
-            base.CompileAndVerify(source, expectedOutput: "12").
+            base.CompileAndVerify(source, verify: Verification.TypedReference, expectedOutput: "12").
                 VerifyIL("Test.Main",
 @"
 {

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenIterators.cs

@@ -2382,7 +2382,7 @@ public System.Collections.IEnumerable SomeNumbers()
 }";
             // The compilation succeeds even though CompilerGeneratedAttribute and DebuggerNonUserCodeAttribute are not available.
             var compilation = CreateCompilation(new[] { Parse(source), Parse(corlib) });
-            var verifier = CompileAndVerify(compilation, verify: Verification.Fails);
+            var verifier = CompileAndVerify(compilation, verify: Verification.TypeLoadFailed);
             verifier.VerifyDiagnostics(
                 // warning CS8021: No value for RuntimeMetadataVersion found. No assembly containing System.Object was found nor was a value for RuntimeMetadataVersion specified through options.
                 Diagnostic(ErrorCode.WRN_NoRuntimeMetadataVersion).WithLocation(1, 1));

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenOverridingAndHiding.cs

@@ -2452,7 +2452,7 @@ public static void Main()
             // from assembly 'Dev10, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null'
             // is overriding a method that is not visible from that assembly.
 
-            CompileAndVerify(outerCompilation, verify: Verification.Fails).VerifyIL("Test.Main", @"
+            CompileAndVerify(outerCompilation, verify: Verification.UnableToResolveToken).VerifyIL("Test.Main", @"
 {
   // Code size       65 (0x41)
   .maxstack  4

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenScriptTests.cs

@@ -552,7 +552,7 @@ public void SubmissionEntryPoint()
                 "s0.dll",
                 SyntaxFactory.ParseSyntaxTree(source0, options: TestOptions.Script),
                 references);
-            var verifier = CompileAndVerify(s0, verify: Verification.Fails);
+            var verifier = CompileAndVerify(s0, verify: Verification.BadName);
             var methodData = verifier.TestData.GetMethodData("<Initialize>");
             Assert.Equal("System.Threading.Tasks.Task<object>", methodData.Method.ReturnType.ToDisplayString());
             methodData.VerifyIL(

src/Compilers/CSharp/Test/Emit/CodeGen/CodeGenStructsAndEnum.cs

@@ -1277,7 +1277,7 @@ public static void Main()
 }
 ";
 
-            var compilation = CompileAndVerify(source, expectedOutput: "11");
+            var compilation = CompileAndVerify(source, expectedOutput: "11", verify: Verification.InvalidProgramVararg);
 
             compilation.VerifyIL("D.Main",
 @"