[tests] Remove the last set of ignored MSBuild tests (#8246)

Fixes: #7777

The last set of ignored test categories have been removed or migrated to
existing tests:

  * Test configurations related to AotMode=Hybrid and AotMode=Full have been removed.

  * BuildTest.CheckSequencePointGeneration has been removed due to it relying on mono symbol output.

  * EnvironmentContentTests.CheckBuildIdIsUnique has been removed due to it relying on mono symbol output.

  * PackagingTest.CheckManagedSymbolsArchive has been removed due to it relying on mono symbol output.

  * InstallAndRunTests.MonoSymbolicateAndroidStackTrace has been removed due to it relying on mono symbol output.

  * InstallAndRunTests.MonoSymbolicateNetStandardStackTrace has been removed due to it relying on mono symbol output.

  * EmbeddedDSOTests.BinariesExist removed as it is implicitly covered by tests that validate .apk contents.

  * EmbeddedDSOTests.EnvironmentFileContents removed as it is covered by BuildTest.CheckAssemblyCounts.

  * EmbeddedDSOTests.DSOPageAlignment assert moved to PackagingTest.EmbeddedDSOs.

  * EmbeddedDSOTests.DSOCompressionMode and EmbeddedDSOTests.AndroidManifestHasFlag removed as they are covered by PackagingTest.EmbeddedDSOs.

build-tools/automation/azure-pipelines.yaml

@@ -273,7 +273,7 @@ stages:
       parameters:
         testRunTitle: Xamarin.Android.Build.Tests - Linux .NET 6 Smoke Tests
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
-        dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)"
+        dotNetTestExtraArgs: --filter "TestCategory = SmokeTests"
         testResultsFile: TestResult-NETSmokeMSBuildTests-Linux-$(XA.Build.Configuration).xml
 
     - template: yaml-templates/upload-results.yaml

build-tools/automation/yaml-templates/build-windows.yaml

@@ -96,7 +96,7 @@ stages:
         testRunTitle: Smoke MSBuild Tests - Windows Dotnet Build
         testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\$(DotNetStableTargetFramework)\Xamarin.Android.Build.Tests.dll
         testResultsFile: TestResult-SmokeMSBuildTests-WinDotnetBuild-$(XA.Build.Configuration).xml
-        dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)"
+        dotNetTestExtraArgs: --filter "TestCategory = SmokeTests"
 
     - template: upload-results.yaml
       parameters:

build-tools/automation/yaml-templates/run-msbuild-tests.yaml

@@ -1,9 +1,9 @@
 parameters:
-  testOS:                 # 'macOS' or 'Windows'
-  jobName:                # Name of the job
-  jobDisplayName:         # Display name of the job
-  agentCount:             # Number of build agents to run in parallel
-  testFilter:             # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
+  testOS: ''              # 'macOS' or 'Windows'
+  jobName: ''             # Name of the job
+  jobDisplayName: ''      # Display name of the job
+  agentCount: 1           # Number of build agents to run in parallel
+  testFilter: ''          # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
   xaSourcePath: $(System.DefaultWorkingDirectory)
   repositoryAlias: 'self'
   commit: ''
@@ -48,7 +48,7 @@ jobs:
   - template: run-sliced-nunit-tests.yaml
     parameters:
       testAssembly: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
-      testFilter: ${{ parameters.testFilter }} $(ExcludedNUnitCategories)
+      testFilter: ${{ parameters.testFilter }}
       testRunTitle: Xamarin.Android.Build.Tests - ${{ parameters.testOS }}
       retryFailedTests: false
       xaSourcePath: ${{ parameters.xaSourcePath }}

build-tools/automation/yaml-templates/run-sliced-nunit-tests.yaml

@@ -1,20 +1,28 @@
 parameters:
-  testAssembly:           # NUnit test assembly to run
-  testFilter:             # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
-  testRunTitle:           # Title of the test run
+  testAssembly: ''        # NUnit test assembly to run
+  testFilter: ''          # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
+  testRunTitle: ''        # Title of the test run
   xaSourcePath: $(System.DefaultWorkingDirectory)
   retryFailedTests: true  # Retry failed tests once
 
 steps:
-- pwsh: |
-    dotnet-test-slicer `
-      slice `
-      --test-assembly="${{ parameters.testAssembly }}" `
-      --test-filter="${{ parameters.testFilter }}" `
-      --slice-number=$(System.JobPositionInPhase) `
-      --total-slices=$(System.TotalJobsInPhase) `
+- ${{if parameters.testFilter}}:
+  - pwsh: >-
+      dotnet-test-slicer slice
+      --test-assembly="${{ parameters.testAssembly }}"
+      --test-filter="${{ parameters.testFilter }}"
+      --slice-number=$(System.JobPositionInPhase)
+      --total-slices=$(System.TotalJobsInPhase)
       --outfile="${{ parameters.testAssembly }}.runsettings"
-  displayName: Slice unit tests
+    displayName: Slice unit tests with filter
+- ${{ else }}:
+  - pwsh: >-
+      dotnet-test-slicer slice
+      --test-assembly="${{ parameters.testAssembly }}"
+      --slice-number=$(System.JobPositionInPhase)
+      --total-slices=$(System.TotalJobsInPhase)
+      --outfile="${{ parameters.testAssembly }}.runsettings"
+    displayName: Slice unit tests
 
 - ${{ if eq(parameters.retryFailedTests, 'false') }}:
   # If we aren't using auto-retry logic, then this is just a simple template call

build-tools/automation/yaml-templates/stage-msbuild-emulator-tests.yaml

@@ -57,7 +57,7 @@ stages:
     - template: run-sliced-nunit-tests.yaml
       parameters:
         testAssembly: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
-        testFilter: cat != TimeZoneInfo & cat != Localization $(ExcludedNUnitCategories)
+        testFilter: $(ExcludedNightlyNUnitCategories)
         testRunTitle: MSBuildDeviceIntegration On Device - macOS
 
     - ${{ if ne(parameters.usesCleanImages, true) }}:

build-tools/automation/yaml-templates/stage-msbuild-tests.yaml

@@ -22,7 +22,6 @@ stages:
       jobName: mac_msbuild_tests
       jobDisplayName: macOS > Tests > MSBuild
       agentCount: 10
-      testFilter: cat != Dummy  # This is because $(ExcludedNUnitCategories) gets appended which starts with '&'
       xaSourcePath: ${{ parameters.xaSourcePath }}
       repositoryAlias: ${{ parameters.repositoryAlias }}
       commit: ${{ parameters.commit }}
@@ -34,7 +33,6 @@ stages:
       jobName: win_msbuild_tests
       jobDisplayName: Windows > Tests > MSBuild
       agentCount: 6
-      testFilter: cat != Dummy  # This is because $(ExcludedNUnitCategories) gets appended which starts with '&'
       xaSourcePath: ${{ parameters.xaSourcePath }}
       repositoryAlias: ${{ parameters.repositoryAlias }}
       commit: ${{ parameters.commit }}

build-tools/automation/yaml-templates/variables.yaml

@@ -48,9 +48,7 @@ variables:
   value: false
 - name: IsRelOrTargetingRel
   value: $[or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['System.PullRequest.TargetBranch'], 'release/'))]
-- name: DotNetNUnitCategories
-  value: '& TestCategory != DotNetIgnore & TestCategory != HybridAOT & TestCategory != MkBundle & TestCategory != MonoSymbolicate & TestCategory != StaticProject & TestCategory != SystemApplication'
 - name: DefaultTestSdkPlatforms  # Comma-separated SDK Platform(s) to install on test agents (no spaces)
   value: 33,34
-- name: ExcludedNUnitCategories
-  value: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != StaticProject & cat != SystemApplication'
+- name: ExcludedNightlyNUnitCategories
+  value: 'cat != SystemApplication & cat != TimeZoneInfo & cat != Localization'

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs

@@ -363,76 +363,20 @@ public static void Foo () {
 			sb.Dispose ();
 		}
 
-		[Test]
-		[Category ("HybridAOT")]
-		public void HybridAOT ([Values ("armeabi-v7a;arm64-v8a", "armeabi-v7a", "arm64-v8a")] string abis)
-		{
-			// There's no point in testing all of the ABIs with and without assembly blobs, let's test just one of them this way
-			bool usesAssemblyBlobs = String.Compare ("arm64-v8a", abis, StringComparison.Ordinal) == 0;
-			var proj = new XamarinAndroidApplicationProject () {
-				IsRelease = true,
-				AotAssemblies = true,
-			};
-			proj.SetProperty ("AndroidAotMode", "Hybrid");
-			// So we can use Mono.Cecil to open assemblies directly
-			proj.SetProperty ("AndroidEnableAssemblyCompression", "False");
-			proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ());
-			proj.SetAndroidSupportedAbis (abis);
-
-			using (var b = CreateApkBuilder ()) {
-
-				if (abis == "armeabi-v7a") {
-					proj.SetProperty ("_AndroidAotModeValidateAbi", "False");
-					b.Build (proj);
-					proj.SetProperty ("_AndroidAotModeValidateAbi", () => null);
-				}
-
-				if (abis.Contains ("armeabi-v7a")) {
-					b.ThrowOnBuildFailure = false;
-					Assert.IsFalse (b.Build (proj), "Build should have failed.");
-					string error = b.LastBuildOutput
-							.SkipWhile (x => !x.StartsWith ("Build FAILED.", StringComparison.Ordinal))
-							.FirstOrDefault (x => x.Contains ("error XA1025:"));
-					Assert.IsNotNull (error, "Build should have failed with XA1025.");
-					return;
-				}
-
-				b.Build (proj);
-
-				var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk");
-				FileAssert.Exists (apk);
-				var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs);
-				Assert.IsTrue (helper.Exists ($"assemblies/{proj.ProjectName}.dll"), $"{proj.ProjectName}.dll should exist in apk!");
-
-				using (var stream = helper.ReadEntry ($"assemblies/{proj.ProjectName}.dll")) {
-					stream.Position = 0;
-					using (var assembly = AssemblyDefinition.ReadAssembly (stream)) {
-						var type = assembly.MainModule.GetType ($"{proj.ProjectName}.MainActivity");
-						var method = type.Methods.First (m => m.Name == "OnCreate");
-						Assert.LessOrEqual (method.Body.Instructions.Count, 1, "OnCreate should have stripped method bodies!");
-					}
-				}
-			}
-		}
-
 		[Test]
 		[Category ("LLVM")]
-		public void NoSymbolsArgShouldReduceAppSize ([Values ("", "Hybrid")] string androidAotMode, [Values (false, true)] bool skipDebugSymbols)
+		public void NoSymbolsArgShouldReduceAppSize ([Values (false, true)] bool skipDebugSymbols)
 		{
 			if (IsWindows)
 				Assert.Ignore ("https://github.com/dotnet/runtime/issues/88625");
 
-			AssertAotModeSupported (androidAotMode);
-
 			var proj = new XamarinAndroidApplicationProject () {
 				IsRelease = true,
 				AotAssemblies = true,
 			};
 			var supportedAbi = "arm64-v8a";
 			proj.SetAndroidSupportedAbis (supportedAbi);
 			proj.SetProperty ("EnableLLVM", true.ToString ());
-			if (!string.IsNullOrEmpty (androidAotMode))
-				proj.SetProperty ("AndroidAotMode", androidAotMode);
 
 			var xaAssemblySize = 0;
 			var xaAssemblySizeNoSymbol = 0;

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs

@@ -148,65 +148,6 @@ public partial class BuildTest : BaseTest
 				/* expectedResult */   CommercialBuildAvailable ? "debug" : "release",
 			},
 		};
-
-		static object [] SequencePointChecks () => new object [] {
-			new object[] {
-				/* isRelease */          true,
-				/* monoSymbolArchive */  true ,
-				/* aotAssemblies */      false,
-				/* debugSymbols */       true,
-				/* expectedRuntime */    "release",
-				/* usesAssemblyBlobs */  false,
-			},
-			new object[] {
-				/* isRelease */          true,
-				/* monoSymbolArchive */  true ,
-				/* aotAssemblies */      true,
-				/* debugSymbols */       true,
-				/* expectedRuntime */    "release",
-				/* usesAssemblyBlobs */  false,
-			},
-			new object[] {
-				/* isRelease */          true,
-				/* monoSymbolArchive */  false ,
-				/* aotAssemblies */      false,
-				/* debugSymbols */       true,
-				/* expectedRuntime */    "release",
-				/* usesAssemblyBlobs */  false,
-			},
-			new object[] {
-				/* isRelease */          true,
-				/* monoSymbolArchive */  false ,
-				/* aotAssemblies */      false,
-				/* debugSymbols */       true,
-				/* expectedRuntime */    "release",
-				/* usesAssemblyBlobs */  true,
-			},
-			new object[] {
-				/* isRelease */          true,
-				/* monoSymbolArchive */  false ,
-				/* aotAssemblies */      true,
-				/* debugSymbols */       false,
-				/* expectedRuntime */    "release",
-				/* usesAssemblyBlobs */  false,
-			},
-			new object[] {
-				/* isRelease */          true,
-				/* monoSymbolArchive */  true ,
-				/* aotAssemblies */      true,
-				/* debugSymbols */       false,
-				/* expectedRuntime */    "release",
-				/* usesAssemblyBlobs */  false,
-			},
-			new object[] {
-				/* isRelease */          true,
-				/* monoSymbolArchive */  true ,
-				/* aotAssemblies */      true,
-				/* debugSymbols */       false,
-				/* expectedRuntime */    "release",
-				/* usesAssemblyBlobs */  true,
-			},
-		};
 #pragma warning restore 414
 	}
 }

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs

@@ -355,64 +355,6 @@ public void CheckWhichRuntimeIsIncluded (string supportedAbi, bool debugSymbols,
 			}
 		}
 
-		[Test]
-		[Category ("AOT"), Category ("MonoSymbolicate")]
-		[TestCaseSource (nameof (SequencePointChecks))]
-		public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive, bool aotAssemblies,
-		       bool debugSymbols, string expectedRuntime, bool usesAssemblyBlobs)
-		{
-			var proj = new XamarinAndroidApplicationProject () {
-				IsRelease = isRelease,
-				AotAssemblies = aotAssemblies
-			};
-			var abis = new [] { "armeabi-v7a", "x86" };
-			proj.SetAndroidSupportedAbis (abis);
-			proj.SetProperty (proj.ActiveConfigurationProperties, "MonoSymbolArchive", monoSymbolArchive);
-			proj.SetProperty (proj.ActiveConfigurationProperties, "DebugSymbols", debugSymbols);
-			proj.SetProperty (proj.ActiveConfigurationProperties, "AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ());
-			using (var b = CreateApkBuilder ()) {
-				Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
-				var apk = Path.Combine (Root, b.ProjectDirectory,
-					proj.OutputPath, $"{proj.PackageName}-Signed.apk");
-				var msymarchive = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, proj.PackageName + ".apk.mSYM");
-				var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs);
-				if (aotAssemblies) {
-					foreach (var abi in abis) {
-						var assemblies = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath,
-						                               "aot", abi, "libaot-UnnamedProject.dll.so");
-						var shouldExist = monoSymbolArchive && debugSymbols;
-						var symbolicateFile = Directory.GetFiles (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath,
-						                                                        "aot", abi), "UnnamedProject.dll.msym", SearchOption.AllDirectories).FirstOrDefault ();
-						if (shouldExist)
-							Assert.IsNotNull (symbolicateFile, "UnnamedProject.dll.msym should exist");
-						else
-							Assert.IsNull (symbolicateFile, "{0} should not exist", symbolicateFile);
-						if (shouldExist) {
-							var foundMsyms = Directory.GetFiles (Path.Combine (msymarchive), "UnnamedProject.dll.msym", SearchOption.AllDirectories).Any ();
-							Assert.IsTrue (foundMsyms, "UnnamedProject.dll.msym should exist in the archive {0}", msymarchive);
-						}
-						Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi);
-						Assert.IsTrue (helper.Exists ($"lib/{abi}/libaot-UnnamedProject.dll.so"),
-						               $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi);
-						Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"),
-						               $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk");
-					}
-				}
-				var runtimeInfo = b.GetSupportedRuntimes ();
-				foreach (var abi in abis) {
-					var runtime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Runtime == expectedRuntime);
-					Assert.IsNotNull (runtime, "Could not find the expected runtime.");
-					var inApk = ZipHelper.ReadFileFromZip (apk, String.Format ("lib/{0}/{1}", abi, runtime.Name));
-					var inApkRuntime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Size == inApk.Length);
-					Assert.IsNotNull (inApkRuntime, "Could not find the actual runtime used.");
-					Assert.AreEqual (runtime.Size, inApkRuntime.Size, "expected {0} got {1}", expectedRuntime, inApkRuntime.Runtime);
-				}
-
-				b.Clean (proj);
-				Assert.IsTrue (!Directory.Exists (msymarchive), "{0} should have been deleted on Clean", msymarchive);
-			}
-		}
-
 		[Test]
 		public void CheckItemMetadata ([Values (true, false)] bool isRelease)
 		{