From d0929da4fc781e54c2a792495a942c8d4423a681 Mon Sep 17 00:00:00 2001 From: vpatakottu <47004464+vpatakottu@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:17:45 -0700 Subject: [PATCH] Add E2E tests for Microsoft.Sbom.Targets project (#658) * add base setup for tests * updates to test * cleanup * Add more tests * update package version * cleanup * mini fix for copying sample project * add unloading step * create separate project for E2E tests * cleanup * rearrange method * cleanup * check for platform * try with locator * disable analyzers for sample project --------- Co-authored-by: vpatakottu --- Directory.Packages.props | 4 +- Microsoft.Sbom.sln | 6 + .../Microsoft.Sbom.Targets.csproj | 6 +- .../Microsoft.Sbom.Targets.targets | 32 +- .../GenerateSbomE2ETests.cs | 344 ++++++++++++++++++ .../Microsoft.Sbom.Targets.E2E.Tests.csproj | 63 ++++ .../ProjectSample1/ProjectSample1.csproj | 21 ++ .../ProjectSample1/SampleLibrary.cs | 7 + .../Microsoft.Sbom.Targets.Tests.csproj | 2 +- .../Utility/GeneratedSbomValidator.cs | 2 +- 10 files changed, 463 insertions(+), 24 deletions(-) create mode 100644 test/Microsoft.Sbom.Targets.E2E.Tests/GenerateSbomE2ETests.cs create mode 100644 test/Microsoft.Sbom.Targets.E2E.Tests/Microsoft.Sbom.Targets.E2E.Tests.csproj create mode 100644 test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/ProjectSample1.csproj create mode 100644 test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/SampleLibrary.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index d4f4fded..acb06ed8 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -14,7 +14,9 @@ + + @@ -58,4 +60,4 @@ - \ No newline at end of file + diff --git a/Microsoft.Sbom.sln b/Microsoft.Sbom.sln index 2aa73d19..643283e9 100644 --- a/Microsoft.Sbom.sln +++ b/Microsoft.Sbom.sln @@ -55,6 +55,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Targets.Test EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Tool.Tests", "test\Microsoft.Sbom.Tool.Tests\Microsoft.Sbom.Tool.Tests.csproj", "{FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Sbom.Targets.E2E.Tests", "test\Microsoft.Sbom.Targets.E2E.Tests\Microsoft.Sbom.Targets.E2E.Tests.csproj", "{3FDE7800-F61F-4C45-93AB-648A4C7979C7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -125,6 +127,10 @@ Global {FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Debug|Any CPU.Build.0 = Debug|Any CPU {FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Release|Any CPU.ActiveCfg = Release|Any CPU {FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Release|Any CPU.Build.0 = Release|Any CPU + {3FDE7800-F61F-4C45-93AB-648A4C7979C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3FDE7800-F61F-4C45-93AB-648A4C7979C7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3FDE7800-F61F-4C45-93AB-648A4C7979C7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3FDE7800-F61F-4C45-93AB-648A4C7979C7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.csproj b/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.csproj index c7202f9b..50aca5cb 100644 --- a/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.csproj +++ b/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.csproj @@ -2,7 +2,7 @@ Microsoft.Sbom.Targets - net8.0;net472 + net6.0;net8.0;net472 win-x64;osx-x64;linux-x64 true true @@ -68,8 +68,8 @@ - - + + diff --git a/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.targets b/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.targets index 370a269c..99f43739 100644 --- a/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.targets +++ b/src/Microsoft.Sbom.Targets/Microsoft.Sbom.Targets.targets @@ -1,19 +1,10 @@ - - - - net472 net8.0 - $([System.IO.Path]::Combine($(MSBuildThisFileDirectory),..,tasks,$(GenerateSbom_TFM),sbom-tool)) - $([System.IO.Path]::Combine($(MSBuildThisFileDirectory),..,tasks,$(GenerateSbom_TFM),Microsoft.Sbom.Targets.dll)) + $([System.IO.Path]::Combine($(MSBuildThisFileDirectory),..,tasks,$(GenerateSbom_TFM),sbom-tool)) + $([System.IO.Path]::Combine($(MSBuildThisFileDirectory),..,tasks,$(GenerateSbom_TFM),Microsoft.Sbom.Targets.dll)) $(SbomToolBinaryOutputPath) @@ -22,7 +13,7 @@ - + false @@ -40,24 +31,29 @@ SPDX:2.2 true $([System.Guid]::NewGuid()) + $([System.String]::Copy('$(UnzipGuid)').Substring(0, 8)) - - + $([System.IO.Path]::GetFullPath('$(PackageOutputPath)')) + - $([System.IO.Path]::Combine($(PackageOutputPath), $(PackageId).$(PackageVersion).nupkg)) + $([System.IO.Path]::Combine($(PackageOutputFullPath), $(PackageId).$(PackageVersion).nupkg)) - $([System.IO.Path]::Combine($(PackageOutputPath), $(PackageId).$(PackageVersion).$(UnzipGuid).temp)) + $([System.IO.Path]::Combine($(PackageOutputFullPath), $(PackageId).$(PackageVersion).$(ShortUnzipGuidFolder).temp)) - + diff --git a/test/Microsoft.Sbom.Targets.E2E.Tests/GenerateSbomE2ETests.cs b/test/Microsoft.Sbom.Targets.E2E.Tests/GenerateSbomE2ETests.cs new file mode 100644 index 00000000..e717718e --- /dev/null +++ b/test/Microsoft.Sbom.Targets.E2E.Tests/GenerateSbomE2ETests.cs @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Targets.E2E.Tests; + +using System; +using System.IO; +using System.IO.Compression; +using System.Runtime.InteropServices; +using Castle.Core.Internal; +using Microsoft.Build.Evaluation; +using Microsoft.Build.Locator; +using Microsoft.Build.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class GenerateSbomE2ETests +{ + /* + * The following tests validate the end-to-end workflow for importing the Microsoft.Sbom.Targets.targets + * into a .NET project, building it, packing it, and validating the generated SBOM contents. + */ + private static readonly bool IsWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + + private static string projectDirectory = Path.Combine(Directory.GetCurrentDirectory(), "ProjectSamples", "ProjectSample1"); + private static string sbomToolPath = Path.Combine(Directory.GetCurrentDirectory(), "sbom-tool"); + private static string generateSbomTaskPath = Path.Combine(Directory.GetCurrentDirectory(), "Microsoft.Sbom.Targets.dll"); + + private static string sbomSpecificationName = "SPDX"; + private static string sbomSpecificationVersion = "2.2"; + private static string sbomSpecificationDirectoryName = $"{sbomSpecificationName}_{sbomSpecificationVersion}".ToLowerInvariant(); + private string manifestPath; + private string expectedPackageName; + private string expectedVersion; + private string expectedSupplier; + private string assemblyName; + private string expectedNamespace; + private string configuration; + + [TestInitialize] + public void SetupLocator() + { + if (MSBuildLocator.CanRegister) + { + MSBuildLocator.RegisterDefaults(); + } + } + + [TestCleanup] + public void CleanOutputFolders() + { + var binDir = Path.Combine(projectDirectory, "bin"); + var objDir = Path.Combine(projectDirectory, "obj"); + + try + { + if (Directory.Exists(binDir)) + { + Directory.Delete(binDir, true); + } + + if (Directory.Exists(objDir)) + { + Directory.Delete(objDir, true); + } + + ProjectCollection.GlobalProjectCollection.UnloadAllProjects(); + } + catch (Exception ex) + { + Assert.Fail($"Failed to cleanup output directories. {ex}"); + } + } + + private Project SetupSampleProject() + { + // Create a Project object for ProjectSample1 + var projectFile = Path.Combine(projectDirectory, "ProjectSample1.csproj"); + var sampleProject = new Project(projectFile); + + // Get all the expected default properties + SetDefaultProperties(sampleProject); + + // Set the TargetFrameworks property to empty. By default, it sets this property to net6.0 and net8.0, which fails for net8.0 builds. + sampleProject.SetProperty("TargetFrameworks", string.Empty); + + // Set the paths to the sbom-tool CLI tool and Microsoft.Sbom.Targets.dll + sampleProject.SetProperty("SbomToolBinaryOutputPath", sbomToolPath); + sampleProject.SetProperty("GenerateSbomTaskAssemblyFilePath", generateSbomTaskPath); + + return sampleProject; + } + + private void SetDefaultProperties(Project sampleProject) + { + expectedPackageName = sampleProject.GetPropertyValue("PackageId"); + expectedVersion = sampleProject.GetPropertyValue("Version"); + assemblyName = sampleProject.GetPropertyValue("AssemblyName"); + configuration = sampleProject.GetPropertyValue("Configuration"); + + if (expectedPackageName.IsNullOrEmpty()) + { + expectedPackageName = assemblyName; + } + + if (expectedVersion.IsNullOrEmpty()) + { + expectedVersion = "1.0.0"; + } + } + + private void RestoreBuildPack(Project sampleProject) + { + var logger = new ConsoleLogger(); + + // Restore the project to create project.assets.json file + var restore = sampleProject.Build("Restore", new[] { logger }); + Assert.IsTrue(restore, "Failed to restore the project"); + + // Next, build the project + var build = sampleProject.Build(logger); + Assert.IsTrue(build, "Failed to build the project"); + + // Finally, pack the project + var pack = sampleProject.Build("Pack", new[] { logger }); + Assert.IsTrue(pack, "Failed to pack the project"); + } + + private void ExtractPackage() + { + // Unzip the contents of the NuGet package + var nupkgPath = Path.Combine(projectDirectory, "bin", configuration); + var nupkgFile = Path.Combine(nupkgPath, $"{expectedPackageName}.{expectedVersion}.nupkg"); + var zipFile = Path.Combine(nupkgPath, $"{expectedPackageName}.{expectedVersion}.zip"); + var extractPath = Path.Combine(projectDirectory, "bin", configuration, $"{Guid.NewGuid()}.temp"); + + // Rename the .nupkg file to .zip + File.Copy(nupkgFile, zipFile, true); + + // Extract the .zip file + ZipFile.ExtractToDirectory(zipFile, extractPath); + + manifestPath = Path.Combine(extractPath, "_manifest", sbomSpecificationDirectoryName, "manifest.spdx.json"); + } + + [TestMethod] + public void SbomGenerationSucceedsForDefaultProperties() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + // Create and setup a Project object for ProjectSample1 + var sampleProject = SetupSampleProject(); + + // Restore, build, and pack the project + RestoreBuildPack(sampleProject); + + // Extract the NuGet package + ExtractPackage(); + + // Validate the SBOM exists in the package. + Assert.IsTrue(File.Exists(manifestPath)); + } + + [TestMethod] + public void SbomGenerationSucceedsForValidNamespaceBaseUriUniquePart() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + // Create and setup a Project object for ProjectSample1 + var sampleProject = SetupSampleProject(); + + // Manually set the NamespaceUriUniquePart + var namespaceUriUniquePart = Guid.NewGuid().ToString(); + sampleProject.SetProperty("SbomGenerationNamespaceUriUniquePart", namespaceUriUniquePart); + + // Restore, build, and pack the project + RestoreBuildPack(sampleProject); + + // Extract the NuGet package + ExtractPackage(); + + // Validate the SBOM exists in the package. + Assert.IsTrue(File.Exists(manifestPath)); + } + + [TestMethod] + public void SbomGenerationSucceedsForValidRequiredParams() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + // Create and setup a Project object for ProjectSample1 + var sampleProject = SetupSampleProject(); + + // Set require params + expectedPackageName = "SampleName"; + expectedVersion = "3.2.5"; + expectedSupplier = "SampleSupplier"; + expectedNamespace = "https://example.com"; + + sampleProject.SetProperty("PackageId", expectedPackageName); + sampleProject.SetProperty("Version", expectedVersion); + sampleProject.SetProperty("SbomGenerationPackageName", expectedPackageName); + sampleProject.SetProperty("SbomGenerationPackageVersion", expectedVersion); + sampleProject.SetProperty("SbomGenerationPackageSupplier", expectedSupplier); + sampleProject.SetProperty("SbomGenerationNamespaceBaseUri", expectedNamespace); + + // Restore, build, and pack the project + RestoreBuildPack(sampleProject); + + // Extract the NuGet package + ExtractPackage(); + + // Validate the SBOM exists in the package. + Assert.IsTrue(File.Exists(manifestPath)); + } + + [TestMethod] + public void SbomGenerationFailsForInvalidNamespaceUri() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + // Create and setup a Project object for ProjectSample1 + var sampleProject = SetupSampleProject(); + + // Set invalid namespace + expectedNamespace = "incorrect_uri"; + sampleProject.SetProperty("SbomGenerationNamespaceBaseUri", expectedNamespace); + + // Restore, build, and pack the project + var logger = new ConsoleLogger(); + + // Restore the project to create project.assets.json file + var restore = sampleProject.Build("Restore", new[] { logger }); + Assert.IsTrue(restore, "Failed to restore the project"); + + // Next, build the project + var build = sampleProject.Build(logger); + Assert.IsTrue(build, "Failed to build the project"); + + // Ensure the packing step fails + var pack = sampleProject.Build("Pack", new[] { logger }); + Assert.IsFalse(pack, "Packing succeeded when it should have failed"); + } + + [TestMethod] + public void SbomGenerationFailsForInvalidSupplierName() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + // Create and setup a Project object for ProjectSample1 + var sampleProject = SetupSampleProject(); + + // Set invalid supplier name + sampleProject.SetProperty("Authors", string.Empty); + sampleProject.SetProperty("AssemblyName", string.Empty); + sampleProject.SetProperty("SbomGenerationPackageSupplier", string.Empty); + + // Restore, build, and pack the project + var logger = new ConsoleLogger(); + + // Restore the project to create project.assets.json file + var restore = sampleProject.Build("Restore", new[] { logger }); + Assert.IsTrue(restore, "Failed to restore the project"); + + // Next, build the project + var build = sampleProject.Build(logger); + Assert.IsTrue(build, "Failed to build the project"); + + // Ensure the packing step fails + var pack = sampleProject.Build("Pack", new[] { logger }); + Assert.IsFalse(pack, "Packing succeeded when it should have failed"); + } + + [TestMethod] + public void SbomGenerationSkipsForUnsetGenerateSBOMFlag() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + // Create and setup a Project object for ProjectSample1 + var sampleProject = SetupSampleProject(); + + // Set the GenerateSBOM property to empty. + sampleProject.SetProperty("GenerateSBOM", "false"); + + // Restore, build, and pack the project + RestoreBuildPack(sampleProject); + + // Extract the NuGet package + ExtractPackage(); + + // Ensure the manifest file was not created + Assert.IsTrue(!File.Exists(manifestPath)); + } + + [TestMethod] + public void SbomGenerationSucceedsForMultiTargetedProject() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + // Create and setup a Project object for ProjectSample1 + var sampleProject = SetupSampleProject(); + + // Set multi-target frameworks + sampleProject.SetProperty("TargetFramework", string.Empty); + sampleProject.SetProperty("TargetFrameworks", "net472;net6.0"); + + // Restore, build, and pack the project + RestoreBuildPack(sampleProject); + + // Extract the NuGet package + ExtractPackage(); + + // Validate the SBOM exists in the package. + Assert.IsTrue(File.Exists(manifestPath)); + } +} diff --git a/test/Microsoft.Sbom.Targets.E2E.Tests/Microsoft.Sbom.Targets.E2E.Tests.csproj b/test/Microsoft.Sbom.Targets.E2E.Tests/Microsoft.Sbom.Targets.E2E.Tests.csproj new file mode 100644 index 00000000..88e02cc9 --- /dev/null +++ b/test/Microsoft.Sbom.Targets.E2E.Tests/Microsoft.Sbom.Targets.E2E.Tests.csproj @@ -0,0 +1,63 @@ + + + + net6.0;net472 + true + false + True + Microsoft.Sbom.Targets.E2E.Tests + net6.0 + $(MSBuildThisFileDirectory)..\..\src\Microsoft.Sbom.Tool\ + $(MSBuildThisFileDirectory)..\..\src\Microsoft.Sbom.Targets\Microsoft.Sbom.Targets.targets + + + + TRACE + + + + + + + + + + + + + + + + + + + + + + + + + + + <_SbomToolFiles Include="$(SBOMCLIToolProjectDir)bin\$(Configuration)\$(SbomCLIToolTargetFramework)\publish\**\*.*"> + false + + + + + + + + + + false + + + + + + + + + + diff --git a/test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/ProjectSample1.csproj b/test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/ProjectSample1.csproj new file mode 100644 index 00000000..ab349408 --- /dev/null +++ b/test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/ProjectSample1.csproj @@ -0,0 +1,21 @@ + + + Library + true + net6.0 + ProjectSample + 1.2.4 + false + true + true + false + + + + + $(NoWarn);NU1507;NU5128 + + + + + diff --git a/test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/SampleLibrary.cs b/test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/SampleLibrary.cs new file mode 100644 index 00000000..05389171 --- /dev/null +++ b/test/Microsoft.Sbom.Targets.E2E.Tests/ProjectSamples/ProjectSample1/SampleLibrary.cs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; + +public class SampleLibrary +{ +} diff --git a/test/Microsoft.Sbom.Targets.Tests/Microsoft.Sbom.Targets.Tests.csproj b/test/Microsoft.Sbom.Targets.Tests/Microsoft.Sbom.Targets.Tests.csproj index 73da9053..4c43dd2d 100644 --- a/test/Microsoft.Sbom.Targets.Tests/Microsoft.Sbom.Targets.Tests.csproj +++ b/test/Microsoft.Sbom.Targets.Tests/Microsoft.Sbom.Targets.Tests.csproj @@ -39,4 +39,4 @@ - + \ No newline at end of file diff --git a/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs b/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs index 1ee8c094..895c0a51 100644 --- a/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs +++ b/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs @@ -66,7 +66,7 @@ internal void AssertSbomIsValid(string manifestPath, string buildDropPath, strin Assert.IsNotNull(packagesValue); if (string.IsNullOrEmpty(buildComponentPath)) { - Assert.IsTrue(packagesValue.Count == 1); + Assert.IsTrue(packagesValue.Count == 1, $"Expected 1 package but actual value was {packagesValue.Count}"); } else {