Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change package testing to use RuntimeTargets rather than RID-specific restore #53575

Merged
merged 9 commits into from
Jun 7, 2021
Merged

Change package testing to use RuntimeTargets rather than RID-specific restore #53575

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f8644d1
Change package testing to use RuntimeTargets rather than RID-specific…
ericstj Jun 1, 2021
2fcaefa
Merge remote-tracking branch 'upstream/main' into packageTestRuntimeT…
ericstj Jun 2, 2021
62eb871
Upload binlogs for package testing
ericstj Jun 2, 2021
ead401c
Address feedback
ericstj Jun 4, 2021
b3ed7fb
Cleanup old suppressions
ericstj Jun 4, 2021
990c9fd
Use ReferenceCopyLocalPaths for runtime testing
ericstj Jun 4, 2021
a0db9db
Don't include runtime dependencies when reference was excluded by con…
ericstj Jun 6, 2021
6e98ff6
Fix import conventions in outer build
ericstj Jun 7, 2021
6c60a8f
Workaround conflict resolution on .NETStandard runtime
ericstj Jun 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions src/libraries/pkg/test/frameworkSettings/net5.0/settings.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
<Project>
<PropertyGroup>
<RuntimeFrameworkVersion>$(MicrosoftNETCoreAppVersion)</RuntimeFrameworkVersion>
</PropertyGroup>
<ItemGroup>
<KnownFrameworkReference Remove="Microsoft.AspNetCore.App" />
<KnownFrameworkReference Remove="Microsoft.WindowsDesktop.App" />
Expand Down
4 changes: 0 additions & 4 deletions src/libraries/pkg/test/frameworkSettings/netcoreapp/settings.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@
<PropertyGroup>
<ShouldVerifyClosure>true</ShouldVerifyClosure>
<ShouldVerifyTypes>true</ShouldVerifyTypes>

<!-- ensure that we get runtime assemblies in ReferenceCopyLocalPaths -->
<SelfContained>true</SelfContained>
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
</PropertyGroup>

<ItemGroup>
Expand Down
32 changes: 0 additions & 32 deletions src/libraries/pkg/test/frameworkSettings/netcoreapp1.0/settings.targets
View file
Open in desktop

This file was deleted.

34 changes: 0 additions & 34 deletions src/libraries/pkg/test/frameworkSettings/netcoreapp1.1/settings.targets
View file
Open in desktop

This file was deleted.

8 changes: 0 additions & 8 deletions src/libraries/pkg/test/frameworkSettings/netcoreapp2.0/settings.targets
View file
Open in desktop

This file was deleted.

101 changes: 50 additions & 51 deletions src/libraries/pkg/test/packageTest.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
<Project>

<PropertyGroup>
<PropertyGroup Condition="'$(TargetFramework)' != ''">
<_targetFrameworkVersionIndex>$(TargetFramework.IndexOfAny(".-0123456789"))</_targetFrameworkVersionIndex>
<_targetFrameworkIdentifier Condition="'$(_runtimeOSVersionIndex)' != '-1'">$(TargetFramework.SubString(0, $(_targetFrameworkVersionIndex)))</_targetFrameworkIdentifier>
<!-- For net5.0 and later use netcoreapp folder. -->
<_targetFrameworkIdentifier Condition="'$(_targetFrameworkIdentifier)' == 'net' and !$(TargetFramework.StartsWith('net4'))">netcoreapp</_targetFrameworkIdentifier>
</PropertyGroup>

<PropertyGroup>
<ToolsDir>$(MSBuildThisFileDirectory)tools\</ToolsDir>

<RestoreSources>
Expand All @@ -31,17 +33,14 @@
<IgnoredTypes Include="System.Collections.Generic.CollectionExtensions" />
</ItemGroup>


<Target Name="LogBeginTest">
<PropertyGroup>
<_message>Testing $(TestPackageID) TFM=$(TargetFramework)</_message>
<_message Condition="'$(RuntimeIdentifier)' != ''">$(_message) RID=$(RuntimeIdentifier)</_message>
</PropertyGroup>
<Message Importance="High" Text="$(_message)" />
<Message Importance="High" Text="Testing $(TestPackageID) TFM=$(TargetFramework)" />
</Target>

<Target Name="VerifyReferenceClosure"
DependsOnTargets="ResolveReferences"
Condition="'$(ShouldVerifyClosure)' == 'true' and '$(RuntimeIdentifier)' == ''">
Condition="'$(ShouldVerifyClosure)' == 'true'">
<ItemGroup>
<_refClosureFileNames Include="@(ReferencePath->'%(FileName)')">
<Original>%(Identity)</Original>
Expand All @@ -57,7 +56,7 @@

<Target Name="VerifyReferenceTypes"
DependsOnTargets="ResolveReferences"
Condition="'$(ShouldVerifyTypes)' == 'true' and '$(RuntimeIdentifier)' == ''">
Condition="'$(ShouldVerifyTypes)' == 'true'">
<ItemGroup>
<_refTypesFileNames Include="@(ReferencePath->'%(FileName)')">
<Original>%(Identity)</Original>
Expand All @@ -71,11 +70,36 @@
IgnoredTypes="@(IgnoredTypes)" />
</Target>

<Target Name="PrepareForRuntimeTesting" DependsOnTargets="ResolveReferences">
<ItemGroup>
<RuntimeLibToTest Include="@(RuntimeCopyLocalItems)" Condition="'%(RuntimeCopyLocalItems.NuGetPackageId)' == '$(TestPackageId)'" RuntimeIdentifier="none" />
<RuntimeLibToTest Include="@(RuntimeTargetsCopyLocalItems)" Condition="'%(RuntimeTargetsCopyLocalItems.NuGetPackageId)' == '$(TestPackageId)' AND '%(RuntimeTargetsCopyLocalItems.CopyLocal)' == 'true'" />
</ItemGroup>

<ItemGroup>
<RuntimeLibToTestDependency Include="@(ReferencePath)" Condition="'%(ReferencePath.NuGetPackageId)' != '$(TestPackageId)'" />

<!-- Some dependent packages may be excluded from compile, consider these as candiates as well.
We assume they must expose a RID agnostic asset, thus we only check RuntimeCopyLocalItems. -->
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To be more correct / replicate the actual runtime behavior here we would need to replicate the host selection logic to test with the right runtime assets for dependencies. It's not so interesting to do this type of testing since the SDK doesn't make it possible for a library to reference a runtime specific asset of another library or project, and we don't typically do this with our custom config system. It's bad practice for library to depend on implementation-only publics of another library; if one library needed it why wouldn't a customer library need it?

So the asset analysis ends up looking like this:

  1. Package under test: Foreach runtime asset group
  2. Package dependencies w/ exclude=comple: RID-less runtime asset group
  3. Package dependencies w/o exclude=compile: compile asset group
  4. Framework: ref-pack / reference assemblies only.

Note that 2-4 are changes from what we previously did, but these are "safe" changes, since if these validations succeed then API compat will ensure that transitively things should succeed.

<RuntimeCopyLocalItems Original="%(Identity)" />
<_runtimeCopyLocalItemsPackages Include="@(RuntimeCopyLocalItems->'%(NuGetPackageId)')"
Exclude="@(ReferencePath->'%(NuGetPackageId)');$(TestPackageId)" />
<RuntimeLibToTestDependency Include="@(_runtimeCopyLocalItemsPackages->'%(Original)')" />
</ItemGroup>

<Error Condition="'@(RuntimeLibToTest)' == ''" Text="Could not locate any runtime items from Package $(TestPackageID)" />

<Message Importance="High" Text="Testing $(TestPackageID) TFM=$(TargetFramework) RIDs=@(RuntimeLibToTest->'%(RuntimeIdentifier)'->Distinct())" />
</Target>

<Target Name="VerifyRuntimeClosure"
DependsOnTargets="ResolveReferences"
Condition="'$(ShouldVerifyClosure)' == 'true' and '$(RuntimeIdentifier)' != '' and '$(SkipVerifyClosureForRuntime)' != 'true'">
Inputs="%(RuntimeLibToTest.RuntimeIdentifier)"
Outputs="unused"
DependsOnTargets="PrepareForRuntimeTesting"
Condition="'$(ShouldVerifyClosure)' == 'true' and '$(SkipVerifyClosureForRuntime)' != 'true'">
<Message Importance="High" Text="Testing closure for $(TestPackageID) TFM=$(TargetFramework) RID=%(RuntimeLibToTest.RuntimeIdentifier)" />
<ItemGroup>
<_runClosureFileNames Include="@(ReferenceCopyLocalPaths->'%(FileName)')">
<_runClosureFileNames Include="@(RuntimeLibToTestDependency->'%(FileName)');@(RuntimeLibToTest->'%(FileName)')">
<Original>%(Identity)</Original>
</_runClosureFileNames>
<_runClosureFileNamesFiltered Include="@(_runClosureFileNames)" Exclude="@(ExcludeFromClosure);@(ExcludeFromClosure->'%(Identity).ni')"/>
Expand All @@ -88,10 +112,13 @@
</Target>

<Target Name="VerifyRuntimeTypes"
DependsOnTargets="ResolveReferences"
Condition="'$(ShouldVerifyTypes)' == 'true' and '$(RuntimeIdentifier)' != ''">
Inputs="%(RuntimeLibToTest.RuntimeIdentifier)"
Outputs="unused"
DependsOnTargets="PrepareForRuntimeTesting"
Condition="'$(ShouldVerifyTypes)' == 'true'">
<Message Importance="High" Text="Testing for duplicate types for for $(TestPackageID) TFM=$(TargetFramework) RID=%(RuntimeLibToTest.RuntimeIdentifier)" />
<ItemGroup>
<_runTypesFileNames Include="@(ReferenceCopyLocalPaths->'%(FileName)')">
<_runTypesFileNames Include="@(RuntimeLibToTestDependency->'%(FileName)');@(RuntimeLibToTest->'%(FileName)')">
<Original>%(Identity)</Original>
</_runTypesFileNames>
<_runTypesFileNamesFiltered Include="@(_runTypesFileNames)" Exclude="@(ExcludeFromTypes);@(ExcludeFromTypes->'%(Identity).ni')"/>
Expand All @@ -105,55 +132,27 @@

<Target Name="VerifyNotDependsOnNetStandard"
DependsOnTargets="ResolveReferences"
Condition="'$(_ShortFrameworkIdentifier)' == 'net' AND '$(_TargetFrameworkVersionWithoutV)' &gt;= '4.6.1' AND '$(RuntimeIdentifier)' == '' AND '$(DisableVerifyNotDependsOnNetStandardTest)' != 'true'">
Condition="'$(_ShortFrameworkIdentifier)' == 'net' AND '$(_TargetFrameworkVersionWithoutV)' &gt;= '4.6.1' AND '$(DisableVerifyNotDependsOnNetStandardTest)' != 'true'">
<Error Condition="'$(DependsOnNetStandard)' == 'true'" Text="Package $(TestPackageId) requires netstandard shims when targeting $(TargetFramework)" />
</Target>

<PropertyGroup>
<_testDependsOn>
<TestDependsOn Condition="'$(TargetFramework)' != ''">
LogBeginTest;
VerifyNotDependsOnNetStandard;
VerifyReferenceClosure;
VerifyReferenceTypes;
VerifyRuntimeClosure;
VerifyRuntimeTypes;
</_testDependsOn>
</TestDependsOn>
<TestDependsOn Condition="'$(TargetFramework)' == ''">
ericstj marked this conversation as resolved.
Show resolved Hide resolved
_ComputeTargetFrameworkItems
ericstj marked this conversation as resolved.
Show resolved Hide resolved
</TestDependsOn>
</PropertyGroup>

<!-- inner target to be called by outer Test Target -->
<Target Name="_test"
DependsOnTargets="$(_testDependsOn)">
<MakeDir Directories="$(IntermediateOutputPath)" />
<Touch AlwaysCreate="true" Files="$(IntermediateOutputPath)\$(RuntimeIdentifier).testComplete" />
</Target>

<Target Name="_getTestProjects">
<ItemGroup>
<_projectRuntime Include="$(RuntimeIdentifiers)" />

<!-- Run _test once without RID, then for each RID -->
<TestProject Include="$(MSBuildProjectFullPath)">
<Semaphore>$(IntermediateOutputPath)\.testComplete</Semaphore>
</TestProject>
<TestProject Include="$(MSBuildProjectFullPath)" Condition="'%(_projectRuntime.Identity)' != ''">
<Semaphore>$(IntermediateOutputPath)\%(_projectRuntime.Identity).testComplete</Semaphore>
<AdditionalProperties>RuntimeIdentifier=%(_projectRuntime.Identity)</AdditionalProperties>
</TestProject>
</ItemGroup>
</Target>


<PropertyGroup>
<TestDependsOn>_getTestProjects</TestDependsOn>
<!-- Runs ResolveReferences once up front in order to ensure we don't run it for the first time concurrently during testing.
This is needed to avoid sharing violations when RAR and ResolvePackageAssets write out state files. -->
<TestDependsOn Condition="'$(ShouldVerifyClosure)' == 'true' or '$(ShouldVerifyTypes)' == 'true'">$(TestDependsOn);ResolveReferences</TestDependsOn>
</PropertyGroup>
<!-- Runs all tests scenarios for this project -->
<Target Name="Test"
DependsOnTargets="$(TestDependsOn)"
Inputs="@(TestProject);$(MSBuildAllProjects);$(ProjectAssetsFile)"
Outputs="@(TestProject->'%(Semaphore)')">
<MSBuild Projects="@(TestProject)" Targets="_test" BuildInParallel="true" />
DependsOnTargets="$(TestDependsOn)">
<MSBuild Projects="@(_InnerBuildProjects)" Targets="Test" BuildInParallel="true" />
</Target>
</Project>
3 changes: 1 addition & 2 deletions src/libraries/pkg/test/project.csproj.template
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>{TFM}</TargetFramework>
<RuntimeIdentifiers>{RIDs}</RuntimeIdentifiers>
<TargetFrameworks>{TFMs}</TargetFrameworks>
<TestPackageId>{PackageId}</TestPackageId>
<TestPackageVersion>{PackageVersion}</TestPackageVersion>
<!-- Turn off end of life target framework checks as we intentionally build older .NETCoreApp configurations. -->
Expand Down
26 changes: 4 additions & 22 deletions src/libraries/pkg/test/testPackages.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@

<ItemGroup>
<SupportedPackage>
<ProjectDir>$(TestProjectDir)%(Identity)/%(TargetFrameworkShort)/</ProjectDir>
<ProjectDir>$(TestProjectDir)%(Identity)/</ProjectDir>
</SupportedPackage>
<SupportedPackage>
<ProjectFile>%(ProjectDir)project.csproj</ProjectFile>
Expand All @@ -114,23 +114,6 @@
</ItemGroup>
</Target>

<Target Name="UpdateRuntimeIdentifiers"
AfterTargets="GetSupportedPackages"
Inputs="%(SupportedPackage.Identity);%(SupportedPackage.TargetFrameworkShort)"
Outputs="unused">
<ItemGroup>
<_supportedPackageRuntimes Remove="@(_supportedPackageRuntimes)" />
<_supportedPackageRuntimes Include="%(SupportedPackage.RuntimeIdentifiers)" />

<_supportedPackageRuntimesToRemove Include="@(_supportedPackageRuntimes)" Exclude="@(RuntimesToInclude)" Condition="'@(RuntimesToInclude)' != ''" />
<_filteredSupportedPackageRuntimes Include="@(_supportedPackageRuntimes)" Exclude="@(RuntimesToExclude);@(_supportedPackageRuntimesToRemove)" />

<SupportedPackage>
<RuntimeIdentifiers>@(_filteredSupportedPackageRuntimes)</RuntimeIdentifiers>
</SupportedPackage>
</ItemGroup>
</Target>

<Target Name="GenerateProjects"
DependsOnTargets="GetSupportedPackages;CreateTestDir"
Inputs="@(TestPackagesPath);$(ProjectTemplate)"
Expand All @@ -140,15 +123,14 @@
<_projectFile>%(SupportedPackage.ProjectFile)</_projectFile>
<_packageId>%(SupportedPackage.Identity)</_packageId>
<_packageVersion>%(SupportedPackage.Version)</_packageVersion>
<_projectTFM>%(SupportedPackage.TargetFrameworkShort)</_projectTFM>
<_projectRIDs>%(SupportedPackage.RuntimeIdentifiers)</_projectRIDs>
<_projectTFMs>@(SupportedPackage->'%(TargetFrameworkShort)')</_projectTFMs>
ericstj marked this conversation as resolved.
Show resolved Hide resolved
</PropertyGroup>

<MakeDir Directories="$(_projectDir)" />
<!-- Update project.json template -->
<!-- Update project.csproj template -->
<WriteLinesToFile
File="$(_projectFile)"
Lines="$([System.IO.File]::ReadAllText('$(ProjectTemplate)').Replace('{PackageId}', $(_packageId)).Replace('{PackageVersion}', $(_packageVersion)).Replace('{TFM}', $(_projectTFM)).Replace('{RIDs}', '$(_projectRIDs)'))"
Lines="$([System.IO.File]::ReadAllText('$(ProjectTemplate)').Replace('{PackageId}', $(_packageId)).Replace('{PackageVersion}', $(_packageVersion)).Replace('{TFMs}', $(_projectTFMs)))"
Overwrite="true" />
<Message Text="Generated $(_projectFile)" />
</Target>
Expand Down