Skip to content

Commit

Permalink
Re-partition MSBuild test groups (dotnet#22885)
Browse files Browse the repository at this point in the history 
This changes a tests partitioning scheme of coreclr tests in several different ways:

1. Make the test groups are of equal (to some extent) sizes keeping the number of tests in one group less than 1000;

2. As a resulf of 1) this increases a number of groups needed for Pri0 tests to 3 and decreases a number of groups for Pri1 tests to 10;

3. This also changes a way of tests partitioning scheme is defined - instead of explicitly specifying MSBuild Include-patterns this defines a boundaries between test groups in a form of _StartsWith ItemGroup. Then I use StringComparer.Ordinal to pick tests that belong to a particular group.
  • Loading branch information
@echesakov
echesakov authored Feb 28, 2019
1 parent 0625973 commit 39b643f
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 204 deletions.
11 changes: 5 additions & 6 deletions build-test.cmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,19 +296,18 @@ REM See https://github.com/Microsoft/msbuild/issues/2993

set __SkipPackageRestore=false
set __SkipTargetingPackBuild=false
set __BuildLoopCount=2
set __TestGroupToBuild=1
set __NumberOfTestGroups=3

if %__Priority% GTR 0 (set __BuildLoopCount=16&set __TestGroupToBuild=2)
echo %__MsgPrefix%Building tests group %__TestGroupToBuild% with %__BuildLoopCount% subgroups
if %__Priority% GTR 0 (set __NumberOfTestGroups=10)
echo %__MsgPrefix%Building tests divided into %__NumberOfTestGroups% test groups

for /l %%G in (1, 1, %__BuildLoopCount%) do (
for /l %%G in (1, 1, %__NumberOfTestGroups%) do (

set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%";Append=!__AppendToLog!
set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%";Append=!__AppendToLog!
set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%";Append=!__AppendToLog!

set TestBuildSlice=%%G
set __TestGroupToBuild=%%G
echo Running: msbuild %__ProjectDir%\tests\build.proj !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! %TargetsWindowsMsbuildArg% %__msbuildArgs% !__PriorityArg! %__UnprocessedBuildArgs%

call "%__ProjectDir%\dotnet.cmd" msbuild %__ProjectDir%\tests\build.proj !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! %TargetsWindowsMsbuildArg% %__msbuildArgs% !__PriorityArg! %__UnprocessedBuildArgs%
Expand Down
13 changes: 6 additions & 7 deletions build-test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,22 +312,21 @@ build_MSBuild_projects()
# __SkipPackageRestore and __SkipTargetingPackBuild used to control build by tests/src/dirs.proj
export __SkipPackageRestore=false
export __SkipTargetingPackBuild=false
export __BuildLoopCount=2
export __TestGroupToBuild=1
export __NumberOfTestGroups=3

__AppendToLog=false

if [ -n "$__priority1" ]; then
export __BuildLoopCount=16
export __TestGroupToBuild=2
export __NumberOfTestGroups=10
fi

for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
for (( testGroupToBuild=1 ; testGroupToBuild <= __NumberOfTestGroups; testGroupToBuild = testGroupToBuild + 1 ))
do
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""

export TestBuildSlice=$slice
export __TestGroupToBuild=$testGroupToBuild

# Generate build command
buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
Expand All @@ -341,7 +340,7 @@ build_MSBuild_projects()
buildArgs+=("${__UnprocessedBuildArgs[@]}")

nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
echo "Building step '$stepName' slice=$slice via $nextCommand"
echo "Building step '$stepName' testGroupToBuild=$testGroupToBuild via $nextCommand"
eval $nextCommand

# Make sure everything is OK
Expand Down
266 changes: 75 additions & 191 deletions tests/src/dirs.proj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,218 +33,102 @@
<DisabledProjects Include="Loader\classloader\generics\GenericMethods\VSW491668.csproj" /> <!-- issue 5501 -->
</ItemGroup>

<!-- Test build is divided in slices which can be created within Test Group
Priority 0 tests are build using Test Group 1 with 2 subgroups or slices -->

<ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<ItemGroup>
<AllProjects Include="*\**\*.csproj" Exclude="@(DisabledProjects)" />
<AllProjects Include="*\**\*.ilproj" Exclude="@(DisabledProjects)" />
</ItemGroup>

<ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<!-- All the test projects are partitioned into the test groups as defined below.
Each of the test groups is meant to be built by a separate MSBuild invocation with specified $(__TestGroupToBuild) property. -->

<!-- Test build is divided in slices which can be created within Test Group
Priority 1 or higher tests are build using Test Group 2 with 16 subgroups or slices -->
<ItemGroup Condition=" '$(CLRTestPriorityToBuild)' == '0' ">
<!-- Group number k consists of all the test projects P such that _GroupStartsWith(k) <= P.Identity < _GroupStartsWith(k+1).
In other words, ItemGroup _GroupStartsWith defines boundaries between the test groups. -->

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
<Project Include="baseservices\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Common\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="baseservices\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Common\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<!-- MSBuild does not allow specifying an item with empty identity but if it was possible Group number 1 would be defined as follows:
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
<Project Include="CoreMangLib\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="CoreMangLib\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="">
<GroupNumber>1</GroupNumber>
</_GroupStartsWith> -->

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
<Project Include="E*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="GC\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="hosting\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Interop\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="E*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="GC\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="hosting\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Interop\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\Methodical\Boxing\callconv\_relinstance_il.ilproj">
<GroupNumber>2</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
<Project Include="JIT\B*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\C*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Directed\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<_GroupStartsWith Include="JIT\Regression\CLR-x86-JIT\V1-M12-Beta2\b35351\b35351.ilproj">
<GroupNumber>3</GroupNumber>
</_GroupStartsWith>
</ItemGroup>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
<Project Include="JIT\B*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\C*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Directed\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<Error Condition=" '$(CLRTestPriorityToBuild)' == '0' And ($(__TestGroupToBuild) &lt; 1 Or $(__TestGroupToBuild) &gt; 3) " Text="__TestGroupToBuild property must be between 1 and 3 for Pri0." />

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
<Project Include="JIT\Generics\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\*Intrinsics\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Generics\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\*Intrinsics\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<ItemGroup Condition=" '$(CLRTestPriorityToBuild)' == '1' ">
<!-- See above for explanation.
<_GroupStartsWith Include="">
<GroupNumber>1</GroupNumber>
</_GroupStartsWith> -->

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
<Project Include="JIT\IL_Conformance\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\IL_Conformance\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\CodeGenBringUpTests\DblNeg_ro.csproj">
<GroupNumber>2</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
<Project Include="JIT\jit64\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\jit64\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\Directed\shift\uint32_d.csproj">
<GroupNumber>3</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
<Project Include="JIT\Methodical\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\Methodical\AsgOp\r8\r8flat_cs_r.csproj">
<GroupNumber>4</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
<Project Include="JIT\Methodical\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\Methodical\eh\finallyexec\tryCatchFinallyThrow_nonlocalexit4_ro.csproj">
<GroupNumber>5</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
<Project Include="JIT\opt\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Performance\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\S*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\opt\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Performance\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\S*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\Regression\CLR-x86-JIT\V1-M09.5-PDC\b25701\b25701.ilproj">
<GroupNumber>6</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
<Project Include="JIT\R*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\Regression\JitBlue\GitHub_19171\GitHub_19171.csproj">
<GroupNumber>7</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
<Project Include="JIT\R*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<_GroupStartsWith Include="JIT\jit64\valuetypes\nullable\box-unbox\value\box-unbox-value030.csproj">
<GroupNumber>8</GroupNumber>
</_GroupStartsWith>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
<Project Include="Loader\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<_GroupStartsWith Include="Loader\classloader\TypeGeneratorTests\TypeGeneratorTest225\Generated225.ilproj">
<GroupNumber>9</GroupNumber>
</_GroupStartsWith>

<_GroupStartsWith Include="Loader\classloader\generics\VSD\Class2_ImplicitOverrideVirtualNewslot.csproj">
<GroupNumber>10</GroupNumber>
</_GroupStartsWith>
</ItemGroup>

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
<Project Include="Loader\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
<Error Condition=" '$(CLRTestPriorityToBuild)' == '1' And ($(__TestGroupToBuild) &lt; 1 Or $(__TestGroupToBuild) &gt; 10)" Text="__TestGroupToBuild property must be between 1 and 10 for Pri1." />

<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
<Project Include="m*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="p*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="r*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="s*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="t*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="m*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="p*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="r*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="s*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="t*\**\*.ilproj" Exclude="@(DisabledProjects)">
<PropertyGroup>
<!-- This computes lower inclusive and upper exclusive boundaries for Group number $(__TestGroupToBuild). -->
<_GroupStartsWith>@(_GroupStartsWith->WithMetadataValue("GroupNumber", $(__TestGroupToBuild)))</_GroupStartsWith>
<_GroupEndsWithExcl>@(_GroupStartsWith->WithMetadataValue("GroupNumber", $([MSBuild]::Add($(__TestGroupToBuild), 1))))</_GroupEndsWithExcl>
</PropertyGroup>

<ItemGroup>
<AllProjects>
<InGroup>True</InGroup>
</AllProjects>

<AllProjects Condition=" '$(_GroupStartsWith)' != '' And $([System.StringComparer]::Ordinal.Compare($(_GroupStartsWith), %(Identity))) &gt; 0 ">
<InGroup>False</InGroup>
</AllProjects>

<AllProjects Condition=" '$(_GroupEndsWithExcl)' != '' And $([System.StringComparer]::Ordinal.Compare(%(Identity), $(_GroupEndsWithExcl))) &gt;= 0 ">
<InGroup>False</InGroup>
</AllProjects>
</ItemGroup>

<ItemGroup>
<Project Include="@(AllProjects->WithMetadataValue('InGroup', 'True'))">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
Expand Down

0 comments on commit 39b643f

Please sign in to comment.