diff --git a/init-tools.cmd b/init-tools.cmd index ec21fe808dac..a3892065f1ef 100644 --- a/init-tools.cmd +++ b/init-tools.cmd @@ -47,9 +47,33 @@ echo Running %0 > "%INIT_TOOLS_LOG%" set /p DOTNET_VERSION=< "%~dp0DotnetCLIVersion.txt" if exist "%DOTNET_CMD%" goto :afterdotnetrestore +REM Use x86 tools on arm64 and x86. +REM arm32 host is not currently supported, please crossbuild. +if /i "%PROCESSOR_ARCHITECTURE%" == "arm" ( + echo "Error, arm32 arch not supported for build tools." + exit /b 1 +) + +if /i "%PROCESSOR_ARCHITECTURE%" == "amd64" ( + set _Arch="x64" + goto ArchSet +) + +REM If this is not amd64 and not arm, then we should be running on arm64 or x86 +REM either way we can (and should) use the x86 dotnet cli +REM +REM TODO: consume native arm64 toolset, blocked by official arm64 windows cli +REM : release. See https://github.com/dotnet/coreclr/issues/19614 for more +REM : information +set _Arch="x86" + +echo "init-tools.cmd: Setting arch to %_Arch% for build tools" + +:ArchSet + echo Installing dotnet cli... if NOT exist "%DOTNET_PATH%" mkdir "%DOTNET_PATH%" -set DOTNET_ZIP_NAME=dotnet-sdk-%DOTNET_VERSION%-win-x64.zip +set DOTNET_ZIP_NAME=dotnet-sdk-%DOTNET_VERSION%-win-%_Arch%.zip set DOTNET_REMOTE_PATH=https://dotnetcli.azureedge.net/dotnet/Sdk/%DOTNET_VERSION%/%DOTNET_ZIP_NAME% set DOTNET_LOCAL_PATH=%DOTNET_PATH%%DOTNET_ZIP_NAME% echo Installing '%DOTNET_REMOTE_PATH%' to '%DOTNET_LOCAL_PATH%' >> "%INIT_TOOLS_LOG%" diff --git a/tests/arm/Tests.lst b/tests/arm/Tests.lst index 30f55e4750e6..7bfd78c1fb23 100644 --- a/tests/arm/Tests.lst +++ b/tests/arm/Tests.lst @@ -53340,14 +53340,6 @@ MaxAllowedDurationSeconds=600 Categories=EXPECTED_PASS;Pri1 HostStyle=0 -[b12011.cmd_6691] -RelativePath=JIT\Regression\VS-ia64-JIT\V1.2-M02\b12011\b12011\b12011.cmd -WorkingDir=JIT\Regression\VS-ia64-JIT\V1.2-M02\b12011\b12011 -Expected=0 -MaxAllowedDurationSeconds=600 -Categories=EXPECTED_PASS -HostStyle=0 - [FlowControlNext.cmd_6692] RelativePath=CoreMangLib\cti\system\reflection\emit\flowcontrol\FlowControlNext\FlowControlNext.cmd WorkingDir=CoreMangLib\cti\system\reflection\emit\flowcontrol\FlowControlNext diff --git a/tests/arm64/Tests.lst b/tests/arm64/Tests.lst index 5af8bd5d24fb..4f3f900a071b 100644 --- a/tests/arm64/Tests.lst +++ b/tests/arm64/Tests.lst @@ -70452,14 +70452,6 @@ MaxAllowedDurationSeconds=600 Categories=JIT;EXPECTED_PASS HostStyle=0 -[b12011.cmd_9160] -RelativePath=JIT\Regression\VS-ia64-JIT\V1.2-M02\b12011\b12011\b12011.cmd -WorkingDir=JIT\Regression\VS-ia64-JIT\V1.2-M02\b12011\b12011 -Expected=0 -MaxAllowedDurationSeconds=600 -Categories=EXPECTED_PASS -HostStyle=0 - [b14355.cmd_9161] RelativePath=JIT\Regression\VS-ia64-JIT\V1.2-M02\b14355\b14355\b14355.cmd WorkingDir=JIT\Regression\VS-ia64-JIT\V1.2-M02\b14355\b14355 diff --git a/tests/issues.targets b/tests/issues.targets index ef8dd508a57a..055072f8e8ca 100644 --- a/tests/issues.targets +++ b/tests/issues.targets @@ -1,443 +1,636 @@ + - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - + 3392 - - 4851 + + 15353 - - 4851 + + 11408 - - 4851 + + + + + + 18381 - - 4851 + + Issue building native components for the test. - - 4849 + + + + + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage + + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + needs triage - - 2451 + + - - 2451 + + times out - - 2451 + + needs triage - - 2451 + + needs triage - - 4851 + + needs triage - - 4851 + + times out - - 4851 + + times out - - 4851 + + needs triage - - 4851 + + times out - - 4851 + + times out - - 2434 + + needs triage - - 2441 + + times out - - 2441 + + times out - - 2441 + + needs triage - - 2441 + + times out - - 2444 + + times out - - 2444 + + needs triage - - 2444 + + times out - - 2444 + + times out - - 2445 + + times out - - 11408 + + times out - - 15353 + + + + + + 8648 - + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + needs triage + + + 19441 + + + + + + 6707 - + 6707 - + 6707 - + 6707 - + 6707 - + 6707 + + 18056 + + + Varargs supported on this platform + + + Varargs supported on this platform + + + Not a shipping feature + - + + + + 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + + 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + + 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + + 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + + 2420. x86 JIT doesn't support implicit tail call optimization or tail. call pop ret sequence + + + 11469, The test causes OutOfMemory exception in crossgen mode. + + + Varargs supported on this platform + + + Varargs supported on this platform + + - - - Not a shipping feature + + + + 15016 - - Not a shipping feature + + 15016 - - Not a shipping feature + + 15016 - - Not a shipping feature + + Uses illagal il tailcall pop ret - - Not a shipping feature + + 9565 - - Not a shipping feature + + 11534 - - Not a shipping feature + + 10636 - - Not a shipping feature + + 9565 - - Not a shipping feature + + 9565 - - Not a shipping feature + + 9565 - - Not a shipping feature + + 9565 - - Not a shipping feature + + 15494 - - Not a shipping feature + + 9565 - - - - - - - 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + 9565 - - 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + 9565 - - 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + 9565 - - 3392, test is useful to have because it can be run manually when making changes to the GC that can have effects in OOM scenarios, but not appropriate to run on our current test infrastructure. + + 9565 - - 2420. x86 JIT doesn't support implicit tail call optimization or tail. call pop ret sequence + + 9565 - - 7163, fails on both legacy backend and RyuJIT + + 9565 - - 11469, The test causes OutOfMemory exception in crossgen mode. + + 18895 + + + 9565 + + + 15494 + + + 15016 + + + 9565 + + + 9565 - - + - + 6217 - + 2420 + + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Varargs not supported on this platform + + + Issue building native components for the test. + + + Issue building native components for the test. + + + 15353 + + + 18381 + + + 18381 + + + Needs Triage + + + Needs Triage + + + Needs Triage + + + timeout + + + Needs Triage + + + + + + + Test times out + + + Test times out + + + Test times out + + + Test times out + + + Test times out + + + + + + + 19340 + + + + - - + + + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage @@ -445,123 +638,128 @@ - + needs triage - + 9314 - + 9314 - + 9314 - + 9314 - + 9314 - + 9314 + + + + Fails on Windows.Nano + - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + 3392 - + 3392 - + 3392 - + 3392 - + needs triage - + needs triage - + needs triage - + needs triage @@ -576,1186 +774,1189 @@ needs triage - + needs triage - + needs triage - + + by design Windows only + + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + needs triage - + by design Windows only @@ -1763,13 +1964,13 @@ - + By Test Infrastructure design - + Bug diff --git a/tests/runtest.cmd b/tests/runtest.cmd index c79c0a438c98..aa3700335d59 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -39,6 +39,8 @@ set __CollectDumps= set __DoCrossgen= set __CrossgenAltJit= set __PerfTests= +set __SkipGenerateLayout= +set __BuildXUnitWrappers= :Arg_Loop if "%1" == "" goto ArgsDone @@ -76,6 +78,8 @@ if /i "%1" == "jitforcerelocs" (set COMPlus_ForceRelocs=1&shift&goto Arg_ if /i "%1" == "jitdisasm" (set __JitDisasm=1&shift&goto Arg_Loop) if /i "%1" == "ilasmroundtrip" (set __IlasmRoundTrip=1&shift&goto Arg_Loop) if /i "%1" == "GenerateLayoutOnly" (set __GenerateLayoutOnly=1&shift&goto Arg_Loop) +if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayout=1&shift&goto Arg_Loop) +if /i "%1" == "buildxunitwrappers" (set __BuildXunitWrappers=1&shift&goto Arg_Loop) if /i "%1" == "PerfTests" (set __PerfTests=true&shift&goto Arg_Loop) if /i "%1" == "runcrossgentests" (set RunCrossGen=true&shift&goto Arg_Loop) if /i "%1" == "link" (set DoLink=true&set ILLINK=%2&shift&shift&goto Arg_Loop) @@ -85,7 +89,7 @@ if /i "%1" == "gcname" (set COMPlus_GCName=%2&shift&shift&goto Ar if /i "%1" == "timeout" (set __TestTimeout=%2&shift&shift&goto Arg_Loop) REM change it to COMPlus_GCStress when we stop using xunit harness -if /i "%1" == "gcstresslevel" (set __GCSTRESSLEVEL=%2&set __TestTimeout=1800000&shift&shift&goto Arg_Loop) +if /i "%1" == "gcstresslevel" (set COMPlus_GCStress=%2&set __TestTimeout=1800000&shift&shift&goto Arg_Loop) if /i "%1" == "collectdumps" (set __CollectDumps=true&shift&goto Arg_Loop) if /i not "%1" == "msbuildargs" goto SkipMsbuildArgs @@ -119,6 +123,76 @@ set "__TestWorkingDir=%__RootBinDir%\tests\%__BuildOS%.%__BuildArch%.%__BuildTyp if not defined XunitTestBinBase set XunitTestBinBase=%__TestWorkingDir% if not defined XunitTestReportDirBase set XunitTestReportDirBase=%XunitTestBinBase%\Reports\ +REM At this point in the script there will be a divergence in how the tests are run. +REM For official builds we will continue to run tests using the un-unified scripting +REM which relies on msbuild and calls runtest.proj directly. For all other scenarios +REM runtest.py will handle setup and will then call runtest.proj + +if defined __AgainstPackages goto SetupMSBuildAndCallRuntestProj +if "%__PerfTests%"=="true" goto SetupMSBuildAndCallRuntestProj +if defined __GenerateLayoutOnly goto SetupMSBuildAndCallRuntestProj + +REM We are not running in the official build scenario, call runtest.py + +set __RuntestPyArgs=-arch %__BuildArch% -build_type %__BuildType% + +if defined DoLink ( + set __RuntestPyArgs=%__RuntestPyArgs% --il_link +) + +if defined __LongGCTests ( + set __RuntestPyArgs=%__RuntestPyArgs% --long_gc +) + +if defined __GCSimulatorTests ( + set __RuntestPyArgs=%__RuntestPyArgs% --gcsimulator +) + +if defined __JitDisasm ( + set __RuntestPyArgs=%__RuntestPyArgs% --jitdisasm +) + +if defined __IlasmRoundTrip ( + set __RuntestPyArgs=%__RuntestPyArgs% --ilasmroundtrip +) + +if defined __TestEnv ( + set __RuntestPyArgs=%__RuntestPyArgs% -test_env %__TestEnv% +) + +if defined __Sequential ( + set __RuntestPyArgs=%__RuntestPyArgs% --sequential +) + +if not defined __SkipGenerateLayout ( + set __RuntestPyArgs=%__RuntestPyArgs% --generate_layout +) + +if defined __GenerateLayoutOnly ( + set __RuntestPyArgs=%__RuntestPyArgs% --generate_layout_only +) + +if defined __BuildXUnitWrappers ( + set __RuntestPyArgs=%__RuntestPyArgs% --build_xunit_test_wrappers +) + +if defined RunCrossGen ( + set __RuntestPyArgs=%__RuntestPyArgs% --run_crossgen_tests +) + +if defined __DoCrossgen ( + set __RuntestPyArgs=%__RuntestPyArgs% --precompile_core_root +) + +REM __ProjectDir is poorly named, it is actually /tests +set NEXTCMD=python "%__ProjectDir%\runtest.py" %__RuntestPyArgs% +echo !NEXTCMD! +!NEXTCMD! + +exit /b %ERRORLEVEL% + +:SetupMSBuildAndCallRuntestProj + :: Set up msbuild and tools environment. Check if msbuild and VS exist. set _msbuildexe= diff --git a/tests/runtest.proj b/tests/runtest.proj index 173626378e51..140c879dc18e 100644 --- a/tests/runtest.proj +++ b/tests/runtest.proj @@ -284,6 +284,9 @@ namespace $([System.String]::Copy($(Category)).Replace(".","_").Replace("\",""). + + + diff --git a/tests/runtest.py b/tests/runtest.py index 7cadfb4b1089..6d0d30eb380a 100755 --- a/tests/runtest.py +++ b/tests/runtest.py @@ -1,12 +1,36 @@ #!/usr/bin/env python -################################################################################ -################################################################################ # -# Module: runtest.py +## Licensed to the .NET Foundation under one or more agreements. +## The .NET Foundation licenses this file to you under the MIT license. +## See the LICENSE file in the project root for more information. +# +## +# Title :runtest.py # # Notes: # -# Universal script to setup and run the xunit msbuild test runner. +# Universal script to setup and run the xunit console runner. The script relies +# on runtest.proj and the bash and batch wrappers. All test excludes will also +# come from issues.targets. If there is a jit stress or gc stress exclude, +# please add GCStressIncompatible or JitOptimizationSensitive to the test's +# ilproj or csproj. +# +# The xunit runner currently relies on tests being built on the same host as the +# target platform. This requires all tests run on linux x64 to be built by the +# same platform and arch. If this is not done, the tests will run correctly; +# however, expect failures due to incorrect exclusions in the xunit +# wrappers setup at build time. +# +# Note that for linux targets the native components to the tests are still built +# by the product build. This requires all native components to be either copied +# into the Core_Root directory or the test's managed directory. The latter is +# prone to failure; however, copying into the Core_Root directory may create +# naming conflicts. +# +# If you are running tests on a different target than the host that built, the +# native tests components must be copied from: +# bin/obj/.... 0: + # Force the build type to be capitalized + build_type = build_type.capitalize() + + if not build_type in valid_build_types: + print "Unsupported configuration: %s." % build_type + print "Supported configurations: %s" % "[%s]" % ", ".join(valid_build_types) + sys.exit(1) + + return build_type + + build_type = check_build_type(build_type) + if test_location is None: - print "Using default test location." - test_location = os.path.join(coreclr_repo_location, "bin", "tests", "%s.%s.%s" % (host_os, arch, build_type)) - print "TestLocation: %s" % test_location - print + default_test_location = os.path.join(coreclr_repo_location, "bin", "tests", "%s.%s.%s" % (host_os, arch, build_type)) + + if os.path.isdir(default_test_location): + test_location = default_test_location + + print "Using default test location." + print "TestLocation: %s" % default_test_location + print + + else: + # The tests for the default location have not been built. + print "Error, unable to find the tests at %s" % default_test_location + + suggested_location = None + possible_test_locations = [item for item in os.listdir(os.path.join(coreclr_repo_location, "bin", "tests")) if host_os in item and arch in item] + if len(possible_test_locations) > 0: + print "Tests are built for the following:" + for item in possible_test_locations: + print item.replace(".", " ") + + print "Please run runtest.py again with the correct build-type by passing -build_type" + else: + print "No tests have been built for this host and arch. Please run build-test.%s" % ("cmd" if host_os == "Windows_NT" else "sh") + + sys.exit(1) + else: + # If we have supplied our own test location then we need to create a test location + # that the scripting will expect. As it is now, there is a dependency on the + # test location being under test/.. + + # Make sure that we are using the correct build_type. This is a test drop, it is possible + # that we are inferring the build type to be Debug incorrectly. + + if build_type not in test_location: + # Remove punctuation + corrected_build_type = re.sub("[%s]" % string.punctuation, "", test_location.split(".")[-1]) + build_type = check_build_type(corrected_build_type) + + default_test_location = os.path.join(coreclr_repo_location, "bin", "tests", "%s.%s.%s" % (host_os, arch, build_type)) + + # Remove optional end os.path.sep + if test_location[-1] == os.path.sep: + test_location = test_location[:-1] + + if test_location != default_test_location and os.path.isdir(default_test_location): + # Remove the existing directory if there is one. + shutil.rmtree(default_test_location) + + print "Non-standard test location being used." + print "Overwrite the standard location with these tests." + print "TODO: Change runtest.proj to allow running from non-standard test location." + print "" + + print "cp -r %s %s" % (test_location, default_test_location) + shutil.copytree(test_location, default_test_location) + + test_location = default_test_location + + # unset core_root so it can be put in the default location + core_root = None + + # Force the core_root to be setup again. + args.generate_layout = True + + else: + test_location = default_test_location + + print "Using default test location." + print "TestLocation: %s" % default_test_location + print if core_root is None: - print "Using default location for core_root." - core_root = os.path.join(test_location, "Tests", "Core_Root") + default_core_root = os.path.join(test_location, "Tests", "Core_Root") + + if os.path.isdir(default_core_root): + core_root = default_core_root + + print "Using default location for core_root." + print "Core_Root: %s" % core_root + print + + elif args.generate_layout is False: + # CORE_ROOT has not been setup correctly. + print "Error, unable to find CORE_ROOT at %s" % default_core_root + print "Please run runtest.py with --generate_layout specified." + + sys.exit(1) + + else: + print "--generate_layout passed. Core_Root will be populated at: %s" % default_core_root + core_root = default_core_root + else: print "Core_Root: %s" % core_root - print if host_os != "Windows_NT": if test_native_bin_location is None: @@ -349,33 +919,23 @@ def setup_args(args): test_native_bin_location = os.path.join(os.path.join(coreclr_repo_location, "bin", "obj", "%s.%s.%s" % (host_os, arch, build_type), "tests")) print "Native bin location: %s" % test_native_bin_location print - - valid_arches = ["x64", "x86", "arm", "arm64"] - if not arch in valid_arches: - print "Unsupported architecture: %s." % arch - print "Supported architectures: %s" % "[%s]" % ", ".join(valid_arches) - sys.exit(1) - - valid_build_types = ["Debug", "Checked", "Release"] - if not build_type in valid_build_types: - print "Unsupported configuration: %s." % build_type - print "Supported configurations: %s" % "[%s]" % ", ".join(valid_build_types) - sys.exit(1) - - if not os.path.isdir(test_location): - print "Error, test location: %s, does not exist." % test_location - sys.exit(1) - - if not os.path.isdir(core_root): - print "Error, core_root: %s, does not exist." % core_root - sys.exit(1) - - if host_os != "Windows_NT": + if not os.path.isdir(test_native_bin_location): print "Error, test_native_bin_location: %s, does not exist." % test_native_bin_location sys.exit(1) - return host_os, arch, build_type, coreclr_repo_location, core_root, test_location, test_native_bin_location + if args.product_location is None and args.generate_layout: + product_location = os.path.join(coreclr_repo_location, "bin", "Product", "%s.%s.%s" % (host_os, arch, build_type)) + if not os.path.isdir(product_location): + print "Error, unable to determine the product location. This is most likely because build_type was" + print "incorrectly passed. Or the product is not built. Please explicitely pass -product_location" + + sys.exit(1) + + else: + product_location = args.product_location + + return host_os, arch, build_type, coreclr_repo_location, product_location, core_root, test_location, test_native_bin_location def setup_tools(host_os, coreclr_repo_location): """ Setup the tools for the repo @@ -392,16 +952,18 @@ def setup_tools(host_os, coreclr_repo_location): is_windows = host_os == "Windows_NT" - if os.path.isfile(os.path.join(tools_dir, "msbuild.%s" % ("cmd" if is_windows else "sh"))): + dotnetcli_location = os.path.join(coreclr_repo_location, "Tools", "dotnetcli", "dotnet%s" % (".exe" if host_os == "Windows_NT" else "")) + + if os.path.isfile(dotnetcli_location): setup = True # init the tools for the repo if not setup: command = None if is_windows: - command = [os.path.join(coreclr_repo_location, "init_tools.cmd")] + command = [os.path.join(coreclr_repo_location, "init-tools.cmd")] else: - command = ["sh", os.path.join(coreclr_repo_location, "init_tools.sh")] + command = ["bash", os.path.join(coreclr_repo_location, "init-tools.sh")] print " ".join(command) subprocess.check_output(command) @@ -410,27 +972,1008 @@ def setup_tools(host_os, coreclr_repo_location): return setup -################################################################################ -# Main -################################################################################ +def setup_coredis_tools(coreclr_repo_location, host_os, arch, core_root): + """ Setup CoreDisTools if needed -def main(args): - host_os, arch, build_type, coreclr_repo_location, core_root, test_location, test_native_bin_location = setup_args(args) + Args: + coreclr_repo_location(str) : coreclr repo location + host_os(str) : os + arch(str) : arch + core_root(str) : core_root + """ + + test_location = os.path.join(coreclr_repo_location, "tests") + + def is_coredis_tools_supported(host_os, arch): + """ Is coredis tools supported on this os/arch + + Args: + host_os(str): os + arch(str) : arch + + """ + unsupported_unix_arches = ["arm", "arm64"] + + if host_os.lower() == "osx": + return False + + return True + + if host_os != "Windows_NT" and arch in unsupported_unix_arches: + return False + + return True + + if is_coredis_tools_supported(host_os, arch): + command = None + if host_os == "Windows_NT": + command = [os.path.join(test_location, "setup-stress-dependencies.cmd"), "/arch", arch, "/outputdir", core_root] + else: + command = [os.path.join(test_location, "setup-stress-dependencies.sh"), "--outputDir=%s" % core_root] + + proc = subprocess.Popen(command) + proc.communicate() + + if proc.returncode != 0: + print "setup_stress_dependencies.sh failed." + sys.exit(1) + else: + print "GCStress C is not supported on your platform." + sys.exit(1) + +def precompile_core_root(test_location, + host_os, + arch, + core_root, + use_jit_disasm=False, + altjit_name=False): + """ Precompile all of the assemblies in the core_root directory + + Args: + test_location(str) : test location + host_os(str) : os + core_root(str) : location of core_root + use_jit_disasm(Bool) : use jit disasm + altjit_name(str) : name of the altjit + + """ + + skip_list = [ + ".*xunit.*", + ".*api-ms-win-core.*", + ".*api-ms-win.*", + ".*System.Private.CoreLib.*" + ] + + unix_skip_list = [ + ".*mscorlib.*", + ".*System.Runtime.WindowsRuntime.*", + ".*System.Runtime.WindowsRuntime.UI.Xaml.*", + ".*R2RDump.dll.*" + ] + + arm64_unix_skip_list = [ + ".*Microsoft.CodeAnalysis.VisualBasic.*", + ".*System.Net.NameResolution.*", + ".*System.Net.Sockets.*", + ".*System.Net.Primitives.*" + ] + + if host_os != "Windows_NT": + skip_list += unix_skip_list + + if arch == "arm64": + skip_list += arm64_unix_skip_list + + assert os.path.isdir(test_location) + assert os.path.isdir(core_root) + + crossgen = os.path.join(core_root, "crossgen%s" % (".exe" if host_os == "Windows_NT" else "")) + assert os.path.isfile(crossgen) + + def call_crossgen(file, env): + assert os.path.isfile(crossgen) + command = [crossgen, "/Platform_Assemblies_Paths", core_root, file] + + if use_jit_disasm: + core_run = os.path.join(core_root, "corerun%s" % (".exe" if host_os == "Windows_NT" else "")) + assert os.path.isfile(core_run) + + command = [core_run, + os.path.join(core_root, "jit-dasm.dll"), + "--crossgen", + crossgen, + "--platform", + core_root, + "--output", + os.path.join(test_location, "dasm"), + file] + + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + proc.communicate() + + return_code = proc.returncode + + passed = False + if return_code == -2146230517: + print "%s is not a managed assembly." % file + return passed + + if return_code != 0: + print "Unable to precompile %s" % file + return passed + + print "Successfully precompiled %s" % file + passed = True + + return passed + + print "Precompiling all assemblies in %s" % core_root + print + + env = os.environ.copy() + + if not altjit_name is None: + env["COMPlus_AltJit"]="*" + env["COMPlus_AltJitNgen"]="*" + env["COMPlus_AltJitName"]=altjit_name + env["COMPlus_AltJitAssertOnNYI"]="1" + env["COMPlus_NoGuiOnAssert"]="1" + env["COMPlus_ContinueOnAssert"]="0" + + dlls = [os.path.join(core_root, item) for item in os.listdir(core_root) if item.endswith("dll") and "mscorlib" not in item] + + def in_skip_list(item): + found = False + for skip_re in skip_list: + if re.match(skip_re, item.lower()) is not None: + found = True + return found + + dlls = [dll for dll in dlls if not in_skip_list(dll)] + + for dll in dlls: + call_crossgen(dll, env) + + print + +def setup_core_root(host_os, + arch, + build_type, + coreclr_repo_location, + test_native_bin_location, + product_location, + test_location, + core_root, + is_corefx=False, + generate_layout=True): + """ Setup the core root + + Args: + host_os(str) : os + arch(str) : architecture + build_type(str) : build configuration + coreclr_repo_location(str) : coreclr repo location + product_location(str) : Product location + core_root(str) : Location for core_root + is_corefx : Building corefx core_root + + Optional Args: + is_corefx(Bool) : Pass if planning on running corex + : tests + + """ + global g_verbose + + assert os.path.isdir(product_location) + + # Create core_root if it does not exist + if os.path.isdir(core_root): + shutil.rmtree(core_root) + + os.makedirs(core_root) + + # Setup the dotnetcli location + dotnetcli_location = os.path.join(coreclr_repo_location, "Tools", "dotnetcli", "dotnet%s" % (".exe" if host_os == "Windows_NT" else "")) + + # Set global env variables. + os.environ["__BuildLogRootName"] = "Restore_Product" + + if host_os != "Windows_NT": + os.environ["__DistroRid"] = "%s-%s" % ("osx" if sys.platform == "darwin" else "linux", arch) + + command = [os.path.join(coreclr_repo_location, "run.%s" % ("cmd" if host_os == "Windows_NT" else "sh")), + "build", + "-Project=%s" % os.path.join(coreclr_repo_location, "tests", "build.proj")] + + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + if not os.path.isdir(logs_dir): + os.makedirs(logs_dir) + + log_path = os.path.join(logs_dir, "Restore_Product%s_%s_%s" % (host_os, arch, build_type)) + build_log = log_path + ".log" + wrn_log = log_path + ".wrn" + err_log = log_path + ".err" + + msbuild_log_params = "/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log + msbuild_wrn_params = "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log + msbuild_err_params = "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log + + command += ["-MsBuildLog=%s" % msbuild_log_params, + "-MsBuildWrn=%s" % msbuild_wrn_params, + "-MsBuildErr=%s" % msbuild_err_params] + + if host_os != "Windows_NT": + command = ["bash"] + command + command += ["-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + + if g_verbose: + command += ["-verbose"] + + command += [ "-BatchRestorePackages", + "-BuildType=%s" % build_type, + "-BuildArch=%s" % arch, + "-BuildOS=%s" % host_os] + + print "Restoring packages..." + print " ".join(command) + + if not g_verbose: + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + proc = subprocess.Popen(command) + + try: + proc.communicate() + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + + if proc.returncode == 1: + "Error test dependency resultion failed." + return False + + os.environ["__BuildLogRootName"] = "" + + # Copy restored packages to core_root + # Set global env variables. + os.environ["__BuildLogRootName"] = "Tests_Overlay_Managed" + + if host_os != "Windows_NT": + os.environ["__DistroRid"] = "%s-%s" % ("osx" if sys.platform == "darwin" else "linux", arch) + os.environ["__RuntimeId"] = os.environ["__DistroRid"] + + os.environ["Core_Root"] = core_root + os.environ["xUnitTestBinBase"] = os.path.dirname(os.path.dirname(core_root)) + + command = [os.path.join(coreclr_repo_location, "run.%s" % ("cmd" if host_os == "Windows_NT" else "sh")), + "build", + "-Project=%s" % os.path.join(coreclr_repo_location, "tests", "runtest.proj")] + + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + if not os.path.isdir(logs_dir): + os.makedirs(logs_dir) + + log_path = os.path.join(logs_dir, "Tests_Overlay_Managed%s_%s_%s" % (host_os, arch, build_type)) + build_log = log_path + ".log" + wrn_log = log_path + ".wrn" + err_log = log_path + ".err" + + msbuild_log_params = "/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log + msbuild_wrn_params = "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log + msbuild_err_params = "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log + + command += ["-MsBuildLog=%s" % msbuild_log_params, + "-MsBuildWrn=%s" % msbuild_wrn_params, + "-MsBuildErr=%s" % msbuild_err_params] + + if host_os != "Windows_NT": + command = ["bash"] + command + command += ["-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + + if g_verbose: + command += ["-verbose"] + + command += [ "-testOverlay", + "-BuildType=%s" % build_type, + "-BuildArch=%s" % arch, + "-BuildOS=%s" % host_os] + + print "" + print "Creating Core_Root..." + print " ".join(command) + + if not g_verbose: + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + proc = subprocess.Popen(command) + + try: + proc.communicate() + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + + if proc.returncode == 1: + "Error test dependency resultion failed." + return False + + os.environ["__BuildLogRootName"] = "" + os.environ["xUnitTestBinBase"] = "" + os.environ["__RuntimeId"] = "" + + def copy_tree(src, dest): + """ Simple copy from src to dest + """ + assert os.path.isdir(src) + assert os.path.isdir(dest) + + for item in os.listdir(src): + if ".nuget" in item: + pass + item = os.path.join(src, item) + + if os.path.isfile(item): + shutil.copy2(item, dest) + + if host_os != "Windows_NT": + # Set executable bit + os.chmod(os.path.join(dest, item), 0774) + else: + new_dir = os.path.join(dest, os.path.basename(item)) + if os.path.isdir(new_dir): + shutil.rmtree(new_dir) + + shutil.copytree(item, new_dir) + + # Copy the product dir to the core_root directory + print + print "Copying Product Bin to Core_Root:" + print "cp -r %s%s* %s" % (product_location, os.path.sep, core_root) + copy_tree(product_location, core_root) + print "---------------------------------------------------------------------" + print + + if is_corefx: + corefx_utility_setup = os.path.join(coreclr_repo_location, + "src", + "Common", + "CoreFX", + "TestFileSetup", + "CoreFX.TestUtils.TestFileSetup.csproj") + + os.environ["__BuildLogRootName"] = "Tests_GenerateTestHost" + msbuild_command = [dotnetcli_location, + "msbuild", + os.path.join(coreclr_repo_location, "tests", "runtest.proj"), + "/p:GenerateRuntimeLayout=true"] + proc = subprocess.Popen(msbuild_command) + proc.communicate() + + if not proc.returncode == 0: + "Error test dependency resultion failed." + return False + + os.environ["__BuildLogRootName"] = "" + + msbuild_command = [dotnetcli_location, + "msbuild", + "/t:Restore", + corefx_utility_setup] + + proc = subprocess.Popen(msbuild_command) + proc.communicate() + + if proc.returncode == 1: + "Error test dependency resultion failed." + return False + + corefx_logpath = os.path.join(coreclr_repo_location, + "bin", + "tests", + "%s.%s.%s" % (host_os, arch, build_type), + "CoreFX", + "CoreFXTestUtilities") + + msbuild_command = [dotnetcli_location, + "msbuild", + "/p:Configuration=%s" % build_type, + "/p:OSGroup=%s" % host_os, + "/p:Platform=%s" % arch, + "/p:OutputPath=%s" % corefx_logpath, + corefx_utility_setup] + + proc = subprocess.Popen(msbuild_command) + proc.communicate() + + if proc.returncode == 1: + "Error test dependency resultion failed." + return False + + print "Core_Root setup." + print "" + + return True + +def delete_existing_wrappers(test_location): + """ Delete the existing xunit wrappers + + Args: + test_location(str) : location of the test + """ + + assert os.path.isdir(test_location) or os.path.isfile(test_location) + + extension = "dll" + + if os.path.isdir(test_location): + for item in os.listdir(test_location): + delete_existing_wrappers(os.path.join(test_location, item)) + elif test_location.endswith(extension) and "xunitwrapper" in test_location.lower(): + # Delete the test wrapper. + + print "rm %s" % test_location + os.remove(test_location) + +def build_test_wrappers(host_os, + arch, + build_type, + coreclr_repo_location, + test_location): + """ Build the coreclr test wrappers + + Args: + host_os(str) : os + arch(str) : architecture + build_type(str) : build configuration + coreclr_repo_location(str) : coreclr repo location + test_location(str) : location of the test + + Notes: + Build the xUnit test wrappers. Note that this will have been done as a + part of build-test.cmd/sh. It is possible that the host has a different + set of dependencies from the target or the exclude list has changed + after building. + + """ + global g_verbose + + delete_existing_wrappers(test_location) + + # Setup the dotnetcli location + dotnetcli_location = os.path.join(coreclr_repo_location, "Tools", "dotnetcli", "dotnet%s" % (".exe" if host_os == "Windows_NT" else "")) + + # Set global env variables. + os.environ["__BuildLogRootName"] = "Tests_XunitWrapper" + os.environ["__Exclude"] = os.path.join(coreclr_repo_location, "tests", "issues.targets") + + command = [dotnetcli_location, + "msbuild", + os.path.join(coreclr_repo_location, "tests", "runtest.proj"), + "/p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/", + "/p:BuildWrappers=true", + "/p:TargetsWindows=%s" % ("true" if host_os == "Windows_NT" else "false")] + + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + if not os.path.isdir(logs_dir): + os.makedirs(logs_dir) + + log_path = os.path.join(logs_dir, "Tests_XunitWrapper%s_%s_%s" % (host_os, arch, build_type)) + build_log = log_path + ".log" + wrn_log = log_path + ".wrn" + err_log = log_path + ".err" + + command += ["/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log, + "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log, + "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log, + "/consoleloggerparameters:Summary"] + + command += ["/p:__BuildOS=%s" % host_os, + "/p:__BuildArch=%s" % arch, + "/p:__BuildType=%s" % build_type, + "/p:__LogsDir=%s" % logs_dir] + + print "Creating test wrappers..." + print " ".join(command) + + if not g_verbose: + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if not running_in_ci(): + try: + expected_time_to_complete = 60*5 # 5 Minutes + estimated_time_running = 0 + + time_delta = 1 + + while True: + time_remaining = expected_time_to_complete - estimated_time_running + time_in_minutes = math.floor(time_remaining / 60) + remaining_seconds = time_remaining % 60 + + sys.stdout.write("\rEstimated time remaining: %d minutes %d seconds" % (time_in_minutes, remaining_seconds)) + sys.stdout.flush() + + time.sleep(time_delta) + estimated_time_running += time_delta + + if estimated_time_running == expected_time_to_complete: + break + if proc.poll() is not None: + break + + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + else: + proc = subprocess.Popen(command) + + try: + proc.communicate() + except KeyboardInterrupt: + proc.kill() + sys.exit(1) + + if proc.returncode == 1: + "Error test dependency resultion failed." + return False + +def find_test_from_name(host_os, test_location, test_name): + """ Given a test's name return the location on disk + + Args: + host_os (str) : os + test_location (str) :path to the coreclr tests + test_name (str) : Name of the test, all special characters will have + : been replaced with underscores. + + Return: + test_path (str): Path of the test based on its name + """ + + location = test_name + + # Lambdas and helpers + is_file_or_dir = lambda path : os.path.isdir(path) or os.path.isfile(path) + def match_filename(test_path): + # Scan through the test directory looking for a similar + # file + global file_name_cache + + if not os.path.isdir(os.path.dirname(test_path)): + pass + + assert os.path.isdir(os.path.dirname(test_path)) + size_of_largest_name_file = 0 + + dir_contents = file_name_cache[os.path.dirname(test_path)] + + if dir_contents is None: + dir_contents = defaultdict(lambda: None) + for item in os.listdir(os.path.dirname(test_path)): + dir_contents[re.sub("[%s]" % string.punctuation, "_", item)] = item + + file_name_cache[os.path.dirname(test_path)] = dir_contents + + # It is possible there has already been a match + # therefore we need to remove the punctuation again. + basename_to_match = re.sub("[%s]" % string.punctuation, "_", os.path.basename(test_path)) + if basename_to_match in dir_contents: + test_path = os.path.join(os.path.dirname(test_path), dir_contents[basename_to_match]) + + size_of_largest_name_file = len(max(dir_contents, key=len)) + + return test_path, size_of_largest_name_file + + def dir_has_nested_substrings(test_path, test_item): + """ A directory has multiple paths where one path is a substring of another + """ + + dir_contents = file_name_cache[os.path.dirname(test_path)] + + if dir_contents is None: + dir_contents = defaultdict(lambda: None) + for item in os.listdir(os.path.dirname(test_path)): + dir_contents[re.sub("[%s]" % string.punctuation, "_", item)] = item + + file_name_cache[os.path.dirname(test_path)] = dir_contents + + test_item = re.sub("[%s]" % string.punctuation, "_", test_item) + + count = 0 + for item in dir_contents: + if test_item in item: + count += 1 + + return count > 1 + + # Find the test by searching down the directory list. + starting_path = test_location + loc_split = location.split("_") + append = False + for index, item in enumerate(loc_split): + if not append: + test_path = os.path.join(starting_path, item) + else: + append = False + test_path, size_of_largest_name_file = match_filename(starting_path + "_" + item) + + if not is_file_or_dir(test_path): + append = True + + # It is possible that there is another directory that is named + # without an underscore. + elif index + 1 < len(loc_split) and os.path.isdir(test_path): + next_test_path = os.path.join(test_path, loc_split[index + 1]) + + if not is_file_or_dir(next_test_path) or dir_has_nested_substrings(test_path, item): + added_path = test_path + for forward_index in range(index + 1, len(loc_split)): + added_path, size_of_largest_name_file = match_filename(added_path + "_" + loc_split[forward_index]) + if is_file_or_dir(added_path): + append = True + break + elif size_of_largest_name_file < len(os.path.basename(added_path)): + break + + starting_path = test_path + + location = starting_path + if not os.path.isfile(location): + pass + + assert(os.path.isfile(location)) + + return location + +def parse_test_results(host_os, arch, build_type, coreclr_repo_location, test_location): + """ Parse the test results for test execution information + + Args: + host_os : os + arch : architecture run on + build_type : build configuration (debug, checked, release) + coreclr_repo_location : coreclr repo location + test_location : path to coreclr tests + + """ + logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") + log_path = os.path.join(logs_dir, "TestRunResults_%s_%s_%s" % (host_os, arch, build_type)) + print "Parsing test results from (%s)" % log_path + + test_run_location = os.path.join(coreclr_repo_location, "bin", "Logs", "testRun.xml") + + if not os.path.isfile(test_run_location): + # Check if this is a casing issue + + found = False + for item in os.listdir(os.path.dirname(test_run_location)): + item_lower = item.lower() + if item_lower == "testrun.xml": + # Correct the name. + os.rename(os.path.join(coreclr_repo_location, "bin", "Logs", item), test_run_location) + found = True + break + + if not found: + print "Unable to find testRun.xml. This normally means the tests did not run." + print "It could also mean there was a problem logging. Please run the tests again." + + return + + if host_os != "Windows_NT" and running_in_ci(): + # Huge hack. + # TODO change netci to parse testRun.xml + shutil.copy2(test_run_location, os.path.join(os.path.dirname(test_run_location), "coreclrtests.xml")) + + assemblies = xml.etree.ElementTree.parse(test_run_location).getroot() + + tests = defaultdict(lambda: None) + for assembly in assemblies: + for collection in assembly: + if collection.tag == "errors" and collection.text != None: + # Something went wrong during running the tests. + print "Error running the tests, please run runtest.py again." + sys.exit(1) + elif collection.tag != "errors": + test_name = None + for test in collection: + type = test.attrib["type"] + method = test.attrib["method"] + + type = type.split("._")[0] + test_name = type + method + + assert test_name != None + + failed = collection.attrib["failed"] + skipped = collection.attrib["skipped"] + passed = collection.attrib["passed"] + time = float(collection.attrib["time"]) + + test_output = None + + if failed == "1": + failure_info = collection[0][0] + + test_output = failure_info[0].text + + test_location_on_filesystem = find_test_from_name(host_os, test_location, test_name) + + assert os.path.isfile(test_location_on_filesystem) + + assert tests[test_name] == None + tests[test_name] = defaultdict(lambda: None, { + "name": test_name, + "test_path": test_location_on_filesystem, + "failed": failed, + "skipped": skipped, + "passed": passed, + "time": time, + "test_output": test_output + }) + + return tests + +def print_summary(tests): + """ Print a summary of the test results + + Args: + tests (defaultdict[String]: { }): The tests that were reported by + : xunit + + """ + + assert tests is not None + + failed_tests = [] + passed_tests = [] + skipped_tests = [] + + for test in tests: + test = tests[test] + + if test["failed"] == "1": + failed_tests.append(test) + elif test["passed"] == "1": + passed_tests.append(test) + else: + skipped_tests.append(test) + + print + print "Total tests run: %d" % len(tests) + print + print "Total passing tests: %d" % len(passed_tests) + print "Total failed tests: %d" % len(failed_tests) + print "Total skipped tests: %d" % len(skipped_tests) + print + + failed_tests.sort(key=lambda item: item["time"], reverse=True) + passed_tests.sort(key=lambda item: item["time"], reverse=True) + skipped_tests.sort(key=lambda item: item["time"], reverse=True) + + def print_tests_helper(tests, stop_count): + for index, item in enumerate(tests): + time = item["time"] + unit = "seconds" + time_remainder = "" + second_unit = "" + saved_time = time + remainder_str = "" + + # If it can be expressed in hours + if time > 60**2: + time = saved_time / (60**2) + time_remainder = saved_time % (60**2) + time_remainder /= 60 + time_remainder = math.floor(time_remainder) + unit = "hours" + second_unit = "minutes" + + remainder_str = " %s %s" % (int(time_remainder), second_unit) + + elif time > 60 and time < 60**2: + time = saved_time / 60 + time_remainder = saved_time % 60 + time_remainder = math.floor(time_remainder) + unit = "minutes" + second_unit = "seconds" + + remainder_str = " %s %s" % (int(time_remainder), second_unit) + + print "%s (%d %s%s)" % (item["test_path"], time, unit, remainder_str) + + if stop_count != None: + if index >= stop_count: + break + + if len(failed_tests) > 0: + print "Failed tests:" + print + print_tests_helper(failed_tests, None) + + + if len(passed_tests) > 50: + print + print "50 slowest passing tests:" + print + print_tests_helper(passed_tests, 50) + + if len(failed_tests) > 0: + print + print "#################################################################" + print "Output of failing tests:" + print + + for item in failed_tests: + print "[%s]: " % item["test_path"] + print + + test_output = item["test_output"] + + # XUnit results are captured as escaped, escaped characters. + test_output = test_output.replace("\\r", "\r") + test_output = test_output.replace("\\n", "\n") + + print test_output + print + + print + print "#################################################################" + print "End of output of failing tests" + print "#################################################################" + print + +def create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_location, tests): + """ Go through the failing tests and create repros for them + + Args: + host_os (String) : os + arch (String) : architecture + build_type (String) : build configuration (debug, checked, release) + core_root (String) : Core_Root path + coreclr_repo_location (String) : Location of coreclr git repo + tests (defaultdict[String]: { }): The tests that were reported by + : xunit + + """ + assert tests is not None + + failed_tests = [tests[item] for item in tests if tests[item]["failed"] == "1"] + if len(failed_tests) == 0: + return + + bin_location = os.path.join(coreclr_repo_location, "bin") + assert os.path.isdir(bin_location) + + repro_location = os.path.join(bin_location, "repro", "%s.%s.%s" % (host_os, arch, build_type)) + if os.path.isdir(repro_location): + shutil.rmtree(repro_location) + + print "mkdir %s" % repro_location + os.makedirs(repro_location) + + print + print "Creating repo files, they can be found at: %s" % repro_location + + assert os.path.isdir(repro_location) + + # Now that the repro_location exists under /bin/repro + # create wrappers which will simply run the test with the correct environment + for test in failed_tests: + debug_env = DebugEnv(host_os, arch, build_type, env, core_root, coreclr_repo_location, test) + debug_env.write_repro() + + print "Repro files written." + print "They can be found at %s" % repro_location + +def do_setup(host_os, + arch, + build_type, + coreclr_repo_location, + product_location, + test_location, + test_native_bin_location, + core_root, + unprocessed_args, + test_env): + global gc_stress_c # Setup the tools for the repo. setup_tools(host_os, coreclr_repo_location) - env = get_environment() - ret_code = create_and_use_test_env(host_os, - env, - lambda path: run_tests(host_os, - arch, - build_type, - core_root, - coreclr_repo_location, - test_location, - test_native_bin_location, - test_env=path)) + if unprocessed_args.generate_layout: + success = setup_core_root(host_os, + arch, + build_type, + coreclr_repo_location, + test_native_bin_location, + product_location, + test_location, + core_root) + + if not success: + print "Error GenerateLayout has failed." + sys.exit(1) + + if unprocessed_args.generate_layout_only: + sys.exit(0) + + if unprocessed_args.precompile_core_root: + precompile_core_root(test_location, host_os, arch, core_root, use_jit_disasm=args.jitdisasm, altjit_name=unprocessed_args.crossgen_altjit) + + # If COMPlus_GCStress is set then we need to setup cordistools + if gc_stress_c: + setup_coredis_tools(coreclr_repo_location, host_os, arch, core_root) + + # Copy all the native libs to core_root + if host_os != "Windows_NT": + copy_native_test_bin_to_core_root(host_os, os.path.join(test_native_bin_location, "src"), core_root) + + correct_line_endings(host_os, test_location) + + if unprocessed_args.build_test_wrappers: + build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) + + run_tests(host_os, + arch, + build_type, + core_root, + coreclr_repo_location, + test_location, + test_native_bin_location, + is_illink=unprocessed_args.il_link, + is_long_gc=unprocessed_args.long_gc, + is_gcsimulator=unprocessed_args.gcsimulator, + is_jitdasm=unprocessed_args.jitdisasm, + is_ilasm=unprocessed_args.ilasmroundtrip, + run_sequential=unprocessed_args.sequential, + run_crossgen_tests=unprocessed_args.run_crossgen_tests, + test_env=test_env) + +################################################################################ +# Main +################################################################################ + +def main(args): + global g_verbose + g_verbose = args.verbose + + host_os, arch, build_type, coreclr_repo_location, product_location, core_root, test_location, test_native_bin_location = setup_args(args) + + env = get_environment(test_env=args.test_env) + if not args.analyze_results_only: + if args.test_env is not None: + ret_code = do_setup(host_os, + arch, + build_type, + coreclr_repo_location, + product_location, + test_location, + test_native_bin_location, + core_root, + args, + args.test_env) + else: + ret_code = create_and_use_test_env(host_os, + env, + lambda path: do_setup(host_os, + arch, + build_type, + coreclr_repo_location, + product_location, + test_location, + test_native_bin_location, + core_root, + args, + path)) + print "Test run finished." + + tests = parse_test_results(host_os, arch, build_type, coreclr_repo_location, test_location) + + if tests is not None: + print_summary(tests) + create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_location, tests) ################################################################################ # __main__ @@ -438,4 +1981,4 @@ def main(args): if __name__ == "__main__": args = parser.parse_args() - sys.exit(main(args)) \ No newline at end of file + sys.exit(main(args)) diff --git a/tests/src/CLRTest.GC.targets b/tests/src/CLRTest.GC.targets index f959aaf6e006..f28eca4b99d0 100644 --- a/tests/src/CLRTest.GC.targets +++ b/tests/src/CLRTest.GC.targets @@ -18,10 +18,7 @@ WARNING: When setting properties based on their current state (for example: --> - - 2 - 0 - + 0 $(CORE_ROOT)\corerun.exe $(CorerunExecutable) $(XunitConsoleRunner) @(TestAssemblies->'%(Identity)', ' ') $(XunitArgs) - + + +