Skip to content
This repository has been archived by the owner on Jan 23, 2023. It is now read-only.

Commit

Permalink
Add parallel test execution
Browse files Browse the repository at this point in the history
  • Loading branch information
A-And committed Jun 14, 2018
1 parent 122b987 commit 8a435fd
Show file tree
Hide file tree
Showing 6 changed files with 309 additions and 111 deletions.
34 changes: 13 additions & 21 deletions tests/runtest.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -329,32 +329,24 @@ if not exist %_CoreFXTestHost%\dotnet.exe echo CoreFX test host not found, pleas
set /p _CoreFXTestRemoteURL=< "%__ProjectFilesDir%\CoreFX\CoreFXTestListURL.txt"
if not defined __CoreFXTestList ( set __CoreFXTestList=%__ProjectFilesDir%\CoreFX\TopN.CoreFX.Windows.issues.json )

echo Downloading CoreFX Test Binaries
echo "%_dotnet%" "%_CoreFXTestUtilitiesOutputPath%\%_CoreFXTestSetupUtilityName%.dll" --clean --outputDirectory "%_CoreFXTestBinariesPath%" --testListJsonPath "%__CoreFXTestList%" --testUrl "%_CoreFXTestRemoteURL%"
call "%_dotnet%" "%_CoreFXTestUtilitiesOutputPath%\%_CoreFXTestSetupUtilityName%.dll" --clean --outputDirectory "%_CoreFXTestBinariesPath%" --testListJsonPath "%__CoreFXTestList%" --testUrl "%_CoreFXTestRemoteURL%"
if errorlevel 1 (
exit /b 1
)

set _CoreFXTestExecutable=xunit.console.netcore.exe
set _CoreFXTestExecutableArgs=-notrait category=nonnetcoreapptests -notrait category=nonwindowstests -notrait category=failing -notrait category=IgnoreForCI
set _CoreFXTestExecutableArgs= --notrait nonnetcoreapptests --notrait nonwindowstests --notrait failing --notrait IgnoreForCI

REM Set the log file name to something Jenkins can understand
set _CoreFX_TestLogFileName=testResults.xml
for /D %%i in ("%_CoreFXTestBinariesPath%\*") do (
pushd %%i
if not exist "%%i\%_CoreFXTestExecutable%" echo "Error running CoreFX tests - %_CoreFXTestExecutable% not found" && exit /b 1

set _TestName=%%~nxi
set _LogPath=%_CoreFXLogsDir%\!_TestName!
if not exist "!_LogPath!" (mkdir "!_LogPath!")

echo %__MsgPrefix%Running !_TestName!
echo Writing logs to !_LogPath!
echo To reproduce directly run:
echo "%_CoreFXTestHost%\dotnet.exe" "%%i\%_CoreFXTestExecutable%" "%%i\!_TestName!.dll" @"%%i\!_TestName!.rsp" -xml "!_LogPath!\%_CoreFX_TestLogFileName%" %_CoreFXTestExecutableArgs%
call "%_CoreFXTestHost%\dotnet.exe" "%%i\%_CoreFXTestExecutable%" "%%i\!_TestName!.dll" @"%%i\!_TestName!.rsp" -xml "!_LogPath!\\%_CoreFX_TestLogFileName%" %_CoreFXTestExecutableArgs%
popd
set _CoreFX_TestRunScriptName=CoreCLR_RunTest.cmd


echo Downloading and Running CoreFX Test Binaries
echo call "%_dotnet%" "%_CoreFXTestUtilitiesOutputPath%\%_CoreFXTestSetupUtilityName%.dll" --clean --outputDirectory "%_CoreFXTestBinariesPath%" --testListJsonPath "%__CoreFXTestList%" --testUrl "%_CoreFXTestRemoteURL%" --runTests --dotnetPath "%_CoreFXTestHost%\dotnet.exe" --executable %_CoreFXTestExecutable% --logPath %_CoreFXLogsDir% --maxProcessCount 5 %_CoreFXTestExecutableArgs%
call "%_dotnet%" "%_CoreFXTestUtilitiesOutputPath%\%_CoreFXTestSetupUtilityName%.dll" --clean --outputDirectory "%_CoreFXTestBinariesPath%" --testListJsonPath "%__CoreFXTestList%" --testUrl "%_CoreFXTestRemoteURL%" --runTests --dotnetPath "%_CoreFXTestHost%\dotnet.exe" --executable %_CoreFXTestExecutable% --log %_CoreFXLogsDir% --maxProcessCount 5 %_CoreFXTestExecutableArgs%
if errorlevel 1 (
echo %__MsgPrefix%Running CoreFX tests finished with Failures
echo %__MsgPrefix%Check %_CoreFXLogsDir% for test run logs
exit /b 1
)

)
goto TestsDone

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
using Newtonsoft.Json.Schema;
using Newtonsoft.Json.Schema.Generation;

namespace CoreFX.TestUtils.TestFileSetup
namespace CoreFX.TestUtils.TestFileSetup.Helpers
{
/// <summary>
/// Defines the set of flags that represent exit codes
Expand All @@ -24,9 +24,10 @@ namespace CoreFX.TestUtils.TestFileSetup
public enum ExitCode : int
{
Success = 0,
HttpError = 1,
IOError = 2,
JsonSchemaValidationError = 3,
TestFailure = 1,
HttpError = 2,
IOError = 3,
JsonSchemaValidationError = 4,
UnknownError = 10

}
Expand All @@ -35,83 +36,23 @@ public enum ExitCode : int
/// This helper class is used to fetch CoreFX tests from a specified URL, unarchive them and create a flat directory structure
/// through which to iterate.
/// </summary>
public static class TestFileSetup
public class TestFileHelper
{
private static HttpClient httpClient;
private static bool cleanTestBuild = false;

private static string outputDir;
private static string testUrl;
private static string testListPath;

public static void Main(string[] args)
private HttpClient httpClient;
public HttpClient HttpClient
{
ExitCode exitCode = ExitCode.UnknownError;
ArgumentSyntax argSyntax = ParseCommandLine(args);

try
get
{
if (!Directory.Exists(outputDir))
Directory.CreateDirectory(outputDir);

if (cleanTestBuild)
if (httpClient == null)
{
CleanBuild(outputDir);
httpClient = new HttpClient();
}

// Map test names to their definitions
Dictionary<string, XUnitTestAssembly> testAssemblyDefinitions = DeserializeTestJson(testListPath);

SetupTests(testUrl, outputDir, testAssemblyDefinitions).Wait();
exitCode = ExitCode.Success;
return httpClient;
}

catch (AggregateException e)
{
e.Handle(innerExc =>
{

if (innerExc is HttpRequestException)
{
exitCode = ExitCode.HttpError;
Console.WriteLine("Error downloading tests from: " + testUrl);
Console.WriteLine(innerExc.Message);
return true;
}
else if (innerExc is IOException)
{
exitCode = ExitCode.IOError;
Console.WriteLine(innerExc.Message);
return true;
}
else if (innerExc is JSchemaValidationException || innerExc is JsonSerializationException)
{
exitCode = ExitCode.JsonSchemaValidationError;
Console.WriteLine("Error validating test list: ");
Console.WriteLine(innerExc.Message);
return true;
}
return false;
});
}

Environment.Exit((int)exitCode);
}

private static ArgumentSyntax ParseCommandLine(string[] args)
{
ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax =>
{
syntax.DefineOption("out|outDir|outputDirectory", ref outputDir, "Directory where tests are downloaded");
syntax.DefineOption("testUrl", ref testUrl, "URL, pointing to the list of tests");
syntax.DefineOption("testListJsonPath", ref testListPath, "JSON-formatted list of test assembly names to download");
syntax.DefineOption("clean|cleanOutputDir", ref cleanTestBuild, "Clean test assembly output directory");
});

return argSyntax;
set{ httpClient = value; }
}

private static Dictionary<string, XUnitTestAssembly> DeserializeTestJson(string testDefinitionFilePath)
public Dictionary<string, XUnitTestAssembly> DeserializeTestJson(string testDefinitionFilePath)
{
JSchemaGenerator jsonGenerator = new JSchemaGenerator();

Expand Down Expand Up @@ -166,7 +107,7 @@ private static Dictionary<string, XUnitTestAssembly> DeserializeTestJson(string
return nameToTestAssemblyDef;
}

private static async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false)
public async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false)
{
Debug.Assert(Directory.Exists(destinationDirectory));
Debug.Assert(runAllTests || testDefinitions != null);
Expand Down Expand Up @@ -195,16 +136,11 @@ private static async Task SetupTests(string jsonUrl, string destinationDirectory
Directory.Delete(tempDirPath);
}

private static async Task<Dictionary<string, XUnitTestAssembly>> GetTestUrls(string jsonUrl, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false)
public async Task<Dictionary<string, XUnitTestAssembly>> GetTestUrls(string jsonUrl, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false)
{
if (httpClient is null)
{
httpClient = new HttpClient();
}

Debug.Assert(runAllTests || testDefinitions != null);
// Set up the json stream reader
using (var responseStream = await httpClient.GetStreamAsync(jsonUrl))
using (var responseStream = await HttpClient.GetStreamAsync(jsonUrl))
using (var streamReader = new StreamReader(responseStream))
using (var jsonReader = new JsonTextReader(streamReader))
{
Expand Down Expand Up @@ -249,21 +185,16 @@ private static async Task<Dictionary<string, XUnitTestAssembly>> GetTestUrls(str
return testDefinitions;
}

private static async Task GetTestArchives(Dictionary<string, XUnitTestAssembly> testPayloads, string downloadDir)
public async Task GetTestArchives(Dictionary<string, XUnitTestAssembly> testPayloads, string downloadDir)
{
if (httpClient is null)
{
httpClient = new HttpClient();
}

foreach (string testName in testPayloads.Keys)
{
string payloadUri = testPayloads[testName].Url;

if (!Uri.IsWellFormedUriString(payloadUri, UriKind.Absolute))
continue;

using (var response = await httpClient.GetStreamAsync(payloadUri))
using (var response = await HttpClient.GetStreamAsync(payloadUri))
{
if (response.CanRead)
{
Expand All @@ -287,7 +218,7 @@ private static async Task GetTestArchives(Dictionary<string, XUnitTestAssembly>
}
}

private static void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true)
public void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true)
{
Debug.Assert(Directory.Exists(archiveDirectory));
Debug.Assert(Directory.Exists(destinationDirectory));
Expand All @@ -309,7 +240,7 @@ private static void ExpandArchivesInDirectory(string archiveDirectory, string de
}
}

private static void CleanBuild(string directoryToClean)
public void CleanBuild(string directoryToClean)
{
Debug.Assert(Directory.Exists(directoryToClean));
DirectoryInfo dirInfo = new DirectoryInfo(directoryToClean);
Expand Down
137 changes: 137 additions & 0 deletions tests/src/Common/CoreFX/TestFileSetup/Helpers/TestRunHelper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;

namespace CoreFX.TestUtils.TestFileSetup.Helpers
{
public class NetCoreTestRunHelper
{
public string DotnetExecutablePath { get; set; }

public string logRootOutputPath { get; set; }

public int TestRunExitCode { get; set; }


public NetCoreTestRunHelper(string DotnetExecutablePath, string logRootOutputPath)
{
this.DotnetExecutablePath = DotnetExecutablePath;
this.logRootOutputPath = logRootOutputPath;
}

public int RunExecutable(string workingDirectory, string executableName, IReadOnlyList<string> xunitTestTraits, string logRootOutputPath)
{
string logPath = Path.Combine(logRootOutputPath, Path.GetFileName(workingDirectory));
if (!Directory.Exists(logPath))
Directory.CreateDirectory(logPath);
string arguments = CalculateCommandLineArguments(workingDirectory, executableName, xunitTestTraits, Path.Combine(logPath,"testResults.xml"));

ProcessStartInfo startInfo = new ProcessStartInfo(DotnetExecutablePath, arguments)
{
Arguments = arguments,
WorkingDirectory = workingDirectory
};


Process executableProcess = new Process();
executableProcess.StartInfo = startInfo;
executableProcess.EnableRaisingEvents = true;
executableProcess.Exited += new EventHandler(ExitEventHandler);
executableProcess.Start();
executableProcess.WaitForExit();

return executableProcess.ExitCode;
}

private void ExitEventHandler(object sender, EventArgs e)
{
TestRunExitCode = (sender as Process).ExitCode;
}

public int RunAllExecutablesInDirectory(string rootDirectory, string executableName, IReadOnlyList<string> xunitTestTraits, int processLimit, string logRootOutputPath = null)
{
int result = 0;
// Do a Depth-First Search to find and run executables with the same name
Stack<string> directories = new Stack<string>();
List<string> testDirectories = new List<string>();
// Push rootdir
directories.Push(rootDirectory);

while (directories.Count > 0)
{
string currentDirectory = directories.Pop();

if (File.Exists(Path.Combine(currentDirectory, executableName)))
testDirectories.Add(currentDirectory);

foreach (string subDir in Directory.GetDirectories(currentDirectory))
directories.Push(subDir);
}

ParallelOptions parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = processLimit;

Parallel.ForEach(testDirectories, parallelOptions,
(testDirectory) =>
{
if (RunExecutable(testDirectory, executableName, xunitTestTraits, logRootOutputPath) != 0)
result = 1;
}
);
return result;
}

private string CalculateCommandLineArguments(string testDirectory, string executableName, IReadOnlyList<string> xunitTestTraits, string logPath)
{
StringBuilder arguments = new StringBuilder();

arguments.Append("\"");
arguments.Append(Path.Combine(testDirectory, Path.GetFileName(executableName)));
arguments.Append("\"");
arguments.Append(" ");

// Append test name dll
arguments.Append("\"");
arguments.Append(Path.Combine(testDirectory, Path.GetFileName(testDirectory)));
arguments.Append(".dll");
arguments.Append("\"");

arguments.Append(" ");

// Append RSP file
arguments.Append("@");
arguments.Append("\"");
arguments.Append(Path.Combine(testDirectory, Path.GetFileName(testDirectory)));
arguments.Append(".rsp");
arguments.Append("\"");
arguments.Append(" ");

if (!String.IsNullOrEmpty(logPath))
{
// Add logging information
arguments.Append("-xml");
arguments.Append(" ");
arguments.Append(logPath);
arguments.Append(" ");
}

// Append all additional arguments
foreach (string traitToExclude in xunitTestTraits)
{
arguments.Append("-notrait");
arguments.Append(" ");
arguments.Append("category=");

arguments.Append(traitToExclude);
arguments.Append(" ");
}


return arguments.ToString();
}
}
}
Loading

0 comments on commit 8a435fd

Please sign in to comment.