diff --git a/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson b/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson index 5d74ba171d72..e02d55f1abe0 100644 --- a/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson @@ -69,6 +69,10 @@ "loc.input.help.configuration": "Build configuration against which the tests should be reported. If you have defined a variable for configuration in your build task, use that here.", "loc.input.label.publishRunAttachments": "Upload test attachments", "loc.input.help.publishRunAttachments": "Opt in/out of publishing run level attachments.", + "loc.input.label.failOnMinTestsNotRun": "Fail the task if a minimum number of tests are not run.", + "loc.input.help.failOnMinTestsNotRun": "Selecting this option will fail the task if specified minimum number of tests is not run.", + "loc.input.label.minimumExpectedTests": "Minimum # of tests", + "loc.input.help.minimumExpectedTests": "Specify the minimum # of tests that should be run for the task to succeed. Total tests executed is calculated as the sum of passed, failed and aborted tests.", "loc.input.label.diagnosticsEnabled": "Collect advanced diagnostics in case of catastrophic failures", "loc.input.help.diagnosticsEnabled": "Collect advanced diagnostics in case of catastrophic failures.", "loc.input.label.collectDumpOn": "Collect process dump and attach to test run report", diff --git a/Tasks/VsTestV2/distributedtest.ts b/Tasks/VsTestV2/distributedtest.ts index 54c8fa986a8e..4036e12601eb 100644 --- a/Tasks/VsTestV2/distributedtest.ts +++ b/Tasks/VsTestV2/distributedtest.ts @@ -75,7 +75,7 @@ export class DistributedTest { const dtaExecutionHostTool = tl.tool(path.join(__dirname, 'Modules/DTAExecutionHost.exe')); dtaExecutionHostTool.arg(['--inputFile', inputFilePath]); - const code = await dtaExecutionHostTool.exec({ ignoreReturnCode:this.inputDataContract.ExecutionSettings.IgnoreTestFailures, env: envVars }); + const code = await dtaExecutionHostTool.exec({ ignoreReturnCode:this.inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures, env: envVars }); //hydra: add consolidated ci for inputs in C# layer for now const consolidatedCiData = { diff --git a/Tasks/VsTestV2/inputdatacontract.ts b/Tasks/VsTestV2/inputdatacontract.ts index f70270bdf10b..6f1af12fc0ea 100644 --- a/Tasks/VsTestV2/inputdatacontract.ts +++ b/Tasks/VsTestV2/inputdatacontract.ts @@ -25,6 +25,7 @@ export interface TestReportingSettings { TestResultsDirectory : string; TestRunSystem : string; TestSourceSettings : TestSourceSettings; + ExecutionStatusSettings : ExecutionStatusSettings; } export interface TestSelectionSettings { @@ -95,8 +96,7 @@ export interface ExecutionSettings { DefaultTestBatchSize : number; AssemblyLevelParallelism : boolean; CodeCoverageEnabled : boolean; - PathToCustomTestAdapters : string; - IgnoreTestFailures : boolean; + PathToCustomTestAdapters : string; ProceedAfterAbortedTestCase : boolean; PathToCustomVsTestConsoleWrapperAssembly : string; SettingsFile : string; @@ -114,6 +114,12 @@ export interface TestSourceSettings { PullRequestTargetBranchName : string; } +export interface ExecutionStatusSettings { + IgnoreTestFailures : boolean; + MinimumExecutedTestsExpected : number; + ActionOnThresholdNotMet : string; +} + export interface DiagnosticsSettings { Enabled : boolean; DumpCollectionType : string; diff --git a/Tasks/VsTestV2/inputparser.ts b/Tasks/VsTestV2/inputparser.ts index 98cde68ced45..5f25f6f421e0 100644 --- a/Tasks/VsTestV2/inputparser.ts +++ b/Tasks/VsTestV2/inputparser.ts @@ -168,7 +168,8 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i inputDataContract.TestReportingSettings.TestSourceSettings = {}; inputDataContract.TestReportingSettings.TestSourceSettings.PullRequestTargetBranchName = tl.getVariable('System.PullRequest.TargetBranch'); - + inputDataContract.TestReportingSettings.ExecutionStatusSettings = {}; + inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures = utils.Helper.stringToBool(tl.getVariable('vstest.ignoretestfailures')); if (utils.Helper.isNullEmptyOrUndefined(inputDataContract.TestReportingSettings.TestRunTitle)) { let definitionName = tl.getVariable('BUILD_DEFINITIONNAME'); @@ -181,6 +182,20 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i inputDataContract.TestReportingSettings.TestRunTitle = `TestRun_${definitionName}_${buildOrReleaseName}`; } + + const actionOnThresholdNotMet = tl.getBoolInput('failOnMinTestsNotRun'); + if (actionOnThresholdNotMet) + { + inputDataContract.TestReportingSettings.ExecutionStatusSettings.ActionOnThresholdNotMet = "fail"; + const miniumExpectedTests = parseInt(tl.getInput('minimumExpectedTests')); + if (!isNaN(miniumExpectedTests)) { + inputDataContract.TestReportingSettings.ExecutionStatusSettings.MinimumExecutedTestsExpected = miniumExpectedTests; + console.log(tl.loc('minimumExpectedTests', inputDataContract.TestReportingSettings.ExecutionStatusSettings.MinimumExecutedTestsExpected)); + } else { + throw new Error(tl.loc('invalidMinimumExpectedTests :' + tl.getInput('minimumExpectedTests'))); + } + } + return inputDataContract; } @@ -346,8 +361,6 @@ function getExecutionSettings(inputDataContract : idc.InputDataContract) : idc.I } console.log(tl.loc('pathToCustomAdaptersInput', inputDataContract.ExecutionSettings.PathToCustomTestAdapters)); - inputDataContract.ExecutionSettings.IgnoreTestFailures = utils.Helper.stringToBool(tl.getVariable('vstest.ignoretestfailures')); - inputDataContract.ExecutionSettings.ProceedAfterAbortedTestCase = false; if (tl.getVariable('ProceedAfterAbortedTestCase') && tl.getVariable('ProceedAfterAbortedTestCase').toUpperCase() === 'TRUE') { inputDataContract.ExecutionSettings.ProceedAfterAbortedTestCase = true; diff --git a/Tasks/VsTestV2/make.json b/Tasks/VsTestV2/make.json index 2dad8141950c..6d5d36cace7b 100644 --- a/Tasks/VsTestV2/make.json +++ b/Tasks/VsTestV2/make.json @@ -6,7 +6,7 @@ "dest": "./" }, { - "url": "https://testexecution.blob.core.windows.net/testexecution/10738269/TestAgent.zip", + "url": "https://testexecution.blob.core.windows.net/testexecution/10889838/TestAgent.zip", "dest": "./Modules" }, { diff --git a/Tasks/VsTestV2/nondistributedtest.ts b/Tasks/VsTestV2/nondistributedtest.ts index 75210334a8b3..356e975232b2 100644 --- a/Tasks/VsTestV2/nondistributedtest.ts +++ b/Tasks/VsTestV2/nondistributedtest.ts @@ -27,7 +27,7 @@ export class NonDistributedTest { const exitCode = await this.startDtaExecutionHost(); tl.debug('DtaExecutionHost finished'); - if (exitCode !== 0 && !this.inputDataContract.ExecutionSettings.IgnoreTestFailures) { + if (exitCode !== 0 && !this.inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures) { tl.debug('Modules/DTAExecutionHost.exe process exited with code ' + exitCode); tl.setResult(tl.TaskResult.Failed, tl.loc('VstestFailed'), true); return; @@ -79,7 +79,7 @@ export class NonDistributedTest { } const execOptions: tr.IExecOptions = { - IgnoreTestFailures: this.inputDataContract.ExecutionSettings.IgnoreTestFailures, + IgnoreTestFailures: this.inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures, env: envVars, failOnStdErr: false, // In effect this will not be called as failOnStdErr is false diff --git a/Tasks/VsTestV2/task.json b/Tasks/VsTestV2/task.json index 24dbdcbe96be..70ef38d76410 100644 --- a/Tasks/VsTestV2/task.json +++ b/Tasks/VsTestV2/task.json @@ -17,8 +17,8 @@ "author": "Microsoft Corporation", "version": { "Major": 2, - "Minor": 160, - "Patch": 2 + "Minor": 161, + "Patch": 0 }, "demands": [ "vstest" @@ -407,6 +407,25 @@ "helpMarkDown": "Opt in/out of publishing run level attachments.", "groupName": "reportingOptions" }, + { + "name": "failOnMinTestsNotRun", + "type": "boolean", + "label": "Fail the task if a minimum number of tests are not run.", + "defaultValue": "False", + "required": false, + "helpMarkDown": "Selecting this option will fail the task if specified minimum number of tests is not run.", + "groupName": "reportingOptions" + }, + { + "name": "minimumExpectedTests", + "type": "string", + "label": "Minimum # of tests", + "defaultValue": "1", + "required": false, + "helpMarkDown": "Specify the minimum # of tests that should be run for the task to succeed. Total tests executed is calculated as the sum of passed, failed and aborted tests.", + "groupName": "reportingOptions", + "visibleRule": "failOnMinTestsNotRun = true" + }, { "name": "diagnosticsEnabled", "type": "boolean", diff --git a/Tasks/VsTestV2/task.loc.json b/Tasks/VsTestV2/task.loc.json index a35318d93ecb..91b501350207 100644 --- a/Tasks/VsTestV2/task.loc.json +++ b/Tasks/VsTestV2/task.loc.json @@ -17,8 +17,8 @@ "author": "Microsoft Corporation", "version": { "Major": 2, - "Minor": 160, - "Patch": 2 + "Minor": 161, + "Patch": 0 }, "demands": [ "vstest" @@ -407,6 +407,25 @@ "helpMarkDown": "ms-resource:loc.input.help.publishRunAttachments", "groupName": "reportingOptions" }, + { + "name": "failOnMinTestsNotRun", + "type": "boolean", + "label": "ms-resource:loc.input.label.failOnMinTestsNotRun", + "defaultValue": "False", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.failOnMinTestsNotRun", + "groupName": "reportingOptions" + }, + { + "name": "minimumExpectedTests", + "type": "string", + "label": "ms-resource:loc.input.label.minimumExpectedTests", + "defaultValue": "1", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.minimumExpectedTests", + "groupName": "reportingOptions", + "visibleRule": "failOnMinTestsNotRun = true" + }, { "name": "diagnosticsEnabled", "type": "boolean", diff --git a/package-lock.json b/package-lock.json index f8da4bfb0614..ee541e4b6f2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -677,7 +677,7 @@ "qs": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==", + "integrity": "sha1-NJzfbu+J7EXBLX1es/wMhwNDptg=", "dev": true }, "randomatic": {