From fe028a8144aabfc93e4972eb8b6689259ae333b6 Mon Sep 17 00:00:00 2001 From: Vagisha Nidhi Date: Mon, 28 Oct 2019 13:28:34 +0530 Subject: [PATCH 1/7] Fail vstest task if minimum specified tests not executed --- .../resources.resjson/en-US/resources.resjson | 4 ++++ Tasks/VsTestV2/inputdatacontract.ts | 6 ++++++ Tasks/VsTestV2/inputparser.ts | 17 +++++++++++++++ Tasks/VsTestV2/task.json | 21 ++++++++++++++++++- Tasks/VsTestV2/task.loc.json | 21 ++++++++++++++++++- package-lock.json | 2 +- 6 files changed, 68 insertions(+), 3 deletions(-) diff --git a/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson b/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson index 5d74ba171d72..0b48842d60ec 100644 --- a/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson @@ -69,6 +69,10 @@ "loc.input.help.configuration": "Build configuration against which the tests should be reported. If you have defined a variable for configuration in your build task, use that here.", "loc.input.label.publishRunAttachments": "Upload test attachments", "loc.input.help.publishRunAttachments": "Opt in/out of publishing run level attachments.", + "loc.input.label.failOnMinTestsNotRun": "Fail task if a minimum number of tests are not run", + "loc.input.help.failOnMinTestsNotRun": "Selecting this option will fail the task if specified minimum number of tests is not run.", + "loc.input.label.minimumExpectedTests": "Minimum # of tests", + "loc.input.help.minimumExpectedTests": "Specify the minimum # of tests that should be executed to not fail the task.", "loc.input.label.diagnosticsEnabled": "Collect advanced diagnostics in case of catastrophic failures", "loc.input.help.diagnosticsEnabled": "Collect advanced diagnostics in case of catastrophic failures.", "loc.input.label.collectDumpOn": "Collect process dump and attach to test run report", diff --git a/Tasks/VsTestV2/inputdatacontract.ts b/Tasks/VsTestV2/inputdatacontract.ts index f70270bdf10b..a34031e51c8c 100644 --- a/Tasks/VsTestV2/inputdatacontract.ts +++ b/Tasks/VsTestV2/inputdatacontract.ts @@ -25,6 +25,7 @@ export interface TestReportingSettings { TestResultsDirectory : string; TestRunSystem : string; TestSourceSettings : TestSourceSettings; + ExecutionStatusSettings: MinimumTestsExecutionStatusSettings; } export interface TestSelectionSettings { @@ -114,6 +115,11 @@ export interface TestSourceSettings { PullRequestTargetBranchName : string; } +export interface MinimumTestsExecutionStatusSettings { + MinimumExecutedTestsExpected : number; + ActionOnThresholdNotMet : string; +} + export interface DiagnosticsSettings { Enabled : boolean; DumpCollectionType : string; diff --git a/Tasks/VsTestV2/inputparser.ts b/Tasks/VsTestV2/inputparser.ts index 98cde68ced45..c16027e8b50c 100644 --- a/Tasks/VsTestV2/inputparser.ts +++ b/Tasks/VsTestV2/inputparser.ts @@ -181,6 +181,23 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i inputDataContract.TestReportingSettings.TestRunTitle = `TestRun_${definitionName}_${buildOrReleaseName}`; } + + const actionOnThresholdNotMet = tl.getInput('failOnMinTestsNotRun'); + + if (actionOnThresholdNotMet) + { + inputDataContract.TestReportingSettings.ExecutionStatusSettings = {}; + inputDataContract.TestReportingSettings.ExecutionStatusSettings.ActionOnThresholdNotMet = "fail"; + + const miniumExpectedTests = parseInt(tl.getInput('minimumExpectedTests')); + if (!isNaN(miniumExpectedTests)) { + inputDataContract.TestReportingSettings.ExecutionStatusSettings.MinimumExecutedTestsExpected = miniumExpectedTests; + console.log(tl.loc('minimumExpectedTests', inputDataContract.TestReportingSettings.ExecutionStatusSettings.MinimumExecutedTestsExpected)); + } else { + tl.warning(tl.loc('invalidMinimumExpectedTests')); + } + } + return inputDataContract; } diff --git a/Tasks/VsTestV2/task.json b/Tasks/VsTestV2/task.json index 88b91fe0c649..943c0f22eb6f 100644 --- a/Tasks/VsTestV2/task.json +++ b/Tasks/VsTestV2/task.json @@ -18,7 +18,7 @@ "version": { "Major": 2, "Minor": 160, - "Patch": 1 + "Patch": 2 }, "demands": [ "vstest" @@ -407,6 +407,25 @@ "helpMarkDown": "Opt in/out of publishing run level attachments.", "groupName": "reportingOptions" }, + { + "name": "failOnMinTestsNotRun", + "type": "boolean", + "label": "Fail task if a minimum number of tests are not run", + "defaultValue": "False", + "required": false, + "helpMarkDown": "Selecting this option will fail the task if specified minimum number of tests is not run.", + "groupName": "reportingOptions" + }, + { + "name": "minimumExpectedTests", + "type": "string", + "label": "Minimum # of tests", + "defaultValue": "1", + "required": false, + "helpMarkDown": "Specify the minimum # of tests that should be executed to not fail the task.", + "groupName": "reportingOptions", + "visibleRule": "failOnMinTestsNotRun = true" + }, { "name": "diagnosticsEnabled", "type": "boolean", diff --git a/Tasks/VsTestV2/task.loc.json b/Tasks/VsTestV2/task.loc.json index a8efdeeacf5f..902a43d7a034 100644 --- a/Tasks/VsTestV2/task.loc.json +++ b/Tasks/VsTestV2/task.loc.json @@ -18,7 +18,7 @@ "version": { "Major": 2, "Minor": 160, - "Patch": 1 + "Patch": 2 }, "demands": [ "vstest" @@ -407,6 +407,25 @@ "helpMarkDown": "ms-resource:loc.input.help.publishRunAttachments", "groupName": "reportingOptions" }, + { + "name": "failOnMinTestsNotRun", + "type": "boolean", + "label": "ms-resource:loc.input.label.failOnMinTestsNotRun", + "defaultValue": "False", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.failOnMinTestsNotRun", + "groupName": "reportingOptions" + }, + { + "name": "minimumExpectedTests", + "type": "string", + "label": "ms-resource:loc.input.label.minimumExpectedTests", + "defaultValue": "1", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.minimumExpectedTests", + "groupName": "reportingOptions", + "visibleRule": "failOnMinTestsNotRun = true" + }, { "name": "diagnosticsEnabled", "type": "boolean", diff --git a/package-lock.json b/package-lock.json index e17d2171172e..7824989136f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -677,7 +677,7 @@ "qs": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==", + "integrity": "sha1-NJzfbu+J7EXBLX1es/wMhwNDptg=", "dev": true }, "randomatic": { From d08c60af75617fa1262f3ded65361ba2dcad0217 Mon Sep 17 00:00:00 2001 From: Vagisha Nidhi Date: Wed, 30 Oct 2019 19:50:16 +0530 Subject: [PATCH 2/7] Addressing review comments --- .../Strings/resources.resjson/en-US/resources.resjson | 4 ++-- Tasks/VsTestV2/inputparser.ts | 4 ++-- Tasks/VsTestV2/task.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson b/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson index 0b48842d60ec..e02d55f1abe0 100644 --- a/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/VsTestV2/Strings/resources.resjson/en-US/resources.resjson @@ -69,10 +69,10 @@ "loc.input.help.configuration": "Build configuration against which the tests should be reported. If you have defined a variable for configuration in your build task, use that here.", "loc.input.label.publishRunAttachments": "Upload test attachments", "loc.input.help.publishRunAttachments": "Opt in/out of publishing run level attachments.", - "loc.input.label.failOnMinTestsNotRun": "Fail task if a minimum number of tests are not run", + "loc.input.label.failOnMinTestsNotRun": "Fail the task if a minimum number of tests are not run.", "loc.input.help.failOnMinTestsNotRun": "Selecting this option will fail the task if specified minimum number of tests is not run.", "loc.input.label.minimumExpectedTests": "Minimum # of tests", - "loc.input.help.minimumExpectedTests": "Specify the minimum # of tests that should be executed to not fail the task.", + "loc.input.help.minimumExpectedTests": "Specify the minimum # of tests that should be run for the task to succeed. Total tests executed is calculated as the sum of passed, failed and aborted tests.", "loc.input.label.diagnosticsEnabled": "Collect advanced diagnostics in case of catastrophic failures", "loc.input.help.diagnosticsEnabled": "Collect advanced diagnostics in case of catastrophic failures.", "loc.input.label.collectDumpOn": "Collect process dump and attach to test run report", diff --git a/Tasks/VsTestV2/inputparser.ts b/Tasks/VsTestV2/inputparser.ts index c16027e8b50c..5afc1e2327f0 100644 --- a/Tasks/VsTestV2/inputparser.ts +++ b/Tasks/VsTestV2/inputparser.ts @@ -182,7 +182,7 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i inputDataContract.TestReportingSettings.TestRunTitle = `TestRun_${definitionName}_${buildOrReleaseName}`; } - const actionOnThresholdNotMet = tl.getInput('failOnMinTestsNotRun'); + const actionOnThresholdNotMet = tl.getBoolInput('failOnMinTestsNotRun'); if (actionOnThresholdNotMet) { @@ -194,7 +194,7 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i inputDataContract.TestReportingSettings.ExecutionStatusSettings.MinimumExecutedTestsExpected = miniumExpectedTests; console.log(tl.loc('minimumExpectedTests', inputDataContract.TestReportingSettings.ExecutionStatusSettings.MinimumExecutedTestsExpected)); } else { - tl.warning(tl.loc('invalidMinimumExpectedTests')); + throw new Error(tl.loc('invalidMinimumExpectedTests :' + tl.getInput('minimumExpectedTests'))); } } diff --git a/Tasks/VsTestV2/task.json b/Tasks/VsTestV2/task.json index 943c0f22eb6f..6e8c9659948e 100644 --- a/Tasks/VsTestV2/task.json +++ b/Tasks/VsTestV2/task.json @@ -410,7 +410,7 @@ { "name": "failOnMinTestsNotRun", "type": "boolean", - "label": "Fail task if a minimum number of tests are not run", + "label": "Fail the task if a minimum number of tests are not run.", "defaultValue": "False", "required": false, "helpMarkDown": "Selecting this option will fail the task if specified minimum number of tests is not run.", @@ -422,7 +422,7 @@ "label": "Minimum # of tests", "defaultValue": "1", "required": false, - "helpMarkDown": "Specify the minimum # of tests that should be executed to not fail the task.", + "helpMarkDown": "Specify the minimum # of tests that should be run for the task to succeed. Total tests executed is calculated as the sum of passed, failed and aborted tests.", "groupName": "reportingOptions", "visibleRule": "failOnMinTestsNotRun = true" }, From f1c5444764b48dc5623dafb6c39328c327f37e3c Mon Sep 17 00:00:00 2001 From: Vagisha Nidhi Date: Fri, 1 Nov 2019 18:01:00 +0530 Subject: [PATCH 3/7] Making ignoretestfailures part of ExecutionStatusSettings --- Tasks/VsTestV2/distributedtest.ts | 2 +- Tasks/VsTestV2/inputdatacontract.ts | 4 ++-- Tasks/VsTestV2/inputparser.ts | 10 +++------- Tasks/VsTestV2/nondistributedtest.ts | 4 ++-- Tasks/VsTestV2/task.json | 4 ++-- Tasks/VsTestV2/task.loc.json | 4 ++-- 6 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Tasks/VsTestV2/distributedtest.ts b/Tasks/VsTestV2/distributedtest.ts index f5401b93eb2a..1a82e861e9e2 100644 --- a/Tasks/VsTestV2/distributedtest.ts +++ b/Tasks/VsTestV2/distributedtest.ts @@ -75,7 +75,7 @@ export class DistributedTest { const dtaExecutionHostTool = tl.tool(path.join(__dirname, 'Modules/DTAExecutionHost.exe')); dtaExecutionHostTool.arg(['--inputFile', inputFilePath]); - const code = await dtaExecutionHostTool.exec({ ignoreReturnCode:this.inputDataContract.ExecutionSettings.IgnoreTestFailures, env: envVars }); + const code = await dtaExecutionHostTool.exec({ ignoreReturnCode:this.inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures, env: envVars }); //hydra: add consolidated ci for inputs in C# layer for now const consolidatedCiData = { diff --git a/Tasks/VsTestV2/inputdatacontract.ts b/Tasks/VsTestV2/inputdatacontract.ts index a34031e51c8c..6dbbb9552fbd 100644 --- a/Tasks/VsTestV2/inputdatacontract.ts +++ b/Tasks/VsTestV2/inputdatacontract.ts @@ -96,8 +96,7 @@ export interface ExecutionSettings { DefaultTestBatchSize : number; AssemblyLevelParallelism : boolean; CodeCoverageEnabled : boolean; - PathToCustomTestAdapters : string; - IgnoreTestFailures : boolean; + PathToCustomTestAdapters : string; ProceedAfterAbortedTestCase : boolean; PathToCustomVsTestConsoleWrapperAssembly : string; SettingsFile : string; @@ -116,6 +115,7 @@ export interface TestSourceSettings { } export interface MinimumTestsExecutionStatusSettings { + IgnoreTestFailures : boolean; MinimumExecutedTestsExpected : number; ActionOnThresholdNotMet : string; } diff --git a/Tasks/VsTestV2/inputparser.ts b/Tasks/VsTestV2/inputparser.ts index 5afc1e2327f0..61db1d078cbe 100644 --- a/Tasks/VsTestV2/inputparser.ts +++ b/Tasks/VsTestV2/inputparser.ts @@ -168,7 +168,8 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i inputDataContract.TestReportingSettings.TestSourceSettings = {}; inputDataContract.TestReportingSettings.TestSourceSettings.PullRequestTargetBranchName = tl.getVariable('System.PullRequest.TargetBranch'); - + inputDataContract.TestReportingSettings.ExecutionStatusSettings = {}; + inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures = utils.Helper.stringToBool(tl.getVariable('vstest.ignoretestfailures')); if (utils.Helper.isNullEmptyOrUndefined(inputDataContract.TestReportingSettings.TestRunTitle)) { let definitionName = tl.getVariable('BUILD_DEFINITIONNAME'); @@ -183,12 +184,9 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i } const actionOnThresholdNotMet = tl.getBoolInput('failOnMinTestsNotRun'); - if (actionOnThresholdNotMet) { - inputDataContract.TestReportingSettings.ExecutionStatusSettings = {}; - inputDataContract.TestReportingSettings.ExecutionStatusSettings.ActionOnThresholdNotMet = "fail"; - + inputDataContract.TestReportingSettings.ExecutionStatusSettings.ActionOnThresholdNotMet = "fail"; const miniumExpectedTests = parseInt(tl.getInput('minimumExpectedTests')); if (!isNaN(miniumExpectedTests)) { inputDataContract.TestReportingSettings.ExecutionStatusSettings.MinimumExecutedTestsExpected = miniumExpectedTests; @@ -363,8 +361,6 @@ function getExecutionSettings(inputDataContract : idc.InputDataContract) : idc.I } console.log(tl.loc('pathToCustomAdaptersInput', inputDataContract.ExecutionSettings.PathToCustomTestAdapters)); - inputDataContract.ExecutionSettings.IgnoreTestFailures = utils.Helper.stringToBool(tl.getVariable('vstest.ignoretestfailures')); - inputDataContract.ExecutionSettings.ProceedAfterAbortedTestCase = false; if (tl.getVariable('ProceedAfterAbortedTestCase') && tl.getVariable('ProceedAfterAbortedTestCase').toUpperCase() === 'TRUE') { inputDataContract.ExecutionSettings.ProceedAfterAbortedTestCase = true; diff --git a/Tasks/VsTestV2/nondistributedtest.ts b/Tasks/VsTestV2/nondistributedtest.ts index 58c170673d97..ecfa12721adc 100644 --- a/Tasks/VsTestV2/nondistributedtest.ts +++ b/Tasks/VsTestV2/nondistributedtest.ts @@ -34,7 +34,7 @@ export class NonDistributedTest { const exitCode = await this.startDtaExecutionHost(); tl.debug('DtaExecutionHost finished'); - if (exitCode !== 0 && !this.inputDataContract.ExecutionSettings.IgnoreTestFailures) { + if (exitCode !== 0 && !this.inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures) { tl.debug('Modules/DTAExecutionHost.exe process exited with code ' + exitCode); tl.setResult(tl.TaskResult.Failed, tl.loc('VstestFailed'), true); return; @@ -86,7 +86,7 @@ export class NonDistributedTest { } const execOptions: tr.IExecOptions = { - IgnoreTestFailures: this.inputDataContract.ExecutionSettings.IgnoreTestFailures, + IgnoreTestFailures: this.inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures, env: envVars, failOnStdErr: false, // In effect this will not be called as failOnStdErr is false diff --git a/Tasks/VsTestV2/task.json b/Tasks/VsTestV2/task.json index 6e8c9659948e..70ef38d76410 100644 --- a/Tasks/VsTestV2/task.json +++ b/Tasks/VsTestV2/task.json @@ -17,8 +17,8 @@ "author": "Microsoft Corporation", "version": { "Major": 2, - "Minor": 160, - "Patch": 2 + "Minor": 161, + "Patch": 0 }, "demands": [ "vstest" diff --git a/Tasks/VsTestV2/task.loc.json b/Tasks/VsTestV2/task.loc.json index 902a43d7a034..91b501350207 100644 --- a/Tasks/VsTestV2/task.loc.json +++ b/Tasks/VsTestV2/task.loc.json @@ -17,8 +17,8 @@ "author": "Microsoft Corporation", "version": { "Major": 2, - "Minor": 160, - "Patch": 2 + "Minor": 161, + "Patch": 0 }, "demands": [ "vstest" From f69d31672b251d06cc767e0ce00a6e01889f42aa Mon Sep 17 00:00:00 2001 From: Vagisha Nidhi Date: Wed, 13 Nov 2019 14:27:02 +0530 Subject: [PATCH 4/7] Updated inputdatacontract --- Tasks/VsTestV2/inputdatacontract.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tasks/VsTestV2/inputdatacontract.ts b/Tasks/VsTestV2/inputdatacontract.ts index 6dbbb9552fbd..b080a031e53f 100644 --- a/Tasks/VsTestV2/inputdatacontract.ts +++ b/Tasks/VsTestV2/inputdatacontract.ts @@ -25,7 +25,7 @@ export interface TestReportingSettings { TestResultsDirectory : string; TestRunSystem : string; TestSourceSettings : TestSourceSettings; - ExecutionStatusSettings: MinimumTestsExecutionStatusSettings; + ExecutionStatusSettings: ExecutionStatusSettings; } export interface TestSelectionSettings { @@ -114,7 +114,7 @@ export interface TestSourceSettings { PullRequestTargetBranchName : string; } -export interface MinimumTestsExecutionStatusSettings { +export interface ExecutionStatusSettings { IgnoreTestFailures : boolean; MinimumExecutedTestsExpected : number; ActionOnThresholdNotMet : string; From 449edbdae453f65090de2b82f9b26c52fc9a6088 Mon Sep 17 00:00:00 2001 From: Vagisha Nidhi Date: Wed, 13 Nov 2019 14:33:24 +0530 Subject: [PATCH 5/7] Fixed build failure --- Tasks/VsTestV2/inputparser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tasks/VsTestV2/inputparser.ts b/Tasks/VsTestV2/inputparser.ts index 61db1d078cbe..5f25f6f421e0 100644 --- a/Tasks/VsTestV2/inputparser.ts +++ b/Tasks/VsTestV2/inputparser.ts @@ -168,7 +168,7 @@ function getTestReportingSettings(inputDataContract : idc.InputDataContract) : i inputDataContract.TestReportingSettings.TestSourceSettings = {}; inputDataContract.TestReportingSettings.TestSourceSettings.PullRequestTargetBranchName = tl.getVariable('System.PullRequest.TargetBranch'); - inputDataContract.TestReportingSettings.ExecutionStatusSettings = {}; + inputDataContract.TestReportingSettings.ExecutionStatusSettings = {}; inputDataContract.TestReportingSettings.ExecutionStatusSettings.IgnoreTestFailures = utils.Helper.stringToBool(tl.getVariable('vstest.ignoretestfailures')); if (utils.Helper.isNullEmptyOrUndefined(inputDataContract.TestReportingSettings.TestRunTitle)) { From aa9d307cf48a1cf1c22eb08052df79ae7320e359 Mon Sep 17 00:00:00 2001 From: Vagisha Nidhi Date: Thu, 14 Nov 2019 13:43:19 +0530 Subject: [PATCH 6/7] Updating testagent version --- Tasks/VsTestV2/make.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tasks/VsTestV2/make.json b/Tasks/VsTestV2/make.json index 2dad8141950c..6d5d36cace7b 100644 --- a/Tasks/VsTestV2/make.json +++ b/Tasks/VsTestV2/make.json @@ -6,7 +6,7 @@ "dest": "./" }, { - "url": "https://testexecution.blob.core.windows.net/testexecution/10738269/TestAgent.zip", + "url": "https://testexecution.blob.core.windows.net/testexecution/10889838/TestAgent.zip", "dest": "./Modules" }, { From ba5dba87e4586cfc41ba66acf18cabd4370fad0b Mon Sep 17 00:00:00 2001 From: Vagisha Nidhi Date: Thu, 14 Nov 2019 14:23:10 +0530 Subject: [PATCH 7/7] Updated inputdatacontract.ts --- Tasks/VsTestV2/inputdatacontract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tasks/VsTestV2/inputdatacontract.ts b/Tasks/VsTestV2/inputdatacontract.ts index b080a031e53f..6f1af12fc0ea 100644 --- a/Tasks/VsTestV2/inputdatacontract.ts +++ b/Tasks/VsTestV2/inputdatacontract.ts @@ -25,7 +25,7 @@ export interface TestReportingSettings { TestResultsDirectory : string; TestRunSystem : string; TestSourceSettings : TestSourceSettings; - ExecutionStatusSettings: ExecutionStatusSettings; + ExecutionStatusSettings : ExecutionStatusSettings; } export interface TestSelectionSettings {