From 10fbb84f68a5bdf5d72e5d25275024223a61acdf Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 26 Apr 2023 11:34:41 +0530 Subject: [PATCH 01/37] Add parallel test support --- .../io/ballerina/cli/cmd/TestCommand.java | 12 +- .../cli/task/RunNativeImageTestTask.java | 11 +- .../io/ballerina/cli/task/RunTestsTask.java | 12 +- .../io/ballerina/cli/utils/NativeUtils.java | 1 + .../TesterinaCompilerPluginConstants.java | 2 + .../TesterinaCompilerPluginUtils.java | 7 +- .../main/ballerina/annotation_processor.bal | 6 +- .../src/main/ballerina/annotations.bal | 2 + .../src/main/ballerina/execute.bal | 287 ++++++++++++++---- .../src/main/ballerina/external.bal | 55 ++-- .../src/main/ballerina/filter.bal | 20 +- .../src/main/ballerina/register.bal | 71 +++-- .../testerina/natives/CommonUtils.java | 22 ++ .../ballerinalang/test/runtime/BTestMain.java | 2 +- .../test/runtime/entity/TestArguments.java | 6 +- .../test/runtime/util/TesterinaConstants.java | 2 + .../BasicCasesTest-testAnnotationAccess.txt | 4 +- .../unix/BasicCasesTest-testAnnotations.txt | 6 +- .../unix/BasicCasesTest-testDependsOn.txt | 8 +- .../GroupingTest-beforeGroupsAfterGroups2.txt | 21 +- ...ipTestsTestCase-testSkipWhenAfterFails.txt | 4 +- ...pTestsTestCase-testSkipWhenBeforeFails.txt | 4 +- 22 files changed, 428 insertions(+), 137 deletions(-) create mode 100644 misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java index 8dc20db5c7b6..8502c8ed42eb 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java @@ -129,6 +129,9 @@ public TestCommand() { @CommandLine.Option(names = "--debug", description = "start in remote debugging mode") private String debugPort; + @CommandLine.Option(names = "--workers", description = "maximum number of parallel test jobs") + private int workers; + @CommandLine.Option(names = "--list-groups", description = "list the groups available in the tests") private boolean listGroups; @@ -252,6 +255,11 @@ public void execute() { return; } + if (workers < 0) { + this.outStream.println("\nWarning: Workers can not be negative or zero. Test execution is proceeded " + + "with default worker count.\n"); + } + // Sets the debug port as a system property, which will be used when setting up debug args before running tests. if (this.debugPort != null) { System.setProperty(SYSTEM_PROP_BAL_DEBUG, this.debugPort); @@ -305,9 +313,9 @@ public void execute() { .addTask(new CompileTask(outStream, errStream, false, isPackageModified, buildOptions.enableCache())) // .addTask(new CopyResourcesTask(), listGroups) // merged with CreateJarTask .addTask(new RunTestsTask(outStream, errStream, rerunTests, groupList, disableGroupList, testList, - includes, coverageFormat, moduleMap, listGroups), project.buildOptions().nativeImage()) + includes, coverageFormat, moduleMap, listGroups, workers), project.buildOptions().nativeImage()) .addTask(new RunNativeImageTestTask(outStream, rerunTests, groupList, disableGroupList, - testList, includes, coverageFormat, moduleMap, listGroups), + testList, includes, coverageFormat, moduleMap, listGroups, workers), !project.buildOptions().nativeImage()) .addTask(new DumpBuildTimeTask(outStream), !project.buildOptions().dumpBuildTime()) .build(); diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java index 7f5a6d2a64c1..bcea55eaab12 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java @@ -86,6 +86,7 @@ import static java.util.Objects.requireNonNull; import static org.ballerinalang.test.runtime.util.TesterinaConstants.CACHE_DIR; import static org.ballerinalang.test.runtime.util.TesterinaConstants.CLASS_EXTENSION; +import static org.ballerinalang.test.runtime.util.TesterinaConstants.DEFAULT_TEST_WORKERS; import static org.ballerinalang.test.runtime.util.TesterinaConstants.DOT; import static org.ballerinalang.test.runtime.util.TesterinaConstants.DOT_REPLACER; import static org.ballerinalang.test.runtime.util.TesterinaConstants.HYPHEN; @@ -134,12 +135,13 @@ public void run() { private boolean isRerunTestExecution; private String singleExecTests; private boolean listGroups; + private int testWorkers; TestReport testReport; public RunNativeImageTestTask(PrintStream out, boolean rerunTests, String groupList, String disableGroupList, String testList, String includes, String coverageFormat, - Map modules, boolean listGroups) { + Map modules, boolean listGroups, int workers) { this.out = out; this.isRerunTestExecution = rerunTests; @@ -153,6 +155,12 @@ public RunNativeImageTestTask(PrintStream out, boolean rerunTests, String groupL singleExecTests = testList; } this.listGroups = listGroups; + + if (workers <= 0) { + testWorkers = DEFAULT_TEST_WORKERS; + } else { + testWorkers = workers; + } } @@ -539,6 +547,7 @@ private int runTestSuiteWithNativeImage(Package currentPackage, Target target, cmdArgs.add(this.singleExecTests != null ? this.singleExecTests : ""); cmdArgs.add(Boolean.toString(isRerunTestExecution)); cmdArgs.add(Boolean.toString(listGroups)); // 8 + cmdArgs.add(Integer.toString(testWorkers)); builder.command(cmdArgs.toArray(new String[0])); process = builder.start(); diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java index 4a334dad6257..18000ca58cc0 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java @@ -66,6 +66,7 @@ import static io.ballerina.cli.utils.TestUtils.generateTesterinaReports; import static io.ballerina.cli.utils.TestUtils.loadModuleStatusFromFile; import static io.ballerina.cli.utils.TestUtils.writeToTestSuiteJson; +import static org.ballerinalang.test.runtime.util.TesterinaConstants.DEFAULT_TEST_WORKERS; import static org.ballerinalang.test.runtime.util.TesterinaConstants.MOCK_FN_DELIMITER; import static org.ballerinalang.test.runtime.util.TesterinaConstants.MOCK_LEGACY_DELIMITER; import static org.wso2.ballerinalang.compiler.util.ProjectDirConstants.BALLERINA_HOME; @@ -92,15 +93,23 @@ public class RunTestsTask implements Task { private Map coverageModules; private boolean listGroups; + private int testWorkers; + TestReport testReport; public RunTestsTask(PrintStream out, PrintStream err, boolean rerunTests, String groupList, String disableGroupList, String testList, String includes, String coverageFormat, - Map modules, boolean listGroups) { + Map modules, boolean listGroups, int workers) { this.out = out; this.err = err; this.isRerunTestExecution = rerunTests; + if (workers <= 0) { + testWorkers = DEFAULT_TEST_WORKERS; + } else { + testWorkers = workers; + } + if (disableGroupList != null) { this.disableGroupList = disableGroupList; } else if (groupList != null) { @@ -305,6 +314,7 @@ private int runTestSuite(Target target, Package currentPackage, JBallerinaBacken cmdArgs.add(this.singleExecTests != null ? this.singleExecTests : ""); cmdArgs.add(Boolean.toString(isRerunTestExecution)); cmdArgs.add(Boolean.toString(listGroups)); + cmdArgs.add(Integer.toString(testWorkers)); ProcessBuilder processBuilder = new ProcessBuilder(cmdArgs).inheritIO(); Process proc = processBuilder.start(); diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java index cacd46f5aa18..d793007d95b1 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java @@ -131,6 +131,7 @@ public static void createReflectConfig(Path nativeConfigPath, Package currentPac "io.ballerina.runtime.api.values.BString", "io.ballerina.runtime.api.values.BString", "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", "io.ballerina.runtime.api.values.BString" } ) diff --git a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java index 27fc38846498..b81ed5f077f8 100644 --- a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java +++ b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java @@ -45,4 +45,6 @@ public class TesterinaCompilerPluginConstants { public static final String TESTS_PARAMETER = "tests"; public static final String RERUN_FAILED_PARAMETER = "rerunFailed"; public static final String LIST_GROUPS_PARAMETER = "listGroups"; + + public static final String TEST_PARALLEL_JOBS_PARAMETER = "testWorkers"; } diff --git a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java index 12a10122cab9..781af5d571d3 100644 --- a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java +++ b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java @@ -72,7 +72,8 @@ public static void addSetTestOptionsCall(List statements) { getPositionalArg(TesterinaCompilerPluginConstants.DISABLE_GROUPS_PARAMETER), getPositionalArg(TesterinaCompilerPluginConstants.TESTS_PARAMETER), getPositionalArg(TesterinaCompilerPluginConstants.RERUN_FAILED_PARAMETER), - getPositionalArg(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER))))); + getPositionalArg(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER), + getPositionalArg(TesterinaCompilerPluginConstants.TEST_PARALLEL_JOBS_PARAMETER))))); } public static void addStartSuiteCall(List statements) { @@ -241,7 +242,9 @@ public static FunctionSignatureNode getFunctionSignature() { NodeFactory.createToken(SyntaxKind.COMMA_TOKEN), getStringParameter(TesterinaCompilerPluginConstants.RERUN_FAILED_PARAMETER), NodeFactory.createToken(SyntaxKind.COMMA_TOKEN), - getStringParameter(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER)), + getStringParameter(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER), + NodeFactory.createToken(SyntaxKind.COMMA_TOKEN), + getStringParameter(TesterinaCompilerPluginConstants.TEST_PARALLEL_JOBS_PARAMETER)), NodeFactory.createToken(SyntaxKind.CLOSE_PAREN_TOKEN), returnTypeDescriptorNode); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 0f7392244583..a95f7c02267a 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -64,7 +64,7 @@ function processConfigAnnotation(string name, function f) returns boolean { testRegistry.addFunction(name = name, executableFunction = f, params = params, before = config.before, after = config.after, groups = config.groups, diagnostics = diagnostics, dependsOn = config.dependsOn, - enabled = enabled, dependsOnCount = config.dependsOn.length(), config = config); + enabled = enabled, dependsOnCount = config.dependsOn.length(), parallelizable = config.parallelizable, config = config); return true; } return false; @@ -144,8 +144,8 @@ function hasTest(string name) returns boolean { if (filter.includes(WILDCARD)) { boolean|error wildCardMatch = matchWildcard(testName, filter); if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { - return true; - } + return true; + } } } return false; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal index 5e25a168261a..c3f7d682398e 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal @@ -22,6 +22,7 @@ # + before - Function to be run before the test is run # + after - Function to be run after the test is run # + dependsOn - A list of functions the test function depends on and will be run before the test +# + parallelizable - Flag to enable/disable parallization for a test function public type TestConfig record { boolean enable = true; string[] groups = []; @@ -29,6 +30,7 @@ public type TestConfig record { function () returns (any|error) before?; function () returns (any|error) after?; function[] dependsOn = []; + boolean parallelizable = true; }; # Configuration of the function to be mocked. diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 0a49a3ce39c4..49ba25e21477 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -1,3 +1,5 @@ +import ballerina/lang.runtime; + // Copyright (c) 2022 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. // // WSO2 Inc. licenses this file to you under the Apache License, @@ -14,9 +16,12 @@ // specific language governing permissions and limitations // under the License. -boolean shouldSkip = false; +isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; -int exitCode = 0; +isolated int exitCode = 0; +TestFunction[] parallelTestExecutionList = []; +TestFunction[] serialTestExecutionList = []; +isolated int unAllocatedTestWorkers = 1; public function startSuite() { // exit if setTestOptions has failed @@ -37,11 +42,15 @@ public function startSuite() { error? err = orderTests(); if err is error { - exitCode = 1; + enableExit(); println(err.message()); } else { executeBeforeSuiteFunctions(); - executeTests(); + err = executeTests(); + if err is error { + enableExit(); + println(err.message()); + } executeAfterSuiteFunctions(); reportGenerators.forEach(reportGen => reportGen(reportData)); } @@ -50,29 +59,74 @@ public function startSuite() { } function exitOnError() { - if exitCode > 0 { - panic (error("")); + lock { + if exitCode > 0 { + panic (error("")); + } } } -function executeTests() { +function executeTests() returns error? { + decimal startTime = currentTimeInMillis(); foreach TestFunction testFunction in testRegistry.getFunctions() { - executeTest(testFunction); + if testFunction.parallelizable { + parallelTestExecutionList.push(testFunction); + } else { + serialTestExecutionList.push(testFunction); + } + + } + + while true { + if (parallelTestExecutionList.length() == 0 && getAvailableWorkerCount() == testWorkers + && serialTestExecutionList.length() == 0) { + break; + } + + if (getAvailableWorkerCount() != 0) { + + if parallelTestExecutionList.length() == 0 && serialTestExecutionList.length() == 0 { + runtime:sleep(0.0001); + continue; + + } + + if (serialTestExecutionList.length() != 0 && getAvailableWorkerCount() == testWorkers) { + + TestFunction testFunction = serialTestExecutionList.remove(0); + allocateWorker(); + future serialWaiter = start executeTest(testFunction); + any _ = check wait serialWaiter; + + } else if parallelTestExecutionList.length() != 0 && serialTestExecutionList.length() == 0 { + TestFunction testFunction = parallelTestExecutionList.remove(0); + allocateWorker(); + future<(error?)> parallelWaiter = start executeTest(testFunction); + runtime:sleep(0.0001); + if isDataDrivenTest(testFunction) { + any _ = check wait parallelWaiter; + } + + } + } else { + runtime:sleep(0.0001); + } } + // println("Test execution time :" + (currentTimeInMillis() - startTime).toString() + "ms"); } -function executeTest(TestFunction testFunction) { +function executeTest(TestFunction testFunction) returns error? { if !testFunction.enabled { + releaseWorker(); return; } error? diagnoseError = testFunction.diagnostics; if diagnoseError is error { - reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); - exitCode = 1; - return; - } - if testFunction.dependsOnCount > 1 { - testFunction.dependsOnCount -= 1; + lock { + reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); + } + enableExit(); + releaseWorker(); return; } @@ -80,14 +134,16 @@ function executeTest(TestFunction testFunction) { executeBeforeEachFunctions(); boolean shouldSkipDependents = false; - if !testFunction.skip && !shouldSkip { + if !testFunction.skip && !getShouldSkip() { if (isDataDrivenTest(testFunction)) { - executeDataDrivenTestSet(testFunction); + check executeDataDrivenTestSet(testFunction); } else { shouldSkipDependents = executeNonDataDrivenTest(testFunction); } } else { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + lock { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + } shouldSkipDependents = true; } @@ -97,37 +153,94 @@ function executeTest(TestFunction testFunction) { if shouldSkipDependents { testFunction.dependents.forEach(function(TestFunction dependent) { - dependent.skip = true; + lock { + dependent.skip = true; + } }); } - testFunction.dependents.forEach(dependent => executeTest(dependent)); + testFunction.dependents.forEach(dependent => checkExecutionReadiness(dependent)); + releaseWorker(); + // isSerialTestExecution = !isSerialTestExecution && !testFunction.parallelizable; } -function executeDataDrivenTestSet(TestFunction testFunction) { +function checkExecutionReadiness(TestFunction testFunction) { + lock { + testFunction.dependsOnCount -= 1; + if testFunction.dependsOnCount == 0 && testFunction.isInExecutionQueue != true { + testFunction.isInExecutionQueue = true; + if testFunction.parallelizable { + parallelTestExecutionList.push(testFunction); + } else { + serialTestExecutionList.push(testFunction); + } + } + } +} + +function executeDataDrivenTestSet(TestFunction testFunction) returns error? { DataProviderReturnType? params = testFunction.params; + string[] keys = []; + AnyOrError[][] values = []; + TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; if params is map { foreach [string, AnyOrError[]] entry in params.entries() { - boolean beforeFailed = executeBeforeFunction(testFunction); - if (beforeFailed) { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); - } else { - executeDataDrivenTest(testFunction, entry[0], DATA_DRIVEN_MAP_OF_TUPLE, entry[1]); - var _ = executeAfterFunction(testFunction); - } + keys.push(entry[0]); + values.push(entry[1]); } } else if params is AnyOrError[][] { + testType = DATA_DRIVEN_TUPLE_OF_TUPLE; int i = 0; foreach AnyOrError[] entry in params { - boolean beforeFailed = executeBeforeFunction(testFunction); - if (beforeFailed) { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); - } else { - executeDataDrivenTest(testFunction, i.toString(), DATA_DRIVEN_TUPLE_OF_TUPLE, entry); - var _ = executeAfterFunction(testFunction); - } + keys.push(i.toString()); + values.push(entry); i += 1; } } + + boolean isIntialJob = true; + + while true { + if keys.length() == 0 { + break; + } + + if isIntialJob || getAvailableWorkerCount() > 0 { + string key = keys.remove(0); + AnyOrError[] value = values.remove(0); + + if !isIntialJob { + allocateWorker(); + } + + future<()> serialWaiter = start prepareDataDrivenTest(testFunction, key, value, testType); + + if !testFunction.parallelizable { + any _ = check wait serialWaiter; + } + + } + + isIntialJob = false; + + if getAvailableWorkerCount() == 0 { + runtime:sleep(0.0001); + continue; + } + } + allocateWorker(); +} + +function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { + boolean beforeFailed = executeBeforeFunction(testFunction); + if (beforeFailed) { + lock { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + } + } else { + executeDataDrivenTest(testFunction, key, testType, value); + var _ = executeAfterFunction(testFunction); + } + releaseWorker(); } function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { @@ -137,9 +250,11 @@ function executeDataDrivenTest(TestFunction testFunction, string suffix, TestTyp ExecutionError|boolean err = executeTestFunction(testFunction, suffix, testType, params); if err is ExecutionError { - reportData.onFailed(name = testFunction.name, message = "[fail data provider for the function " + testFunction.name + lock { + reportData.onFailed(name = testFunction.name, message = "[fail data provider for the function " + testFunction.name + "]\n" + getErrorMessage(err), testType = testType); - exitCode = 1; + enableExit(); + } } } @@ -148,14 +263,20 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { boolean beforeFailed = executeBeforeFunction(testFunction); if (beforeFailed) { testFunction.skip = true; - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + lock { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + } return true; } ExecutionError|boolean output = executeTestFunction(testFunction, "", GENERAL_TEST); if output is ExecutionError { failed = true; - reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); - } else if output { + lock { + reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); + } + } + + else if output { failed = true; } boolean afterFailed = executeAfterFunction(testFunction); @@ -163,14 +284,15 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { return true; } return failed; + } function executeBeforeSuiteFunctions() { ExecutionError? err = executeFunctions(beforeSuiteRegistry.getFunctions()); if err is ExecutionError { - shouldSkip = true; + enableShouldSkip(); shouldAfterSuiteSkip = true; - exitCode = 1; + enableExit(); printExecutionError(err, "before test suite function"); } } @@ -178,35 +300,35 @@ function executeBeforeSuiteFunctions() { function executeAfterSuiteFunctions() { ExecutionError? err = executeFunctions(afterSuiteRegistry.getFunctions(), shouldAfterSuiteSkip); if err is ExecutionError { - exitCode = 1; + enableExit(); printExecutionError(err, "after test suite function"); } } function executeBeforeEachFunctions() { - ExecutionError? err = executeFunctions(beforeEachRegistry.getFunctions(), shouldSkip); + ExecutionError? err = executeFunctions(beforeEachRegistry.getFunctions(), getShouldSkip()); if err is ExecutionError { - shouldSkip = true; - exitCode = 1; + enableShouldSkip(); + enableExit(); printExecutionError(err, "before each test function for the test"); } } function executeAfterEachFunctions() { - ExecutionError? err = executeFunctions(afterEachRegistry.getFunctions(), shouldSkip); + ExecutionError? err = executeFunctions(afterEachRegistry.getFunctions(), getShouldSkip()); if err is ExecutionError { - shouldSkip = true; - exitCode = 1; + enableShouldSkip(); + enableExit(); printExecutionError(err, "after each test function for the test"); } } function executeBeforeFunction(TestFunction testFunction) returns boolean { boolean failed = false; - if testFunction.before is function && !shouldSkip && !testFunction.skip { + if testFunction.before is function && !getShouldSkip() && !testFunction.skip { ExecutionError? err = executeFunction(testFunction.before); if err is ExecutionError { - exitCode = 1; + enableExit(); printExecutionError(err, "before test function for the test"); failed = true; } @@ -216,10 +338,10 @@ function executeBeforeFunction(TestFunction testFunction) returns boolean { function executeAfterFunction(TestFunction testFunction) returns boolean { boolean failed = false; - if testFunction.after is function && !shouldSkip && !testFunction.skip { + if testFunction.after is function && !getShouldSkip() && !testFunction.skip { ExecutionError? err = executeFunction(testFunction.after); if err is ExecutionError { - exitCode = 1; + enableExit(); printExecutionError(err, "after test function for the test"); failed = true; } @@ -231,11 +353,11 @@ function executeBeforeGroupFunctions(TestFunction testFunction) { foreach string group in testFunction.groups { TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions(group); if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted(group) { - ExecutionError? err = executeFunctions(beforeGroupFunctions, shouldSkip); + ExecutionError? err = executeFunctions(beforeGroupFunctions, getShouldSkip()); if err is ExecutionError { testFunction.skip = true; groupStatusRegistry.setSkipAfterGroup(group); - exitCode = 1; + enableExit(); printExecutionError(err, "before test group function for the test"); } } @@ -247,9 +369,9 @@ function executeAfterGroupFunctions(TestFunction testFunction) { TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions(group); if afterGroupFunctions != () && groupStatusRegistry.lastExecuted(group) { ExecutionError? err = executeFunctions(afterGroupFunctions, - shouldSkip || groupStatusRegistry.getSkipAfterGroup(group)); + getShouldSkip() || groupStatusRegistry.getSkipAfterGroup(group)); if err is ExecutionError { - exitCode = 1; + enableExit(); printExecutionError(err, "after test group function for the test"); } } @@ -340,14 +462,18 @@ function executeTestFunction(TestFunction testFunction, string suffix, TestType any|error output = params == () ? trap function:call(testFunction.executableFunction) : trap function:call(testFunction.executableFunction, ...params); if output is TestError { - exitCode = 1; - reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); + enableExit(); + lock { + reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); + } return true; } else if output is any { - reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); + lock { + reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); + } return false; } else { - exitCode = 1; + enableExit(); return error(getErrorMessage(output), functionName = testFunction.name); } } @@ -355,7 +481,7 @@ function executeTestFunction(TestFunction testFunction, string suffix, TestType function executeFunction(TestFunction|function testFunction) returns ExecutionError? { any|error output = trap function:call(testFunction is function ? testFunction : testFunction.executableFunction); if output is error { - exitCode = 1; + enableExit(); return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); } } @@ -432,3 +558,40 @@ function nestedEnabledDependentsAvailable(TestFunction[] dependents) returns boo } return nestedEnabledDependentsAvailable(queue); } + +isolated function allocateWorker() { + lock { + unAllocatedTestWorkers -= 1; + } +} + +isolated function releaseWorker() { + lock { + unAllocatedTestWorkers += 1; + } +} + +isolated function getAvailableWorkerCount() returns int { + lock { + return unAllocatedTestWorkers; + } +} + +isolated function enableShouldSkip() { + lock { + shouldSkip = true; + } +} + +isolated function getShouldSkip() returns boolean { + lock { + return shouldSkip; + } +} + +isolated function enableExit() { + lock { + exitCode = 1; + } +} + diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal index e0ee18282549..53efdb6b6f58 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal @@ -48,56 +48,60 @@ isolated function splitExternal(handle receiver, handle delimiter) returns handl } external; isolated function getBallerinaStringArray(handle h) returns string[] = @java:Method { - 'class:"io.ballerina.runtime.api.utils.StringUtils", - name:"fromStringArray", - paramTypes:["[Ljava.lang.String;"] + 'class: "io.ballerina.runtime.api.utils.StringUtils", + name: "fromStringArray", + paramTypes: ["[Ljava.lang.String;"] } external; isolated function writeContent(string filePath, string content) returns error? = @java:Method { - 'class:"org.ballerinalang.testerina.natives.io.FileUtils", - name:"writeContent" + 'class: "org.ballerinalang.testerina.natives.io.FileUtils", + name: "writeContent" } external; isolated function readContent(string filePath) returns string = @java:Method { - 'class:"org.ballerinalang.testerina.natives.io.FileUtils", - name:"readContent" + 'class: "org.ballerinalang.testerina.natives.io.FileUtils", + name: "readContent" } external; isolated function fileExists(string filePath) returns boolean = @java:Method { - 'class:"org.ballerinalang.testerina.natives.io.FileUtils", - name:"fileExists" + 'class: "org.ballerinalang.testerina.natives.io.FileUtils", + name: "fileExists" } external; isolated function isSystemConsole() returns boolean = @java:Method { - 'class:"org.ballerinalang.testerina.natives.io.StringUtils", - name:"isSystemConsole" + 'class: "org.ballerinalang.testerina.natives.io.StringUtils", + name: "isSystemConsole" } external; isolated function sprintf(string format, (any|error)... args) returns string = @java:Method { - name : "sprintf", - 'class : "org.ballerinalang.testerina.natives.io.StringUtils" + name: "sprintf", + 'class: "org.ballerinalang.testerina.natives.io.StringUtils" } external; isolated function matchWildcard(string functionName, string functionPattern) returns boolean|error = @java:Method { - name : "matchWildcard", - 'class : "org.ballerinalang.testerina.natives.io.StringUtils" + name: "matchWildcard", + 'class: "org.ballerinalang.testerina.natives.io.StringUtils" } external; isolated function decode(string str, string charset) returns string|error = @java:Method { - name : "decode", - 'class : "org.ballerinalang.testerina.natives.io.StringUtils" + name: "decode", + 'class: "org.ballerinalang.testerina.natives.io.StringUtils" +} external; + +isolated function sleep(decimal seconds) returns error? = @java:Method { + name: "sleep", + 'class: "org.ballerinalang.testerina.natives.CommonUtils" } external; isolated function getBallerinaType((any|error) value) returns string = @java:Method { - name : "getBallerinaType", - 'class : "org.ballerinalang.testerina.core.BallerinaTypeCheck" + name: "getBallerinaType", + 'class: "org.ballerinalang.testerina.core.BallerinaTypeCheck" } external; isolated function getStringDiff(string actual, string expected) returns string = @java:Method { - name : "getStringDiff", - 'class : "org.ballerinalang.testerina.core.AssertionDiffEvaluator" - } external; - + name: "getStringDiff", + 'class: "org.ballerinalang.testerina.core.AssertionDiffEvaluator" +} external; isolated function getKeysDiff(string[] actualKeys, string[] expectedKeys) returns string = @java:Method { name: "getKeysDiff", @@ -108,3 +112,8 @@ isolated function escapeSpecialCharacters(string key) returns string|error = @ja name: "escapeSpecialCharacters", 'class: "org.ballerinalang.testerina.natives.io.StringUtils" } external; + +isolated function currentTimeInMillis() returns decimal = @java:Method { + name: "currentTimeInMillis", + 'class: "org.ballerinalang.testerina.natives.CommonUtils" +} external; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index 7bdaebafb85c..a0a7c3734fe7 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -25,10 +25,12 @@ boolean hasFilteredTests = false; string targetPath = ""; boolean terminate = false; boolean listGroups = false; +int testWorkers = 1; public function setTestOptions(string inTargetPath, string inPackageName, string inModuleName, string inReport, string inCoverage, string inGroups, string inDisableGroups, string inTests, string inRerunFailed, - string inListGroups) { + string inListGroups, string inTestWorkers) { + targetPath = inTargetPath; packageName = inPackageName; moduleName = inModuleName; @@ -38,12 +40,16 @@ public function setTestOptions(string inTargetPath, string inPackageName, string boolean testReport = parseBooleanInput(inReport, "test-report"); boolean codeCoverage = parseBooleanInput(inCoverage, "code-coverage"); listGroups = parseBooleanInput(inListGroups, "list-groups"); + testWorkers = parseIntegerInput(inTestWorkers, "testWorkers"); + lock { + unAllocatedTestWorkers = testWorkers; + } if rerunFailed { error? err = parseRerunJson(); if err is error { println("error: " + err.message()); - exitCode = 1; + enableExit(); return; } hasFilteredTests = true; @@ -96,6 +102,16 @@ function parseBooleanInput(string input, string variableName) returns boolean { return booleanVariable; } +function parseIntegerInput(string input, string variableName) returns int { + int|error intVariable = int:fromString(input); + if intVariable is error { + println(string `Invalid '${variableName}' parameter: ${intVariable.message()}`); + terminate = true; + return 0; + } + return intVariable; +} + function parseRerunJson() returns error? { string rerunJsonFilePath = targetPath + "/" + RERUN_JSON_FILE; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 0b959d5e2995..769ead307b8b 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -39,6 +39,8 @@ type TestFunction record {| int dependsOnCount = 0; TestFunction[] dependents = []; boolean visited = false; + boolean isInExecutionQueue = false; + boolean parallelizable = true; TestConfig? config = (); |}; @@ -92,48 +94,71 @@ class GroupRegistry { } } -class GroupStatusRegistry { +isolated class GroupStatusRegistry { private final map enabledTests = {}; private final map totalTests = {}; private final map executedTests = {}; private final map skip = {}; - function firstExecuted(string group) returns boolean => self.executedTests.get(group) > 0; + isolated function firstExecuted(string group) returns boolean { + lock { + return self.executedTests.get(group) > 0; + } + } + isolated function lastExecuted(string group) returns boolean { + lock { - function lastExecuted(string group) returns boolean => self.executedTests.get(group) == self.enabledTests.get(group); + return self.executedTests.get(group) == self.enabledTests.get(group); + } + } - function incrementTotalTest(string group, boolean enabled) { - if self.totalTests.hasKey(group) { - self.totalTests[group] = self.totalTests.get(group) + 1; - } else { - self.totalTests[group] = 1; + isolated function incrementTotalTest(string group, boolean enabled) { + lock { + if self.totalTests.hasKey(group) { + self.totalTests[group] = self.totalTests.get(group) + 1; + } else { + self.totalTests[group] = 1; + } + if enabled { + self.skip[group] = false; + if self.enabledTests.hasKey(group) { + self.enabledTests[group] = self.enabledTests.get(group) + 1; + } else { + self.enabledTests[group] = 1; + self.executedTests[group] = 0; + } + } } - if enabled { - self.skip[group] = false; - if self.enabledTests.hasKey(group) { - self.enabledTests[group] = self.enabledTests.get(group) + 1; + } + + isolated function incrementExecutedTest(string group) { + lock { + if self.executedTests.hasKey(group) { + self.executedTests[group] = self.executedTests.get(group) + 1; } else { - self.enabledTests[group] = 1; - self.executedTests[group] = 0; + self.executedTests[group] = 1; } } } - function incrementExecutedTest(string group) { - if self.executedTests.hasKey(group) { - self.executedTests[group] = self.executedTests.get(group) + 1; - } else { - self.executedTests[group] = 1; + isolated function setSkipAfterGroup(string group) { + lock { + self.skip[group] = true; } } - function setSkipAfterGroup(string group) { - self.skip[group] = true; + isolated function getSkipAfterGroup(string group) returns boolean { + lock { + return self.skip.get(group); + } } - function getSkipAfterGroup(string group) returns boolean => self.skip.get(group); + isolated function getGroupsList() returns string[] { + lock { + return self.totalTests.keys().clone(); + } + } - function getGroupsList() returns string[] => self.totalTests.keys(); } isolated function testFunctionsSort(TestFunction testFunction) returns string => testFunction.name; diff --git a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java new file mode 100644 index 000000000000..042a989c06da --- /dev/null +++ b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java @@ -0,0 +1,22 @@ +package org.ballerinalang.testerina.natives; + +import io.ballerina.runtime.api.values.BDecimal; +import io.ballerina.runtime.internal.util.exceptions.BLangExceptionHelper; +import io.ballerina.runtime.internal.util.exceptions.RuntimeErrors; + +public class CommonUtils { + public static Object sleep(BDecimal seconds) { + try { + Thread.sleep(seconds.intValue() * 1000); + } catch (InterruptedException e) { + return BLangExceptionHelper.getRuntimeException( + RuntimeErrors.OPERATION_NOT_SUPPORTED_ERROR, "Invalid duration: " + e.getMessage()); + } + return null; + } + + public static BDecimal currentTimeInMillis() { + long currentTime = System.currentTimeMillis(); + return BDecimal.valueOf(currentTime); + } +} diff --git a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java index 892aa358abb6..5606cba68d57 100644 --- a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java +++ b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java @@ -126,7 +126,7 @@ public static void main(String[] args) throws IOException { result = startTestSuit(Paths.get(testSuite.getSourceRootPath()), testSuite, classLoader, new TestArguments(args[0], packageName, moduleName, - args[2], args[3], args[4], args[5], args[6], args[7], args[8])); + args[2], args[3], args[4], args[5], args[6], args[7], args[8], args[9])); exitStatus = (result == 1) ? result : exitStatus; } } else { diff --git a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/entity/TestArguments.java b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/entity/TestArguments.java index b92dfd7d5f84..0cd0a6b5f497 100644 --- a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/entity/TestArguments.java +++ b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/entity/TestArguments.java @@ -27,13 +27,14 @@ */ public class TestArguments { - private static final int ARGUMENTS_NUMBER = 10; + private static final int ARGUMENTS_NUMBER = 11; private final Class[] argTypes; private final Object[] argValues; public TestArguments(String targetPath, String packageName, String moduleName, String report, String coverage, - String groups, String disableGroups, String tests, String rerunFailed, String listGroups) { + String groups, String disableGroups, String tests, String rerunFailed, String listGroups, + String testWorkers) { argTypes = new Class[ARGUMENTS_NUMBER + 1]; argValues = new Object[ARGUMENTS_NUMBER + 1]; @@ -51,6 +52,7 @@ public TestArguments(String targetPath, String packageName, String moduleName, S assignValues(7, tests); assignValues(8, rerunFailed); assignValues(9, listGroups); + assignValues(10, testWorkers); } public Class[] getArgTypes() { diff --git a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/util/TesterinaConstants.java b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/util/TesterinaConstants.java index 412c78715d00..d433ceaee1cc 100644 --- a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/util/TesterinaConstants.java +++ b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/util/TesterinaConstants.java @@ -86,4 +86,6 @@ public class TesterinaConstants { public static final String MOCK_FN_DELIMITER = "#"; public static final String MOCK_LEGACY_DELIMITER = "~"; + + public static final int DEFAULT_TEST_WORKERS = 1; } diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotationAccess.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotationAccess.txt index 689eb62f65af..3ce47870c013 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotationAccess.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotationAccess.txt @@ -20,9 +20,9 @@ java.lang.ClassCastException: class org.wso2.ballerinalang.compiler.semantics.mo at org.ballerinalang.testerina.core.TestProcessor.addUtilityFunctions(TestProcessor.java:315) at org.ballerinalang.testerina.core.TestProcessor.generateTestSuite(TestProcessor.java:169) at org.ballerinalang.testerina.core.TestProcessor.testSuite(TestProcessor.java:118) - at io.ballerina.cli.task.RunTestsTask.execute(RunTestsTask.java:168) + at io.ballerina.cli.task.RunTestsTask.execute(RunTestsTask.java:177) at io.ballerina.cli.TaskExecutor.executeTasks(TaskExecutor.java:40) - at io.ballerina.cli.cmd.TestCommand.execute(TestCommand.java:315) + at io.ballerina.cli.cmd.TestCommand.execute(TestCommand.java:323) at java.base/java.util.Optional.ifPresent(Optional.java:183) at io.ballerina.cli.launcher.Main.main(Main.java:51) \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt index 84c0b28e1627..8ad5a1616210 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt @@ -10,14 +10,14 @@ Running Tests with Coverage [pass] stringDataProviderTest#2 [pass] stringDataProviderTest2#0 [pass] test1 - [pass] testAfter [pass] test2 - [pass] testAfterAlone [pass] test3 [pass] test4 [pass] test5 - [pass] testDependsOn1 [pass] testBefore + [pass] testAfter + [pass] testAfterAlone + [pass] testDependsOn1 14 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt index ba3d2354ba79..6c1ff9fa82d7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt @@ -5,17 +5,17 @@ Running Tests with Coverage depends_on [pass] test1 + [pass] testAddingValues#0 + [pass] testAddingValues#1 + [pass] testAddingValues#2 [pass] test2 + [pass] testAddingValuesDependant [pass] test3 [pass] test4 [pass] testWithBefore1 [pass] testWithBefore2 [pass] testWithBefore3 [pass] testWithBefore4 - [pass] testAddingValues#0 - [pass] testAddingValues#1 - [pass] testAddingValues#2 - [pass] testAddingValuesDependant 12 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt index 595bf7fa68d1..78140dc4129a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt @@ -6,13 +6,30 @@ Compiling source Running Tests before-groups-after-groups-test2.bal + [fail] afterSuiteFunc[after test suite function]: + Assertion Failed! + +expected: '123456787' +actual : '123467587' + +Diff : + +--- actual ++++ expected + + @@ -1,1 +1,1 @@ + + -123467587 ++123456787 + [pass] testFunction [pass] testFunction2 - [pass] testFunction3 [pass] testFunction4 + [pass] testFunction3 [pass] testFunction5 5 passing 0 failing - 0 skipped \ No newline at end of file + 0 skipped +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt index 4300ed7e8f6e..8390ce3d9fca 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -13,7 +13,7 @@ Running Tests Assertion Failed! expected: 'beforetestafterEachtestafterEach' -actual : 'beforetestafterEachafterEachtestafterEach' +actual : 'beforetestafterEachtestafterEachafterEach' Diff : @@ -22,7 +22,7 @@ Diff : @@ -1,1 +1,1 @@ - -beforetestafterEachafterEachtestafterEach + -beforetestafterEachtestafterEachafterEach +beforetestafterEachtestafterEach [pass] test1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt index 3d1f93a2fa36..e50badf2a339 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -13,7 +13,7 @@ Running Tests Assertion Failed! expected: 'beforetest3afterEach' -actual : 'beforeafterEachafterEachtest3afterEach' +actual : 'beforeafterEachtest3afterEachafterEach' Diff : @@ -22,7 +22,7 @@ Diff : @@ -1,1 +1,1 @@ - -beforeafterEachafterEachtest3afterEach + -beforeafterEachtest3afterEachafterEach +beforetest3afterEach [pass] test3 From a751e26ed54439898da87e9c35a468fea2995037 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 26 Apr 2023 13:51:33 +0530 Subject: [PATCH 02/37] Fix build failure --- .../src/main/ballerina/execute.bal | 5 +-- .../ballerinalang/test/runtime/BTestMain.java | 2 +- ...iderTest-testDataProviderSingleFailure.txt | 10 ++---- .../GroupingTest-beforeGroupsAfterGroups2.txt | 35 +++++++++++-------- ...oviderTestCase-testInvalidDataProvider.txt | 20 ++++------- ...viderTestCase-testInvalidDataProvider2.txt | 20 ++++------- ...rTestCase-testInvalidTupleDataProvider.txt | 20 ++++------- ...ipTestsTestCase-testSkipWhenAfterFails.txt | 18 +++++----- ...pTestsTestCase-testSkipWhenBeforeFails.txt | 18 +++++----- 9 files changed, 63 insertions(+), 85 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 60fdecaea8a3..504046a8a9bc 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -15,7 +15,8 @@ // under the License. import ballerina/lang.'error as langError; -boolean shouldSkip = false; +import ballerina/lang.runtime; + isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; isolated int exitCode = 0; @@ -253,7 +254,7 @@ function executeDataDrivenTest(TestFunction testFunction, string suffix, TestTyp lock { reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + "]\n" + getErrorMessage(err), testType = testType); - enableExit(); + enableExit(); } } } diff --git a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java index 6814214ae40d..425db00b1e26 100644 --- a/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java +++ b/misc/testerina/modules/testerina-runtime/src/main/java/org/ballerinalang/test/runtime/BTestMain.java @@ -128,7 +128,7 @@ public static void main(String[] args) throws IOException { result = startTestSuit(Paths.get(testSuite.getSourceRootPath()), testSuite, classLoader, new TestArguments(args[0], packageName, moduleName, args[2], args[3], args[4], args[5], args[6], args[7], - args[8], args[9]), Arrays.copyOfRange(args, 9, args.length)); + args[8], args[9]), Arrays.copyOfRange(args, 10, args.length)); exitStatus = (result == 1) ? result : exitStatus; } } else { diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt index 0b93506ece30..6ab81b27d362 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt @@ -19,13 +19,9 @@ Running Tests with Coverage callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 17 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 26 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt index 78140dc4129a..cb5079d3b3cb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt @@ -7,21 +7,26 @@ Running Tests before-groups-after-groups-test2.bal [fail] afterSuiteFunc[after test suite function]: - Assertion Failed! - -expected: '123456787' -actual : '123467587' - -Diff : - ---- actual -+++ expected - - @@ -1,1 +1,1 @@ - - -123467587 -+123456787 - + error {ballerina/test:0}TestError ("Assertion Failed! + + expected: '123456787' + actual : '123467587' + + Diff : + + --- actual + +++ expected + + @@ -1,1 +1,1 @@ + + -123467587 + +123456787 + ") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 + callableName: afterSuiteFunc fileName: before-groups-after-groups-test2.bal lineNumber: 80 + callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 91 + [pass] testFunction [pass] testFunction2 [pass] testFunction4 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt index feb43e735988..00cd9ce11e26 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -13,21 +13,13 @@ Running Tests [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 343 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 463 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index 534be747dcd0..d1d413b206e6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -15,21 +15,13 @@ Running Tests [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 343 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 38 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 463 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 38 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 3435ef273bc0..94e214f74b65 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -12,21 +12,13 @@ Running Tests [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 343 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 35 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 463 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 35 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt index 694c9ea2fafc..fe7abe96c29f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -11,20 +11,20 @@ Running Tests error("{ballerina}DivisionByZero",message=" / by zero") callableName: afterFunc fileName: skip-when-after-fails.bal lineNumber: 30 callableName: afterFunc$lambda6$ fileName: skip-when-after-fails.bal lineNumber: 35 - + [fail] afterSuite[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! - + expected: 'beforetestafterEachtestafterEach' actual : 'beforetestafterEachtestafterEachafterEach' - + Diff : - + --- actual - +++ expected - - @@ -1,1 +1,1 @@ - + +++ expected + + @@ -1,1 +1,1 @@ + -beforetestafterEachtestafterEachafterEach +beforetestafterEachtestafterEach ") @@ -32,7 +32,7 @@ Running Tests callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuite fileName: skip-when-after-fails.bal lineNumber: 57 callableName: afterSuite$lambda5$ fileName: skip-when-after-fails.bal lineNumber: 64 - + [pass] test1 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt index b0dd228b5854..ac702e2a7322 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -11,20 +11,20 @@ Running Tests error("{ballerina}DivisionByZero",message=" / by zero") callableName: before fileName: skip-when-before-fails.bal lineNumber: 28 callableName: before$lambda6$ fileName: skip-when-before-fails.bal lineNumber: 32 - + [fail] afterSuite[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! - + expected: 'beforetest3afterEach' actual : 'beforeafterEachtest3afterEachafterEach' - + Diff : - + --- actual - +++ expected - - @@ -1,1 +1,1 @@ - + +++ expected + + @@ -1,1 +1,1 @@ + -beforeafterEachtest3afterEachafterEach +beforetest3afterEach ") @@ -32,7 +32,7 @@ Running Tests callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuite fileName: skip-when-before-fails.bal lineNumber: 54 callableName: afterSuite$lambda5$ fileName: skip-when-before-fails.bal lineNumber: 61 - + [pass] test3 From 3089586cf469a88f83230f78aa7eec353db4076d Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 11 May 2023 10:20:42 +0530 Subject: [PATCH 03/37] Add tests for test parallelization --- .../src/main/ballerina/execute.bal | 66 ++- .../test/TestparallelizationTest.java | 156 +++++++ .../testerina/test/utils/CommonUtils.java | 4 + .../Ballerina.toml | 8 + .../main.bal | 55 +++ .../tests/data_driven_test.bal | 60 +++ .../Ballerina.toml | 8 + .../main.bal | 55 +++ .../tests/data_driven_test.bal | 51 +++ .../Ballerina.toml | 8 + .../main.bal | 55 +++ .../tests/data_driven_test.bal | 59 +++ .../Ballerina.toml | 8 + .../parallelisation-parallelizable/main.bal | 55 +++ .../tests/normalTest.bal | 383 ++++++++++++++++++ .../Ballerina.toml | 7 + .../parallelisation-simple-test/main.bal | 55 +++ .../tests/normalTest.bal | 383 ++++++++++++++++++ .../Ballerina.toml | 8 + .../main.bal | 55 +++ .../tests/data_driven_test.bal | 51 +++ .../src/test/resources/testng.xml | 1 + 22 files changed, 1571 insertions(+), 20 deletions(-) create mode 100644 tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/tests/normalTest.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 504046a8a9bc..4b0d914e31af 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -78,42 +78,58 @@ function executeTests() returns error? { } + int paralalTestExecutionListLength = parallelTestExecutionList.length(); + int serialTestExecutionListLength = serialTestExecutionList.length(); + while true { - if (parallelTestExecutionList.length() == 0 && getAvailableWorkerCount() == testWorkers - && serialTestExecutionList.length() == 0) { + + TestFunction? testFunction = (); + lock { + paralalTestExecutionListLength = parallelTestExecutionList.length(); + serialTestExecutionListLength = serialTestExecutionList.length(); + } + //Exit from the loop if there are no tests to execute and all jobs are released + if (paralalTestExecutionListLength == 0 && getAvailableWorkerCount() == testWorkers + && serialTestExecutionListLength == 0) { break; } if (getAvailableWorkerCount() != 0) { - if parallelTestExecutionList.length() == 0 && serialTestExecutionList.length() == 0 { + if paralalTestExecutionListLength == 0 && serialTestExecutionListLength == 0 { runtime:sleep(0.0001); continue; } - if (serialTestExecutionList.length() != 0 && getAvailableWorkerCount() == testWorkers) { + if (serialTestExecutionListLength != 0 && getAvailableWorkerCount() == testWorkers) { + lock { + testFunction = serialTestExecutionList.remove(0); + } - TestFunction testFunction = serialTestExecutionList.remove(0); - allocateWorker(); - future serialWaiter = start executeTest(testFunction); - any _ = check wait serialWaiter; + if testFunction is TestFunction { + allocateWorker(); + future serialWaiter = start executeTest(testFunction); + any _ = check wait serialWaiter; + } - } else if parallelTestExecutionList.length() != 0 && serialTestExecutionList.length() == 0 { - TestFunction testFunction = parallelTestExecutionList.remove(0); - allocateWorker(); - future<(error?)> parallelWaiter = start executeTest(testFunction); - runtime:sleep(0.0001); - if isDataDrivenTest(testFunction) { - any _ = check wait parallelWaiter; + } else if paralalTestExecutionListLength != 0 && serialTestExecutionListLength == 0 { + lock { + testFunction = parallelTestExecutionList.remove(0); + } + if testFunction is TestFunction { + allocateWorker(); + future<(error?)> parallelWaiter = start executeTest(testFunction); + if isDataDrivenTest(testFunction) { + any _ = check wait parallelWaiter; + } } } - } else { - runtime:sleep(0.0001); } + runtime:sleep(0.0001); } - // println("Test execution time :" + (currentTimeInMillis() - startTime).toString() + "ms"); + println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n"); } function executeTest(TestFunction testFunction) returns error? { @@ -161,7 +177,6 @@ function executeTest(TestFunction testFunction) returns error? { } testFunction.dependents.forEach(dependent => checkExecutionReadiness(dependent)); releaseWorker(); - // isSerialTestExecution = !isSerialTestExecution && !testFunction.parallelizable; } function checkExecutionReadiness(TestFunction testFunction) { @@ -227,8 +242,19 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { runtime:sleep(0.0001); continue; } + runtime:sleep(0.0001); + } + + while true { + if getAvailableWorkerCount() > 0 { + break; + } + runtime:sleep(0.0001); + } + + if !isIntialJob { + allocateWorker(); } - allocateWorker(); } function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java new file mode 100644 index 000000000000..6f69ed8c48ee --- /dev/null +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -0,0 +1,156 @@ +package org.ballerinalang.testerina.test; + +import org.ballerinalang.test.context.BMainInstance; +import org.ballerinalang.test.context.BallerinaTestException; +import org.ballerinalang.testerina.test.utils.AssertionUtils; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.HashMap; + +import static org.ballerinalang.testerina.test.utils.CommonUtils.replaceExecutionTime; + +public class TestparallelizationTest extends BaseTestCase { + + private BMainInstance balClient; + private String projectPath; + + @BeforeClass + public void setup() throws BallerinaTestException { + balClient = new BMainInstance(balServer); + projectPath = projectBasedTestsPath.resolve("parallelisation-test").toString(); + } + + @Test + public void testParallelization() throws BallerinaTestException, IOException { + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-simple-test"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testParallelization_w1.txt", output); + + + args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-simple-test", }); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testParallelization_w30.txt", output); + + Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30); + + } + + @Test + public void testNonParallelizable() throws BallerinaTestException, IOException { + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-parallelizable"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testNonParallelizable_w1.txt", output); + + + args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-parallelizable", }); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testNonParallelizable_w30.txt", output); + + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + + @Test + public void testParalallelizableTupleDataProvider() throws BallerinaTestException, IOException { + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-tuple-data-provider"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableTupleDataProvider_w1.txt", output); + + + args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-tuple-data-provider", }); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableTupleDataProvider_w30.txt", output); + + Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30); + } + + @Test + public void testParalallelizableMapDataProvider() throws BallerinaTestException, IOException { + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-map-data-provider"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableMapDataProvider_w1.txt", output); + + + args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", }); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableMapDataProvider_w30.txt", output); + + Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30); + } + + @Test + public void testNonParalallelizableTupleDataProvider() throws BallerinaTestException, IOException { + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-parallelisation-tuple-data-provider"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableTupleDataProvider_w1.txt", +// output); + + + args = mergeCoverageArgs(new String[]{"--workers=1", "non-parallelisation-tuple-data-provider", }); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableTupleDataProvider_w30.txt", +// output); + + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testNonParalallelizableMapDataProvider() throws BallerinaTestException, IOException { + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-parallelisation-map-data-provider"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableMapDataProvider_w1.txt", output); + + + args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", }); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + output = replaceExecutionTime(output); +// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableMapDataProvider_w30.txt", output); + + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + private float getTimeForTestExecution(String output) { + int firstPos = output.indexOf("Test execution time :") + ("Test execution time :").length(); + int lastPos = output.indexOf("ms"); + String executionTime = output.substring(firstPos, lastPos); + return Float.parseFloat(executionTime); + } +} diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/CommonUtils.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/CommonUtils.java index d2a921f468d4..3ec2b442c7f3 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/CommonUtils.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/CommonUtils.java @@ -28,6 +28,10 @@ public class CommonUtils { private CommonUtils() { } + public static String replaceExecutionTime(String content) { + return replaceVaryingString("Test execution time :", "ms", content); + } + public static String replaceVaryingString(String firstString, String endString, String content) { String modifiedContent = content; int firstPos = modifiedContent.indexOf(firstString); diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal new file mode 100644 index 000000000000..6e72c437a4de --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal @@ -0,0 +1,60 @@ + +import ballerina/lang.runtime; +import ballerina/test; + +@test:Config { + dataProvider: mapDataProvider, + parallelizable: false +} +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.5); + test:assertEquals(fruit.length(), 6); +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal new file mode 100644 index 000000000000..24f48cf0f53e --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal @@ -0,0 +1,51 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:Config { + dataProvider: stringDataProvider, + parallelizable: false +} +function testAddingValues0(string fValue, string sValue, string result) returns error? { + int value1 = check 'int:fromString(fValue); + int value2 = check 'int:fromString(sValue); + int result1 = check 'int:fromString(result); + runtime:sleep(1); + test:assertEquals(value1 + value2, result1, msg = "Incorrect Sum"); +} + +function stringDataProvider() returns (string[][]) { + return [ + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"] + , + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"] + , + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"] + ]; +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/tests/data_driven_test.bal new file mode 100644 index 000000000000..4edbaafb8182 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/tests/data_driven_test.bal @@ -0,0 +1,59 @@ + +import ballerina/lang.runtime; +import ballerina/test; + +@test:Config { + dataProvider: mapDataProvider +} +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.5); + test:assertEquals(fruit.length(), 6); +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/tests/normalTest.bal new file mode 100644 index 000000000000..4763450f2cd9 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/tests/normalTest.bal @@ -0,0 +1,383 @@ + +import ballerina/test; +import ballerina/lang.runtime; + +@test:BeforeSuite +function beforeSuiteFunc() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testFunction() { + test:assertTrue(true, msg = "Failed!"); +} + +@test:Config {parallelizable: false} +function testAssertEquals1() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1]} +function testAssertEquals2() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2]} +function testAssertEquals3() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2]} +function testAssertEquals4() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals4]} +function testAssertEquals5() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals4, testAssertEquals5]} +function testAssertEquals6() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals7() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals6]} +function testAssertEquals8() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals4, testAssertEquals8]} +function testAssertEquals9() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals8, testAssertEquals7, testAssertEquals11]} +function testAssertEquals10() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals11() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals7]} +function testAssertEquals12() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals6]} +function testAssertEquals13() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals14() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals15() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals13]} +function testAssertEquals16() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals17() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals17]} +function testAssertEquals18() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals16]} +function testAssertEquals19() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals16]} +function testAssertEquals20() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals14]} +function testAssertEquals21() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals14, testAssertEquals15]} +function testAssertEquals22() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals15, testAssertEquals21]} +function testAssertEquals23() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals19, testAssertEquals20]} +function testAssertEquals24() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals20, testAssertEquals24, testAssertEquals12, testAssertEquals11]} +function testAssertEquals25() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals25, testAssertEquals24]} +function testAssertEquals26() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals24, testAssertEquals26]} +function testAssertEquals27() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals28() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +function testAssertEquals29() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +function testAssertEquals30() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +function testAssertEquals31() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +function testAssertEquals32() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +function testAssertEquals33() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals34() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals35() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals36() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals37() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals38() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals39() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals40() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals41() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals42() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals43() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals44() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals45() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals46() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals47() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals48() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals49() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals50() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals51() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals52() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals53() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals54() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals55() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals56() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals57() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals58() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals59() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals60() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +// After Suite Function + +@test:AfterSuite +function afterSuiteFunc() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/Ballerina.toml new file mode 100644 index 000000000000..10927eae3866 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/Ballerina.toml @@ -0,0 +1,7 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" + + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal new file mode 100644 index 000000000000..4e440ae4e673 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal @@ -0,0 +1,383 @@ + +import ballerina/test; +import ballerina/lang.runtime; + +@test:BeforeSuite +function beforeSuiteFunc() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +function testFunction() { + test:assertTrue(true, msg = "Failed!"); +} + +@test:Config {} +function testAssertEquals1() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1]} +function testAssertEquals2() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} +function testAssertEquals3() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} +function testAssertEquals4() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals4]} +function testAssertEquals5() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals4, testAssertEquals5], parallelizable: false} +function testAssertEquals6() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +function testAssertEquals7() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals6]} +function testAssertEquals8() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals4, testAssertEquals8]} +function testAssertEquals9() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals8, testAssertEquals7, testAssertEquals11]} +function testAssertEquals10() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals11() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals7]} +function testAssertEquals12() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals6]} +function testAssertEquals13() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals14() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +function testAssertEquals15() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals13]} +function testAssertEquals16() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +function testAssertEquals17() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals17]} +function testAssertEquals18() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals16]} +function testAssertEquals19() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals16]} +function testAssertEquals20() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals14]} +function testAssertEquals21() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals14, testAssertEquals15]} +function testAssertEquals22() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals15, testAssertEquals21]} +function testAssertEquals23() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals19, testAssertEquals20], parallelizable: false} +function testAssertEquals24() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals20, testAssertEquals24, testAssertEquals12, testAssertEquals11]} +function testAssertEquals25() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals25, testAssertEquals24]} +function testAssertEquals26() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals24, testAssertEquals26]} +function testAssertEquals27() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {parallelizable: false} +function testAssertEquals28() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +function testAssertEquals29() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +function testAssertEquals30() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +function testAssertEquals31() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +function testAssertEquals32() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +function testAssertEquals33() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals34() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals35() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals36() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals37() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals38() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals39() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals40() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals41() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals42() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals43() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals44() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals45() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals46() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals47() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals48() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals49() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals50() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals51() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals52() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals53() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals54() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals55() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals56() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals57() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals58() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals59() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +function testAssertEquals60() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +// After Suite Function + +@test:AfterSuite +function afterSuiteFunc() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal new file mode 100644 index 000000000000..ac145f2c4e51 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal @@ -0,0 +1,51 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:Config { + dataProvider: stringDataProvider, + parallelizable: true +} +function testAddingValues0(string fValue, string sValue, string result) returns error? { + int value1 = check 'int:fromString(fValue); + int value2 = check 'int:fromString(sValue); + int result1 = check 'int:fromString(result); + runtime:sleep(1); + test:assertEquals(value1 + value2, result1, msg = "Incorrect Sum"); +} + +function stringDataProvider() returns (string[][]) { + return [ + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"] + , + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"] + , + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"], + ["1", "2", "3"], + ["10", "20", "30"], + ["5", "6", "11"] + ]; +} diff --git a/tests/testerina-integration-test/src/test/resources/testng.xml b/tests/testerina-integration-test/src/test/resources/testng.xml index c827e46ae5b1..81b6bc32d891 100644 --- a/tests/testerina-integration-test/src/test/resources/testng.xml +++ b/tests/testerina-integration-test/src/test/resources/testng.xml @@ -54,6 +54,7 @@ under the License. + From 88c0f99bc26346cce5a1aba6c041818300dbcbdb Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Mon, 22 May 2023 11:47:58 +0530 Subject: [PATCH 04/37] Add concurrency safety validation --- .../main/ballerina/annotation_processor.bal | 28 ++++++++++- .../src/main/ballerina/annotations.bal | 4 +- .../src/main/ballerina/execute.bal | 6 ++- .../src/main/ballerina/external.bal | 5 ++ .../testerina/natives/CommonUtils.java | 47 ++++++++++++++++++- 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index a95f7c02267a..624180b74e3a 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -46,25 +46,51 @@ public function registerTest(string name, function f) { function processConfigAnnotation(string name, function f) returns boolean { TestConfig? config = (typeof f).@Config; if config != () { + boolean isTestFunctionIsolated = f is isolated function; + boolean isDataProviderIsolated = true; + boolean isTestFunctionParamSafe = true; + boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated; + string[] reasonToSerialExecution = []; + DataProviderReturnType? params = (); error? diagnostics = (); if config.dataProvider != () { var providerFn = config.dataProvider; if providerFn is function () returns (DataProviderReturnType?) { + isDataProviderIsolated = (providerFn is isolated function); + isTestFunctionParamSafe = isFunctionParamConcurrencySafe(f); + isSatisfiedParallelizableConditions = isTestFunctionIsolated && isDataProviderIsolated && isTestFunctionParamSafe; DataProviderReturnType providerOutput = providerFn(); params = providerOutput; } else { diagnostics = error("Failed to execute the data provider"); } + + if !isTestFunctionIsolated { + reasonToSerialExecution.push("non-isolated test function"); + } + + if !isDataProviderIsolated { + reasonToSerialExecution.push("non-isolated data-provider function"); + } + + if !isTestFunctionParamSafe { + reasonToSerialExecution.push("unsafe test parameters"); + } + if !isSatisfiedParallelizableConditions && !config.serialExecution { + println("WARNING : Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); + } + } + boolean enabled = config.enable && (filterGroups.length() == 0 ? true : hasGroup(config.groups, filterGroups)) && (filterDisableGroups.length() == 0 ? true : !hasGroup(config.groups, filterDisableGroups)) && hasTest(name); config.groups.forEach(group => groupStatusRegistry.incrementTotalTest(group, enabled)); testRegistry.addFunction(name = name, executableFunction = f, params = params, before = config.before, after = config.after, groups = config.groups, diagnostics = diagnostics, dependsOn = config.dependsOn, - enabled = enabled, dependsOnCount = config.dependsOn.length(), parallelizable = config.parallelizable, config = config); + enabled = enabled, dependsOnCount = config.dependsOn.length(), parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions), config = config); return true; } return false; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal index c3f7d682398e..3fb8e8722261 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal @@ -22,7 +22,7 @@ # + before - Function to be run before the test is run # + after - Function to be run after the test is run # + dependsOn - A list of functions the test function depends on and will be run before the test -# + parallelizable - Flag to enable/disable parallization for a test function +# + serialExecution - Flag to enable/disable parallization for a test function public type TestConfig record { boolean enable = true; string[] groups = []; @@ -30,7 +30,7 @@ public type TestConfig record { function () returns (any|error) before?; function () returns (any|error) after?; function[] dependsOn = []; - boolean parallelizable = true; + boolean serialExecution = false; }; # Configuration of the function to be mocked. diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 4b0d914e31af..142b56065974 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -141,6 +141,7 @@ function executeTest(TestFunction testFunction) returns error? { if diagnoseError is error { lock { reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); + println("\n****************************************************\n" + testFunction.name + " has failed.\n****************************************************\n"); } enableExit(); releaseWorker(); @@ -251,7 +252,7 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { } runtime:sleep(0.0001); } - + if !isIntialJob { allocateWorker(); } @@ -280,6 +281,7 @@ function executeDataDrivenTest(TestFunction testFunction, string suffix, TestTyp lock { reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + "]\n" + getErrorMessage(err), testType = testType); + println("\n****************************************************\n" + testFunction.name + ":" + suffix + " has failed.\n****************************************************\n"); enableExit(); } } @@ -300,6 +302,7 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { failed = true; lock { reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); + println("\n****************************************************\n" + testFunction.name + " has failed.\n****************************************************\n"); } } @@ -491,6 +494,7 @@ function executeTestFunction(TestFunction testFunction, string suffix, TestType enableExit(); lock { reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); + println("\n****************************************************\n" + testFunction.name + ":" + suffix + " has failed.\n****************************************************\n"); } return true; } else if output is any { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal index 53efdb6b6f58..a7bd808bab81 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal @@ -117,3 +117,8 @@ isolated function currentTimeInMillis() returns decimal = @java:Method { name: "currentTimeInMillis", 'class: "org.ballerinalang.testerina.natives.CommonUtils" } external; + +isolated function isFunctionParamConcurrencySafe(function func) returns boolean = @java:Method { + name: "isFunctionParamConcurrencySafe", + 'class: "org.ballerinalang.testerina.natives.CommonUtils" +} external; diff --git a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java index 042a989c06da..600450995eed 100644 --- a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java +++ b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java @@ -1,11 +1,22 @@ package org.ballerinalang.testerina.natives; +import io.ballerina.runtime.api.types.FunctionType; +import io.ballerina.runtime.api.types.ObjectType; +import io.ballerina.runtime.api.types.Parameter; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.UnionType; import io.ballerina.runtime.api.values.BDecimal; +import io.ballerina.runtime.api.values.BFunctionPointer; import io.ballerina.runtime.internal.util.exceptions.BLangExceptionHelper; import io.ballerina.runtime.internal.util.exceptions.RuntimeErrors; +import org.wso2.ballerinalang.compiler.semantics.model.symbols.Symbols; +import org.wso2.ballerinalang.compiler.semantics.model.types.BType; +import org.wso2.ballerinalang.compiler.semantics.model.types.BUnionType; +import org.wso2.ballerinalang.compiler.util.TypeTags; +import org.wso2.ballerinalang.util.Flags; public class CommonUtils { - public static Object sleep(BDecimal seconds) { + public static Object sleep(BDecimal seconds) { try { Thread.sleep(seconds.intValue() * 1000); } catch (InterruptedException e) { @@ -19,4 +30,38 @@ public static BDecimal currentTimeInMillis() { long currentTime = System.currentTimeMillis(); return BDecimal.valueOf(currentTime); } + + public static Object isFunctionParamConcurrencySafe(BFunctionPointer func) { + FunctionType functionType = (FunctionType) func.getType(); + Parameter[] functionParameters = functionType.getParameters(); + for (Parameter functionParameter : functionParameters) { + Type parameterType = functionParameter.type; + if (isSubTypeOfReadOnlyOrIsolatedObjectUnion(parameterType)) { + continue; + } + return false; + } + return true; + } + + private static boolean isSubTypeOfReadOnlyOrIsolatedObjectUnion(Type type) { + if (type.isReadOnly()) { + return true; + } + + if (type instanceof ObjectType) { + return ((ObjectType) type).isIsolated(); + } + + if (!(type instanceof UnionType)) { + return false; + } + + for (Type memberType : ((UnionType) type).getMemberTypes()) { + if (!isSubTypeOfReadOnlyOrIsolatedObjectUnion(memberType)) { + return false; + } + } + return true; + } } From 9ae9e791044bf71f2433467a35d54fe9b2b4ba70 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 24 May 2023 11:10:47 +0530 Subject: [PATCH 05/37] Add isolation related tests --- .../internal/testable/TestArguments.java | 3 +- .../io/ballerina/cli/utils/NativeUtils.java | 1 + .../main/ballerina/annotation_processor.bal | 25 +- .../test/TestparallelizationTest.java | 117 ++++-- .../Ballerina.toml | 0 .../main.bal | 0 .../tests/data_driven_test.bal | 58 +++ .../non-isolated-test-params/Ballerina.toml | 8 + .../non-isolated-test-params/main.bal | 55 +++ .../tests/data_driven_test.bal | 58 +++ .../non-isolated-tests/Ballerina.toml | 7 + .../non-isolated-tests/main.bal | 55 +++ .../non-isolated-tests/tests/normalTest.bal | 378 ++++++++++++++++++ .../tests/data_driven_test.bal | 2 +- .../tests/data_driven_test.bal | 2 +- .../Ballerina.toml | 8 + .../parallelisation-serialExecution/main.bal | 55 +++ .../tests/normalTest.bal | 122 +++--- .../tests/normalTest.bal | 10 +- .../tests/data_driven_test.bal | 3 +- 20 files changed, 842 insertions(+), 125 deletions(-) rename tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/{parallelisation-parallelizable => non-isolated-data-provider}/Ballerina.toml (100%) rename tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/{parallelisation-parallelizable => non-isolated-data-provider}/main.bal (100%) create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/tests/normalTest.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal rename tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/{parallelisation-parallelizable => parallelisation-serialExecution}/tests/normalTest.bal (74%) diff --git a/bvm/ballerina-runtime/src/main/java/io/ballerina/runtime/internal/testable/TestArguments.java b/bvm/ballerina-runtime/src/main/java/io/ballerina/runtime/internal/testable/TestArguments.java index 6b09dd6d3b27..73fb7031e6c4 100644 --- a/bvm/ballerina-runtime/src/main/java/io/ballerina/runtime/internal/testable/TestArguments.java +++ b/bvm/ballerina-runtime/src/main/java/io/ballerina/runtime/internal/testable/TestArguments.java @@ -28,7 +28,7 @@ */ public class TestArguments { - private static final int ARGUMENTS_NUMBER = 10; + private static final int ARGUMENTS_NUMBER = 11; private final Class[] argTypes; private final Object[] argValues; @@ -49,6 +49,7 @@ public TestArguments(String... args) { assignValues(7, args[7]); assignValues(8, args[8]); assignValues(9, args[9]); + assignValues(10, args[10]); } public Object[] getArgValues() { diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java index 08e495c8d391..60fd0d971828 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java @@ -74,6 +74,7 @@ public class NativeUtils { "io.ballerina.runtime.api.values.BString", "io.ballerina.runtime.api.values.BString", "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", "io.ballerina.runtime.api.values.BString" }); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 624180b74e3a..9001f73acd3a 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -66,22 +66,21 @@ function processConfigAnnotation(string name, function f) returns boolean { } else { diagnostics = error("Failed to execute the data provider"); } + } - if !isTestFunctionIsolated { - reasonToSerialExecution.push("non-isolated test function"); - } - - if !isDataProviderIsolated { - reasonToSerialExecution.push("non-isolated data-provider function"); - } + if !isTestFunctionIsolated { + reasonToSerialExecution.push("non-isolated test function"); + } - if !isTestFunctionParamSafe { - reasonToSerialExecution.push("unsafe test parameters"); - } - if !isSatisfiedParallelizableConditions && !config.serialExecution { - println("WARNING : Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); - } + if !isDataProviderIsolated { + reasonToSerialExecution.push("non-isolated data-provider function"); + } + if !isTestFunctionParamSafe { + reasonToSerialExecution.push("unsafe test parameters"); + } + if !isSatisfiedParallelizableConditions && !config.serialExecution { + println("WARNING : Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); } boolean enabled = config.enable && (filterGroups.length() == 0 ? true : hasGroup(config.groups, filterGroups)) diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index 6f69ed8c48ee..373a10bda3f5 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -29,38 +29,29 @@ public void testParallelization() throws BallerinaTestException, IOException { String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testParallelization_w1.txt", output); - - + Assert.assertTrue(output.contains("61 passing") && output.contains("0 failing")); args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-simple-test", }); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testParallelization_w30.txt", output); - + Assert.assertTrue(output.contains("61 passing") && output.contains("0 failing")); Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30); } @Test public void testNonParallelizable() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-parallelizable"}); + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-serialExecution"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testNonParallelizable_w1.txt", output); + Assert.assertTrue(output.contains("61 passing") && output.contains("0 failing")); - - args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-parallelizable", }); + args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-serialExecution", }); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testNonParallelizable_w30.txt", output); - + Assert.assertTrue(output.contains("61 passing") && output.contains("0 failing")); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -71,17 +62,12 @@ public void testParalallelizableTupleDataProvider() throws BallerinaTestExceptio String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableTupleDataProvider_w1.txt", output); - - + Assert.assertTrue(output.contains("30 passing") && output.contains("0 failing")); args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-tuple-data-provider", }); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableTupleDataProvider_w30.txt", output); - + Assert.assertTrue(output.contains("30 passing") && output.contains("0 failing")); Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30); } @@ -91,17 +77,13 @@ public void testParalallelizableMapDataProvider() throws BallerinaTestException, String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableMapDataProvider_w1.txt", output); - + Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", }); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableMapDataProvider_w30.txt", output); - + Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30); } @@ -111,19 +93,76 @@ public void testNonParalallelizableTupleDataProvider() throws BallerinaTestExcep String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableTupleDataProvider_w1.txt", -// output); + Assert.assertTrue(output.contains("30 passing") && output.contains("0 failing")); + + args = mergeCoverageArgs(new String[]{"--workers=1", "non-parallelisation-tuple-data-provider"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("30 passing") && output.contains("0 failing")); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testNonIsolatedTestFunction() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING : Test function 'testAssertEquals*' cannot be parallelized due to " + + "non-isolated test function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-tests"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("60 passing") && output.contains("0 failing")); + for (int testNo = 1; testNo < 61; testNo++) { + Assert.assertTrue(output.contains(warningDiagnostics.replace("*", Integer.toString(testNo)))); + } + args = mergeCoverageArgs(new String[]{"--workers=1", "non-isolated-tests"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("60 passing") && output.contains("0 failing")); + for (int testNo = 1; testNo < 61; testNo++) { + Assert.assertTrue(output.contains(warningDiagnostics.replace("*", Integer.toString(testNo)))); + } + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + @Test + public void testNonIsolatedTestParameter() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING : Test function 'mapDataProviderTest' cannot be parallelized due to " + + "unsafe test parameters"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-test-params"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=1", "non-parallelisation-tuple-data-provider", }); + args = mergeCoverageArgs(new String[]{"--workers=1", "non-isolated-test-params"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableTupleDataProvider_w30.txt", -// output); + Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + @Test + public void testNonIsolatedDataProvider() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING : Test function 'mapDataProviderTest' cannot be parallelized due to " + + "non-isolated data-provider function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-data-provider"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + + args = mergeCoverageArgs(new String[]{"--workers=1", "non-isolated-data-provider"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -133,17 +172,13 @@ public void testNonParalallelizableMapDataProvider() throws BallerinaTestExcepti String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableMapDataProvider_w1.txt", output); - + Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", }); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); - output = replaceExecutionTime(output); -// AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableMapDataProvider_w30.txt", output); - + Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/Ballerina.toml similarity index 100% rename from tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/Ballerina.toml rename to tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/Ballerina.toml diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/main.bal similarity index 100% rename from tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/main.bal rename to tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/main.bal diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/tests/data_driven_test.bal new file mode 100644 index 000000000000..c320b2838878 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/tests/data_driven_test.bal @@ -0,0 +1,58 @@ + +import ballerina/lang.runtime; +import ballerina/test; + +@test:Config { + dataProvider: mapDataProvider +} +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.5); +} + +public function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/tests/data_driven_test.bal new file mode 100644 index 000000000000..943bcf6d2777 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/tests/data_driven_test.bal @@ -0,0 +1,58 @@ + +import ballerina/lang.runtime; +import ballerina/test; + +@test:Config { + dataProvider: mapDataProvider +} +function mapDataProviderTest(int value1, int value2, string fruit, string[] emptyArr) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.5); + test:assertEquals(fruit.length(), 6); +} + +function mapDataProvider() returns map<[int, int, string, string[]]>|error { + map<[int, int, string, string[]]> dataSet = { + "banana": [10, 10, "banana", []], + "cherry": [5, 5, "cherry", []], + "apple": [5, 5, "apple", []], + "orange": [5, 5, "orange", []], + "carrot": [5, 5, "carrot", []], + "lemon": [5, 5, "lemon", []], + "tomatto": [5, 5, "tomatto", []], + "papaya": [5, 5, "papaya", []], + "grapes": [5, 5, "grapes", []], + "mango": [5, 5, "mango", []], + "pineapple": [5, 5, "pineapple", []], + "watermelon": [5, 5, "watermelon", []], + "strawberry": [5, 5, "strawberry", []], + "melon": [5, 5, "melon", []], + "guava": [5, 5, "guava", []], + "pomegranate": [5, 5, "pomegranate", []], + "jackfruit": [5, 5, "jackfruit", []], + "coconut": [5, 5, "coconut", []], + "peach": [5, 5, "peach", []], + "pear": [5, 5, "pear", []], + "plum": [5, 5, "plum", []], + "blueberry": [5, 5, "blueberry", []], + "raspberry": [5, 5, "raspberry", []], + "kiwi": [5, 5, "kiwi", []], + "avocado": [5, 5, "avocado", []], + "cucumber": [5, 5, "cucumber", []], + "pepper": [5, 5, "pepper", []], + "onion": [5, 5, "onion", []], + "potato": [5, 5, "potato", []], + "tomato": [5, 5, "tomato", []], + "garlic": [5, 5, "garlic", []], + "ginger": [5, 5, "ginger", []], + "spinach": [5, 5, "spinach", []], + "broccoli": [5, 5, "broccoli", []], + "cauliflower": [5, 5, "cauliflower", []], + "cabbage": [5, 5, "cabbage", []], + "beetroot": [5, 5, "beetroot", []], + "celery": [5, 5, "celery", []], + "corn": [5, 5, "corn", []], + "mushroom": [5, 5, "mushroom", []] + }; + return dataSet; +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/Ballerina.toml new file mode 100644 index 000000000000..10927eae3866 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/Ballerina.toml @@ -0,0 +1,7 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" + + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/tests/normalTest.bal new file mode 100644 index 000000000000..c92e46c81ce7 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/tests/normalTest.bal @@ -0,0 +1,378 @@ + +import ballerina/test; +import ballerina/lang.runtime; + +@test:BeforeSuite +public function beforeSuiteFunc() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +public function testAssertEquals1() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1]} +public function testAssertEquals2() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} +public function testAssertEquals3() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} +public function testAssertEquals4() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals4]} +public function testAssertEquals5() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals4, testAssertEquals5]} +public function testAssertEquals6() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +public function testAssertEquals7() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals6]} +public function testAssertEquals8() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals4, testAssertEquals8]} +public function testAssertEquals9() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals8, testAssertEquals7, testAssertEquals11]} +public function testAssertEquals10() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +public function testAssertEquals11() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals7]} +public function testAssertEquals12() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals6]} +public function testAssertEquals13() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +public function testAssertEquals14() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +public function testAssertEquals15() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals13]} +public function testAssertEquals16() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +public function testAssertEquals17() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals17]} +public function testAssertEquals18() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals16]} +public function testAssertEquals19() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals16]} +public function testAssertEquals20() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals14]} +public function testAssertEquals21() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals14, testAssertEquals15]} +public function testAssertEquals22() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals15, testAssertEquals21]} +public function testAssertEquals23() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals19, testAssertEquals20]} +public function testAssertEquals24() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals20, testAssertEquals24, testAssertEquals12, testAssertEquals11]} +public function testAssertEquals25() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals25, testAssertEquals24]} +public function testAssertEquals26() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals24, testAssertEquals26]} +public function testAssertEquals27() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {} +public function testAssertEquals28() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +public function testAssertEquals29() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +public function testAssertEquals30() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +public function testAssertEquals31() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +public function testAssertEquals32() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals28]} +public function testAssertEquals33() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals34() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals35() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals36() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals37() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals38() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals39() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals40() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals41() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals42() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals43() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals44() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals45() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals46() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals47() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals48() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals49() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals50() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals51() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals52() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals53() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals54() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals55() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals56() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals57() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals58() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals59() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +@test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10]} +public function testAssertEquals60() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + +// After Suite public function + +@test:AfterSuite +public function afterSuiteFunc() { + runtime:sleep(0.5); + test:assertEquals(100, 100); +} + diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal index 6e72c437a4de..8a57de038e91 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal @@ -4,7 +4,7 @@ import ballerina/test; @test:Config { dataProvider: mapDataProvider, - parallelizable: false + serialExecution: true } function mapDataProviderTest(int value1, int value2, string fruit) returns error? { test:assertEquals(value1, value2, msg = "The provided values are not equal"); diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal index 24f48cf0f53e..b8b3bfbb8e7e 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/tests/data_driven_test.bal @@ -3,7 +3,7 @@ import ballerina/test; @test:Config { dataProvider: stringDataProvider, - parallelizable: false + serialExecution: true } function testAddingValues0(string fValue, string sValue, string result) returns error? { int value1 = check 'int:fromString(fValue); diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/tests/normalTest.bal similarity index 74% rename from tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/tests/normalTest.bal rename to tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/tests/normalTest.bal index 4763450f2cd9..677a7ea8ae3c 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-parallelizable/tests/normalTest.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/tests/normalTest.bal @@ -8,366 +8,366 @@ function beforeSuiteFunc() { test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testFunction() { test:assertTrue(true, msg = "Failed!"); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals1() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1]} function testAssertEquals2() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2]} function testAssertEquals3() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2]} function testAssertEquals4() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals4]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals4]} function testAssertEquals5() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals4, testAssertEquals5]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals4, testAssertEquals5]} function testAssertEquals6() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals7() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals6]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals6]} function testAssertEquals8() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals4, testAssertEquals8]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals4, testAssertEquals8]} function testAssertEquals9() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals8, testAssertEquals7, testAssertEquals11]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals8, testAssertEquals7, testAssertEquals11]} function testAssertEquals10() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals11() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals7]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals7]} function testAssertEquals12() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals6]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals6]} function testAssertEquals13() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals14() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals15() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals13]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals13]} function testAssertEquals16() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals17() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals17]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals17]} function testAssertEquals18() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals16]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals16]} function testAssertEquals19() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals16]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals16]} function testAssertEquals20() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals14]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals14]} function testAssertEquals21() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals14, testAssertEquals15]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals14, testAssertEquals15]} function testAssertEquals22() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals15, testAssertEquals21]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals15, testAssertEquals21]} function testAssertEquals23() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals19, testAssertEquals20]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals19, testAssertEquals20]} function testAssertEquals24() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals20, testAssertEquals24, testAssertEquals12, testAssertEquals11]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals20, testAssertEquals24, testAssertEquals12, testAssertEquals11]} function testAssertEquals25() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals25, testAssertEquals24]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals25, testAssertEquals24]} function testAssertEquals26() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals24, testAssertEquals26]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals24, testAssertEquals26]} function testAssertEquals27() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals28() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals28]} function testAssertEquals29() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals28]} function testAssertEquals30() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals28]} function testAssertEquals31() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals28]} function testAssertEquals32() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals28]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals28]} function testAssertEquals33() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals34() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals35() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals36() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals37() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals38() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals39() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals40() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals41() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals42() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals43() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals44() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals45() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals46() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals47() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals48() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals49() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals50() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals51() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals52() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals53() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals54() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals55() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals56() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals57() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals58() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals59() { runtime:sleep(0.5); test:assertEquals(100, 100); } -@test:Config {parallelizable: false, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} +@test:Config {serialExecution: true, dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals60() { runtime:sleep(0.5); test:assertEquals(100, 100); diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal index 4e440ae4e673..3e04d23518d9 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal @@ -43,7 +43,7 @@ function testAssertEquals5() { test:assertEquals(100, 100); } -@test:Config {dependsOn: [testAssertEquals4, testAssertEquals5], parallelizable: false} +@test:Config {dependsOn: [testAssertEquals4, testAssertEquals5], serialExecution: true} function testAssertEquals6() { runtime:sleep(0.5); test:assertEquals(100, 100); @@ -73,7 +73,7 @@ function testAssertEquals10() { test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals11() { runtime:sleep(0.5); test:assertEquals(100, 100); @@ -91,7 +91,7 @@ function testAssertEquals13() { test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals14() { runtime:sleep(0.5); test:assertEquals(100, 100); @@ -151,7 +151,7 @@ function testAssertEquals23() { test:assertEquals(100, 100); } -@test:Config {dependsOn: [testAssertEquals19, testAssertEquals20], parallelizable: false} +@test:Config {dependsOn: [testAssertEquals19, testAssertEquals20], serialExecution: true} function testAssertEquals24() { runtime:sleep(0.5); test:assertEquals(100, 100); @@ -175,7 +175,7 @@ function testAssertEquals27() { test:assertEquals(100, 100); } -@test:Config {parallelizable: false} +@test:Config {serialExecution: true} function testAssertEquals28() { runtime:sleep(0.5); test:assertEquals(100, 100); diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal index ac145f2c4e51..14b9a4db316c 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/tests/data_driven_test.bal @@ -2,8 +2,7 @@ import ballerina/lang.runtime; import ballerina/test; @test:Config { - dataProvider: stringDataProvider, - parallelizable: true + dataProvider: stringDataProvider } function testAddingValues0(string fValue, string sValue, string result) returns error? { int value1 = check 'int:fromString(fValue); From 7564640944c09fbdab99c459632007bc3b57a9c7 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 24 May 2023 13:46:35 +0530 Subject: [PATCH 06/37] Modify integration test output --- .../main/ballerina/annotation_processor.bal | 2 +- .../test/TestparallelizationTest.java | 6 +- .../testerina/test/utils/AssertionUtils.java | 2 + .../unix/BasicCasesTest-testAnnotations.txt | 11 ++- .../unix/BasicCasesTest-testAnydataType.txt | 3 + ...sicCasesTest-testAssertBehavioralTypes.txt | 3 + .../BasicCasesTest-testAssertDiffError.txt | 3 + ...BasicCasesTest-testAssertSequenceTypes.txt | 3 + ...sicCasesTest-testAssertStructuralTypes.txt | 3 + ...sicCasesTest-testAssertionErrorMessage.txt | 3 + .../unix/BasicCasesTest-testAssertions.txt | 3 + .../BasicCasesTest-testAsyncInvocation.txt | 3 + .../unix/BasicCasesTest-testBeforeAfter.txt | 3 + ...BasicCasesTest-testBeforeEachAfterEach.txt | 3 + .../unix/BasicCasesTest-testDependsOn.txt | 11 ++- .../BasicCasesTest-testIntersectionTypes.txt | 3 + .../BasicCasesTest-testIsolatedFunctions.txt | 5 +- .../unix/BasicCasesTest-testJavaInterops.txt | 3 + .../unix/BasicCasesTest-testRuntimeApi.txt | 3 + ...iderTest-testArrayDataProviderWithFail.txt | 16 ++++ ...viderTest-testArrayDataRerunFailedTest.txt | 16 ++++ ...DataProviderTest-testCodeFragmentKeys0.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys1.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys2.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys3.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys4.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys5.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys6.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard0.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard1.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard2.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard3.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard4.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard5.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard6.txt | 6 ++ ...iderTest-testDataProviderSingleFailure.txt | 17 ++++- ...iderTest-testDataProviderWithMixedType.txt | 6 ++ ...aProviderTest-testMapValueDataProvider.txt | 11 +++ ...iderTest-testMultiModuleSingleTestExec.txt | 6 ++ .../DataProviderTest-testRerunFailedTest.txt | 16 ++++ ...DataProviderTest-testValidDataProvider.txt | 6 ++ ...ProviderTest-testValidDataProviderCase.txt | 6 ++ ...-testValidDataProviderWithAfterFailing.txt | 6 ++ ...idDataProviderWithBeforeAfterFunctions.txt | 6 ++ ...testValidDataProviderWithBeforeFailing.txt | 6 ++ ...iderTest-testValidDataProviderWithFail.txt | 16 ++++ .../DataProviderTest-testWithSpecialKeys.txt | 6 ++ ...isableTestsTestCase-testDisablingTests.txt | 3 + ...lidationTest-validateFunctionNamesTest.txt | 3 + ...oupingTest-afterGroupsWithDisabledTest.txt | 3 + .../GroupingTest-beforeGroupsAfterGroups1.txt | 3 + .../GroupingTest-beforeGroupsAfterGroups2.txt | 3 + .../GroupingTest-failedBeforeEachTest.txt | 3 + .../GroupingTest-failedBeforeGroupTest.txt | 3 + ...roupingTest-testMultipleGroupExclusion.txt | 3 + ...roupingTest-testMultipleGroupExecution.txt | 3 + ...pingTest-testNonExistingGroupExclusion.txt | 13 ++++ ...pingTest-testNonExistingGroupInclusion.txt | 3 + .../GroupingTest-testSingleGroupExclusion.txt | 8 ++ .../GroupingTest-testSingleGroupExecution.txt | 3 + .../GroupingTest-testWhenAfterGroupsFails.txt | 3 + .../unix/ImportTest-testImportTest.txt | 3 + ...oviderTestCase-testInvalidDataProvider.txt | 20 +++-- ...viderTestCase-testInvalidDataProvider2.txt | 20 +++-- ...rTestCase-testInvalidTupleDataProvider.txt | 20 +++-- ...stMockingFunctionWithIncompatibleTypes.txt | 8 ++ .../unix/MockTest-testCoverageWithMocking.txt | 6 ++ ...kTest-testFuncMockInMultiModulesWDepen.txt | 6 ++ .../unix/MockTest-testFunctionMocking.txt | 21 ++++++ .../MockTest-testFunctionMockingLegacy.txt | 3 + ...ockTest-testFunctionMockingModuleLevel.txt | 21 ++++++ ...FunctionMockingThenReturnWithNilRetVal.txt | 8 ++ .../unix/MockTest-testObjectMockDouble.txt | 3 + .../unix/MockTest-testObjectMocking.txt | 74 ++++++++++++++----- ...kTest-testObjectMocking_NegativeCases1.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases2.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases3.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases4.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases5.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases6.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases7.txt | 8 ++ ...kTest-testObjectMocking_NonPublicField.txt | 8 ++ ...ecutionFlowTest-test-listener-shutdown.txt | 8 +- ...cutionTest-test_DefaultModule_AllTests.txt | 6 ++ ...est-test_DefaultModule_EndWildCardTest.txt | 6 ++ ...-test_DefaultModule_MiddleWildCardTest.txt | 6 ++ ...tionTest-test_DefaultModule_SingleTest.txt | 6 ++ ...t-test_DefaultModule_StartWildCardTest.txt | 6 ++ ...uleExecutionTest-test_Module1_AllTests.txt | 6 ++ ...eExecutionTest-test_Module1_SingleTest.txt | 6 ++ ...xecutionTest-test_Module1_WildCardTest.txt | 6 ++ ...eExecutionTest-test_Module1_WithGroups.txt | 6 ++ .../ModuleExecutionTest-test_WildCardTest.txt | 6 ++ ...erificationTest-verifyTestsOutsidePath.txt | 3 + .../unix/RerunFailedTest-testFullTest.txt | 13 ++++ .../RerunFailedTest-testRerunFailedTest.txt | 13 ++++ ...ionTest-testDependentFunctionExecution.txt | 3 + ...tionTest-testDisabledFunctionExecution.txt | 3 + ...tionTest-testMultipleFunctionExecution.txt | 3 + ...nTest-testNonExistingFunctionExecution.txt | 3 + ...nctionTest-testSingleFunctionExecution.txt | 3 + ...stsTestCase-testSkipWhenAfterEachFails.txt | 3 + ...ipTestsTestCase-testSkipWhenAfterFails.txt | 3 + ...tsTestCase-testSkipWhenBeforeEachFails.txt | 3 + ...pTestsTestCase-testSkipWhenBeforeFails.txt | 3 + ...TestCase-testSkipWhenBeforeGroupsFails.txt | 3 + ...sTestCase-testSkipWhenBeforeSuiteFails.txt | 3 + ...ase-testSkipWhenDependsOnFunctionFails.txt | 8 ++ ..._DefaultModuleSourceOnly_TestExecution.txt | 6 ++ ...ts-test_SourcelessModule_TestExecution.txt | 6 ++ ...s-test_SourcelessProject_TestExecution.txt | 9 +++ ...tTest-testWarningForCoverageFormatFlag.txt | 14 ++++ ...stReportTest-testWarningForReportTools.txt | 14 ++++ 113 files changed, 768 insertions(+), 53 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 9001f73acd3a..b402cbef0777 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -79,7 +79,7 @@ function processConfigAnnotation(string name, function f) returns boolean { if !isTestFunctionParamSafe { reasonToSerialExecution.push("unsafe test parameters"); } - if !isSatisfiedParallelizableConditions && !config.serialExecution { + if !isSatisfiedParallelizableConditions && !config.serialExecution && (testWorkers > 1) { println("WARNING : Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); } diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index 373a10bda3f5..8f79004c9090 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -115,7 +115,7 @@ public void testNonIsolatedTestFunction() throws BallerinaTestException, IOExcep for (int testNo = 1; testNo < 61; testNo++) { Assert.assertTrue(output.contains(warningDiagnostics.replace("*", Integer.toString(testNo)))); } - args = mergeCoverageArgs(new String[]{"--workers=1", "non-isolated-tests"}); + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-tests"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); @@ -137,7 +137,7 @@ public void testNonIsolatedTestParameter() throws BallerinaTestException, IOExce Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=1", "non-isolated-test-params"}); + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-test-params"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); @@ -157,7 +157,7 @@ public void testNonIsolatedDataProvider() throws BallerinaTestException, IOExcep Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=1", "non-isolated-data-provider"}); + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-data-provider"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/AssertionUtils.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/AssertionUtils.java index c7fe87e4b36c..484856c16ac3 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/AssertionUtils.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/AssertionUtils.java @@ -48,9 +48,11 @@ public static void assertForTestFailures(String programOutput, String errMessage public static void assertOutput(String outputFileName, String output) throws IOException { if (isWindows) { + output = CommonUtils.replaceExecutionTime(output); String fileContent = Files.readString(commandOutputsDir.resolve("windows").resolve(outputFileName)); Assert.assertEquals(output.replaceAll("\r\n|\r", "\n"), fileContent.replaceAll("\r\n|\r", "\n")); } else { + output = CommonUtils.replaceExecutionTime(output); String fileContent = Files.readString(commandOutputsDir.resolve("unix").resolve(outputFileName)); Assert.assertEquals(output.stripTrailing(), fileContent.stripTrailing()); } diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt index 8ad5a1616210..3c43d99f9a39 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt @@ -4,11 +4,10 @@ Compiling source Running Tests with Coverage annotations + + Test execution time :*****ms + [pass] jsonDataProviderTest#0 - [pass] stringDataProviderTest#0 - [pass] stringDataProviderTest#1 - [pass] stringDataProviderTest#2 - [pass] stringDataProviderTest2#0 [pass] test1 [pass] test2 [pass] test3 @@ -18,6 +17,10 @@ Running Tests with Coverage [pass] testAfter [pass] testAfterAlone [pass] testDependsOn1 + [pass] stringDataProviderTest#0 + [pass] stringDataProviderTest#1 + [pass] stringDataProviderTest#2 + [pass] stringDataProviderTest2#0 14 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnydataType.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnydataType.txt index e80746da0c4b..251a100f91a7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnydataType.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnydataType.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage anydata_type + + Test execution time :*****ms + [pass] testFn [pass] testFoo diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertBehavioralTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertBehavioralTypes.txt index 8094ac7014c8..e73dcf8230f0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertBehavioralTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertBehavioralTypes.txt @@ -7,6 +7,9 @@ WARNING [tests/assert-test-behavioral-types.bal:(24:5,24:32)] undocumented field Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertObjectArrayEquals [pass] testAssertObjectArrayEqualsNegative [pass] testAssertObjectArrayNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertDiffError.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertDiffError.txt index 4b559fabe080..cf58c29748bb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertDiffError.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertDiffError.txt @@ -7,6 +7,9 @@ WARNING [tests/assertions-diff-error-messages.bal:(24:5,24:32)] undocumented fie Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertDecimalValues [pass] testAssertIntValues [pass] testAssertJsonArray diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertSequenceTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertSequenceTypes.txt index 5039e3ad3b74..dca971041dc8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertSequenceTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertSequenceTypes.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertStringEquals [pass] testAssertStringEqualsNegative [pass] testAssertStringNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertStructuralTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertStructuralTypes.txt index c505d701066d..d9eba46174f4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertStructuralTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertStructuralTypes.txt @@ -7,6 +7,9 @@ WARNING [tests/assert-test-structural-types.bal:(24:5,24:18)] undocumented field Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertByteArrayEquals [pass] testAssertByteArrayEqualsNegative [pass] testAssertByteArrayNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertionErrorMessage.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertionErrorMessage.txt index 2a1f6069b005..88f3c4f861b4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertionErrorMessage.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertionErrorMessage.txt @@ -7,6 +7,9 @@ WARNING [tests/assertions-error-messages.bal:(24:5,24:32)] undocumented field 'p Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertAnnonymousRecords [pass] testAssertDecimalAndFloat [pass] testAssertDifferentObjects diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertions.txt index 174d59f90c93..a38252892ddf 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAssertions.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertBooleanEquals [pass] testAssertBooleanEqualsNegative [pass] testAssertBooleanNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAsyncInvocation.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAsyncInvocation.txt index 324b12bb6388..b838fd755867 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAsyncInvocation.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAsyncInvocation.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage async + + Test execution time :*****ms + [pass] testAsyncRemoteFunctionMain [pass] testAsyncRemoteFunctionTest diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeAfter.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeAfter.txt index fcadd56160e8..9801f014f366 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeAfter.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeAfter.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage before_after + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeEachAfterEach.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeEachAfterEach.txt index 604adaa14094..6f0ff836e2e6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeEachAfterEach.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testBeforeEachAfterEach.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage before_each_after_each + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt index 6c1ff9fa82d7..157431325eb7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt @@ -4,18 +4,21 @@ Compiling source Running Tests with Coverage depends_on + + Test execution time :*****ms + [pass] test1 - [pass] testAddingValues#0 - [pass] testAddingValues#1 - [pass] testAddingValues#2 [pass] test2 - [pass] testAddingValuesDependant [pass] test3 [pass] test4 [pass] testWithBefore1 [pass] testWithBefore2 [pass] testWithBefore3 [pass] testWithBefore4 + [pass] testAddingValues#0 + [pass] testAddingValues#1 + [pass] testAddingValues#2 + [pass] testAddingValuesDependant 12 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIntersectionTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIntersectionTypes.txt index c72c7d85dea8..4c8da78dd02f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIntersectionTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIntersectionTypes.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage intersection_type + + Test execution time :*****ms + [pass] testIntersectionTypes diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt index 02a776acccfd..970ff353a1a7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt @@ -4,8 +4,11 @@ Compiling source Running Tests with Coverage isolated_functions - [pass] testCallingIsolatedFunction + + Test execution time :*****ms + [pass] testFunc + [pass] testCallingIsolatedFunction [pass] testIsolatedTestFunction diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testJavaInterops.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testJavaInterops.txt index acdf428a0ccb..d34f50bafa2a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testJavaInterops.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testJavaInterops.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage interops + + Test execution time :*****ms + [pass] testInteropWithRestArgs diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testRuntimeApi.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testRuntimeApi.txt index 3e4b90f908ae..2fefbc59503f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testRuntimeApi.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testRuntimeApi.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage runtime_api + + Test execution time :*****ms + [pass] testRuntimeApi diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt index a5d144f7e87c..b15d0858176b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt @@ -6,6 +6,19 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +intArrayDataProviderTest:0 has failed. +**************************************************** + + +**************************************************** +intArrayDataProviderTest:1 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] intArrayDataProviderTest#2 [fail] intArrayDataProviderTest#0: @@ -39,6 +52,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt index 538e76599098..7322a902d51b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt @@ -7,6 +7,19 @@ Running Tests with Coverage dataproviders +**************************************************** +intArrayDataProviderTest:0 has failed. +**************************************************** + + +**************************************************** +intArrayDataProviderTest:1 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] intArrayDataProviderTest#0: error {ballerina/test:0}TestError ("The sum is not correct @@ -38,6 +51,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys0.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys0.txt index cdcfc3bafd4a..1a13e31ed4de 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys0.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys0.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#`'"\a"" @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys1.txt index 4c60ebe25dba..9cd385a52bde 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys1.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#"\u{D7FF}"" " @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys2.txt index e2fc31ea782c..18e4e31d9fab 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys2.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a + b @@ -17,6 +20,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys3.txt index 6f37022e5334..df714964b2f7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys3.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(x * 1) != (y / 3) || (a ^ b) == (b & c) >> (1 % 2) @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys4.txt index 8a335dc5b4b6..361c0e4c93f3 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys4.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(1 @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys5.txt index 5b776d3557eb..7440c74f32d5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys5.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a:x(c,d)[]; ^(x|y).ok(); @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys6.txt index be52ca3dbfa0..3e1e46873468 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeys6.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#map v = { "x": 1 }; @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt index cdcfc3bafd4a..1a13e31ed4de 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#`'"\a"" @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt index 4c60ebe25dba..9cd385a52bde 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#"\u{D7FF}"" " @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt index e2fc31ea782c..18e4e31d9fab 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a + b @@ -17,6 +20,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt index 6f37022e5334..df714964b2f7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(x * 1) != (y / 3) || (a ^ b) == (b & c) >> (1 % 2) @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt index 8a335dc5b4b6..361c0e4c93f3 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(1 @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt index 5b776d3557eb..7440c74f32d5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a:x(c,d)[]; ^(x|y).ok(); @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt index be52ca3dbfa0..3e1e46873468 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#map v = { "x": 1 }; @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt index 6ab81b27d362..6108305ae478 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt @@ -6,6 +6,14 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +testDividingValuesNegative:2 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testDividingValuesNegative#0 [pass] testDividingValuesNegative#1 [pass] testDividingValuesNegative#3 @@ -19,9 +27,9 @@ Running Tests with Coverage callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 17 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 @@ -31,6 +39,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderWithMixedType.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderWithMixedType.txt index 7ed941639e7f..d5ecda443430 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderWithMixedType.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderWithMixedType.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction1#CaseNew1 [pass] testFunction1#CaseNew2 @@ -16,6 +19,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt index c7739517debf..cab0fb57f9b5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt @@ -6,6 +6,14 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +testGetState:1 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testGetState#0 [fail] testGetState#1: @@ -27,6 +35,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMultiModuleSingleTestExec.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMultiModuleSingleTestExec.txt index 83d80ca79a27..8a406dd6afc1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMultiModuleSingleTestExec.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMultiModuleSingleTestExec.txt @@ -7,10 +7,16 @@ Running Tests with Coverage dataproviders + Test execution time :*****ms + + No tests found dataproviders.module1 + + Test execution time :*****ms + [pass] stringDataProviderMod1Test#1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt index 94506c7f8a60..6647c78558fd 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt @@ -7,6 +7,19 @@ Running Tests with Coverage dataproviders +**************************************************** +intDataProviderTest:Case1 has failed. +**************************************************** + + +**************************************************** +intDataProviderTest:Case2 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] intDataProviderTest#Case1: error {ballerina/test:0}TestError ("The sum is not correct @@ -38,6 +51,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProvider.txt index 71d15324a483..1b52641d6a2d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProvider.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] jsonDataProviderTest#json1 [pass] jsonDataProviderTest#json2 @@ -16,6 +19,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderCase.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderCase.txt index 60384e0ed00e..f4df9532f26d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderCase.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderCase.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] jsonDataProviderTest#json1 @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt index d36d71cebf35..6d116d4dcbe4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt @@ -11,6 +11,9 @@ Running Tests with Coverage callableName: afterFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 153 callableName: afterFailsFunction$lambda37$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 + + Test execution time :*****ms + [pass] testDividingValuesWithAfterFailing#0 [pass] testDividingValuesWithAfterFailing#1 [pass] testDividingValuesWithAfterFailing#2 @@ -25,6 +28,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt index f14baa102f1b..77c327f83721 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testDividingValues#0 [pass] testDividingValues#1 [pass] testDividingValues#2 @@ -20,6 +23,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt index 63994fce48ca..951bd75e0574 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt @@ -11,6 +11,9 @@ Running Tests with Coverage callableName: beforeFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 121 callableName: beforeFailsFunction$lambda33$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 + + Test execution time :*****ms + [pass] testDividingValuesWithBeforeFailing#0 [pass] testDividingValuesWithBeforeFailing#1 [pass] testDividingValuesWithBeforeFailing#3 @@ -24,6 +27,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt index ed9c1bd143c4..8148cd806279 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt @@ -6,6 +6,19 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +intDataProviderTest:Case1 has failed. +**************************************************** + + +**************************************************** +intDataProviderTest:Case2 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] intDataProviderTest#Case3 [fail] intDataProviderTest#Case1: @@ -39,6 +52,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testWithSpecialKeys.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testWithSpecialKeys.txt index 703235c928c8..d5f8fefd43b0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testWithSpecialKeys.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testWithSpecialKeys.txt @@ -6,6 +6,9 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction2#Case#1 [pass] testFunction2#Case#2 [pass] testFunction2#Case#3 @@ -17,6 +20,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DisableTestsTestCase-testDisablingTests.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DisableTestsTestCase-testDisablingTests.txt index 00f88dc26288..5fcf5df6b14e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DisableTestsTestCase-testDisablingTests.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DisableTestsTestCase-testDisablingTests.txt @@ -6,6 +6,9 @@ Compiling source Running Tests disable-test.bal + + Test execution time :*****ms + [pass] testDisableFunc2 [pass] testDisableFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/FunctionNameValidationTest-validateFunctionNamesTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/FunctionNameValidationTest-validateFunctionNamesTest.txt index 19d69c3fc610..837ef4d7aa80 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/FunctionNameValidationTest-validateFunctionNamesTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/FunctionNameValidationTest-validateFunctionNamesTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage validate_function_names + + Test execution time :*****ms + [pass] validateFunctionName diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-afterGroupsWithDisabledTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-afterGroupsWithDisabledTest.txt index 329c8df44d91..6c61704d237c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-afterGroupsWithDisabledTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-afterGroupsWithDisabledTest.txt @@ -6,6 +6,9 @@ Compiling source Running Tests after-groups-with-disabled-test.bal + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups1.txt index 86daf222f532..8b9c3327459b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups1.txt @@ -6,6 +6,9 @@ Compiling source Running Tests before-groups-after-groups-test.bal + + Test execution time :*****ms + [pass] testFunction [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt index cb5079d3b3cb..101b23b28c97 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt @@ -6,6 +6,9 @@ Compiling source Running Tests before-groups-after-groups-test2.bal + + Test execution time :*****ms + [fail] afterSuiteFunc[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeEachTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeEachTest.txt index cad1ee3cdd5a..a6d7cd0c08f4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeEachTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeEachTest.txt @@ -14,6 +14,9 @@ Running Tests callableName: beforeEachFunction$lambda4$ fileName: failed-before-each-with-groups.bal lineNumber: 85 + Test execution time :*****ms + + 0 passing 0 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeGroupTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeGroupTest.txt index 69742eaaedc4..f4748a02783d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeGroupTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-failedBeforeGroupTest.txt @@ -11,6 +11,9 @@ Running Tests callableName: beforeGroupsFunc1 fileName: failed-before-groups-test.bal lineNumber: 28 callableName: beforeGroupsFunc1$lambda1$ fileName: failed-before-groups-test.bal lineNumber: 95 + + Test execution time :*****ms + [pass] testFunction [pass] testFunction4 [pass] testFunction5 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExclusion.txt index ce698ac1b0b0..75ab1cf48466 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExclusion.txt @@ -6,6 +6,9 @@ Compiling source Running Tests groups-test.bal + + Test execution time :*****ms + [pass] testFunc6 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExecution.txt index b30a265a09cf..38c77b98e245 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testMultipleGroupExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests groups-test.bal + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt index c06402b6d9e9..bf3cc441adae 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt @@ -6,6 +6,19 @@ Compiling source Running Tests groups-test.bal + +**************************************************** +testFunc4: has failed. +**************************************************** + + +**************************************************** +testFunc5: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupInclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupInclusion.txt index 90f822f17937..f1de4bdf0378 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupInclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupInclusion.txt @@ -7,5 +7,8 @@ Running Tests groups-test.bal + Test execution time :*****ms + + No tests found \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt index b8ed161ddef9..c4316eef0276 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt @@ -6,6 +6,14 @@ Compiling source Running Tests groups-test.bal + +**************************************************** +testFunc5: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExecution.txt index b30a265a09cf..38c77b98e245 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests groups-test.bal + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testWhenAfterGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testWhenAfterGroupsFails.txt index 58d0ff75bc89..87aeb629e9c9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testWhenAfterGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testWhenAfterGroupsFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: afterGroupsFunc1 fileName: failed-after-groups-test.bal lineNumber: 37 callableName: afterGroupsFunc1$lambda3$ fileName: failed-after-groups-test.bal lineNumber: 79 + + Test execution time :*****ms + [pass] testFunction [pass] testFunction2 [pass] testFunction3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ImportTest-testImportTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ImportTest-testImportTest.txt index 62933d7933f2..7e070263545a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ImportTest-testImportTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ImportTest-testImportTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage pre_declared_imports + + Test execution time :*****ms + [pass] testImportTest1 [pass] testImportTest2 [pass] testImportTest3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt index 00cd9ce11e26..691b426e44af 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -8,18 +8,26 @@ Running Tests invalid-data-provider-test.bal +**************************************************** +testInvalidDataProvider:0 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testInvalidDataProvider#0: [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 463 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index d1d413b206e6..89a83384cf0b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -10,18 +10,26 @@ Running Tests invalid-data-provider-test2.bal +**************************************************** +testInvalidDataProvider2:0 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testInvalidDataProvider2#0: [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 463 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 94e214f74b65..4ee3c92d555b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -7,18 +7,26 @@ Running Tests invalid-data-provider-test3.bal +**************************************************** +testInvalidTupleDataProvider:0 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testInvalidTupleDataProvider#0: [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 463 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 477 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 252 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 241 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt index 70cd2ef15226..be2b6d68d317 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt @@ -5,6 +5,14 @@ Running Tests incompatible_type_mock +**************************************************** +functionMockingTest has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] functionMockingTest: error {ballerina/test:0}FunctionSignatureMismatchError ("Return type of function stringHello does not match function intAdd") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testCoverageWithMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testCoverageWithMocking.txt index 78cfc93406a5..1d8cf42fa8b7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testCoverageWithMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testCoverageWithMocking.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage mocking_coverage.mod2 + + Test execution time :*****ms + [pass] testFloatAdd [pass] testStringAdd @@ -13,6 +16,9 @@ Running Tests with Coverage 0 skipped mocking_coverage + + Test execution time :*****ms + [pass] testFloatAdd [pass] testStringAdd diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFuncMockInMultiModulesWDepen.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFuncMockInMultiModulesWDepen.txt index 63505dbaf9ed..74682d9f167f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFuncMockInMultiModulesWDepen.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFuncMockInMultiModulesWDepen.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage function_mocking_with_dependencies.a_dependency + + Test execution time :*****ms + [pass] test1 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped function_mocking_with_dependencies.b_dependent + + Test execution time :*****ms + [pass] test1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt index 684f5e227a10..6ed66acea85b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt @@ -4,6 +4,24 @@ Compiling source Running Tests with Coverage function_mocking + +**************************************************** +call_Test3 has failed. +**************************************************** + + +**************************************************** +call_Test4 has failed. +**************************************************** + + +**************************************************** +call_Test5 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] callOriginal_Test1 [pass] callOriginal_Test2 [pass] callOriginal_Test3 @@ -49,6 +67,9 @@ Running Tests with Coverage 0 skipped function_mocking.mock2 + + Test execution time :*****ms + [pass] test1 [pass] test2 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingLegacy.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingLegacy.txt index b808cc17140d..8caac34075f6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingLegacy.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingLegacy.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage function_mocking_legacy + + Test execution time :*****ms + [pass] testIntAdd diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt index 684f5e227a10..6ed66acea85b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt @@ -4,6 +4,24 @@ Compiling source Running Tests with Coverage function_mocking + +**************************************************** +call_Test3 has failed. +**************************************************** + + +**************************************************** +call_Test4 has failed. +**************************************************** + + +**************************************************** +call_Test5 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] callOriginal_Test1 [pass] callOriginal_Test2 [pass] callOriginal_Test3 @@ -49,6 +67,9 @@ Running Tests with Coverage 0 skipped function_mocking.mock2 + + Test execution time :*****ms + [pass] test1 [pass] test2 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt index 53521f025670..69633db91d42 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt @@ -4,6 +4,14 @@ Compiling source Running Tests with Coverage function_mocking + +**************************************************** +testFunctionMock3 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunctionMock [pass] testFunctionMock2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMockDouble.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMockDouble.txt index a5f3db67f776..9ccb255ebca5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMockDouble.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMockDouble.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage object_mocking2 + + Test execution time :*****ms + [pass] testFn diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt index e2f049cca626..7a3607262fc6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt @@ -6,6 +6,44 @@ WARNING [main.bal:(47:5,47:47)] unused variable 'closeErr' Running Tests with Coverage object_mocking + +**************************************************** +testDefaultMockInvalidFieldName has failed. +**************************************************** + + +**************************************************** +testMockInvalidStream has failed. +**************************************************** + + +**************************************************** +testDefaultIncompatibleArgs has failed. +**************************************************** + + +**************************************************** +testDefaultInvalidMemberReturnValue has failed. +**************************************************** + + +**************************************************** +testDefaultMockInvalidReturnValue has failed. +**************************************************** + + +**************************************************** +testDefaultMockWrongAction has failed. +**************************************************** + + +**************************************************** +testDefaultTooManyArgs has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testDependentlyTypedFunctions_testDouble [pass] testDependentlyTypedFunctions_thenReturn [pass] testMockMemberVariable @@ -16,6 +54,24 @@ Running Tests with Coverage [pass] testProvideErrorReturnValue [pass] testUserDefinedMockObject + [fail] testDefaultMockInvalidFieldName: + + error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") + callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 + callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 + callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 + callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 7 + + + [fail] testMockInvalidStream: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 + callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 + callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 + + [fail] testDefaultIncompatibleArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") @@ -34,15 +90,6 @@ Running Tests with Coverage callableName: testDefaultInvalidMemberReturnValue$lambda5$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 - [fail] testDefaultMockInvalidFieldName: - - error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") - callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 - callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 - callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 - callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 7 - - [fail] testDefaultMockInvalidReturnValue: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") @@ -70,15 +117,6 @@ Running Tests with Coverage callableName: testDefaultTooManyArgs$lambda2$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 5 - [fail] testMockInvalidStream: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 - callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 - callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 - - 9 passing 7 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt index ae7206737dc9..47513eec2f5c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultIncompatibleArgs has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultIncompatibleArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt index f7e3c24cad61..30d67ded5139 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultInvalidMemberReturnValue has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultInvalidMemberReturnValue: error {ballerina/test:0}InvalidMemberFieldError ("return value provided does not match the type of 'url'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt index 28c56c38a078..06dbd63af55f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultMockInvalidFieldName has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultMockInvalidFieldName: error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt index a05f025d09c6..132e141ca51c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultMockInvalidReturnValue has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultMockInvalidReturnValue: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt index b2c6dfb7c833..f4663714fbfb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultMockWrongAction has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultMockWrongAction: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt index 105e8a794831..0f201695e411 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultTooManyArgs has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultTooManyArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("too many argument provided to mock the function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt index 715980f1179a..34cebaf70957 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testMockInvalidStream has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testMockInvalidStream: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt index 108e56c320cc..cbe0eff07abb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt @@ -5,6 +5,14 @@ Running Tests with Coverage non_public_field_mock +**************************************************** +testNonPublicMemberFieldMock has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testNonPublicMemberFieldMock: error("NonPublicMemberFieldError",message="member field should be public to be mocked. The provided field 'url' is not public") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionFlowTest-test-listener-shutdown.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionFlowTest-test-listener-shutdown.txt index 43796ff5625e..f1476a2823e8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionFlowTest-test-listener-shutdown.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionFlowTest-test-listener-shutdown.txt @@ -8,6 +8,9 @@ Calling init for 'moduleA listener' Calling init for 'current module listener' Calling start for 'moduleA listener' Calling start for 'current module listener' + + Test execution time :*****ms + [pass] main_test1 @@ -20,10 +23,13 @@ Calling stop for 'moduleA listener' moduleExecutionFlow.moduleA Calling init for 'moduleA listener' Calling start for 'moduleA listener' + + Test execution time :*****ms + [pass] test1 1 passing 0 failing 0 skipped -Calling stop for 'moduleA listener' +Calling stop for 'moduleA listener' \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_AllTests.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_AllTests.txt index fcfe501471ca..d3f5a3311430 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_AllTests.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_AllTests.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] commonTest [pass] main_test1 [pass] main_test2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt index 38d038876ae6..07753dc5ef30 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] main_test1 [pass] main_test2 [pass] main_test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt index 38d038876ae6..07753dc5ef30 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] main_test1 [pass] main_test2 [pass] main_test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_SingleTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_SingleTest.txt index 19e73d7656db..33a43bcc96ec 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_SingleTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_SingleTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] main_test1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt index 795c661a824c..f40b38f1fdc3 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] commonTest diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_AllTests.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_AllTests.txt index a75060a81c51..6b082fed9fb0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_AllTests.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_AllTests.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] commonTest_Module1 [pass] module1_test1 [pass] module1_test2 @@ -15,6 +18,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_SingleTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_SingleTest.txt index a817aaf192df..a10ba65191e7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_SingleTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_SingleTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] module1_test1 @@ -13,6 +16,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WildCardTest.txt index 1fb3cc592951..4f0f06a6fbfa 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WildCardTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] module1_test1 [pass] module1_test2 @@ -14,6 +17,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WithGroups.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WithGroups.txt index ccc33806b6cf..ef0706fcff75 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WithGroups.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_Module1_WithGroups.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] module1_test2 @@ -13,6 +16,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_WildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_WildCardTest.txt index 9922e268ca95..e22fb6c1e8d2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_WildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionTest-test_WildCardTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] commonTest_Module1 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped moduleExecution + + Test execution time :*****ms + [pass] commonTest diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/PathVerificationTest-verifyTestsOutsidePath.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/PathVerificationTest-verifyTestsOutsidePath.txt index b7c7e573e776..a63626ad9de2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/PathVerificationTest-verifyTestsOutsidePath.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/PathVerificationTest-verifyTestsOutsidePath.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage path_verification + + Test execution time :*****ms + [pass] testFunction [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt index 7ea2a79059e5..5f30cf1614dc 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt @@ -4,6 +4,19 @@ Compiling source Running Tests with Coverage rerun_failed + +**************************************************** +testFunctionFail1: has failed. +**************************************************** + + +**************************************************** +testFunctionFail2: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunctionPass1 [pass] testFunctionPass2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt index bb6a11cd55a9..0e0ce8f98171 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt @@ -5,6 +5,19 @@ Running Tests with Coverage rerun_failed +**************************************************** +testFunctionFail1: has failed. +**************************************************** + + +**************************************************** +testFunctionFail2: has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testFunctionFail1: error {ballerina/test:0}TestError ("Failed!") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDependentFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDependentFunctionExecution.txt index 747430d2d09d..603ae8d110a1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDependentFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDependentFunctionExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests single-test-execution.bal + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDisabledFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDisabledFunctionExecution.txt index 004643d4ee8e..0ddb09b90383 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDisabledFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testDisabledFunctionExecution.txt @@ -7,5 +7,8 @@ Running Tests single-test-execution.bal + Test execution time :*****ms + + No tests found \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testMultipleFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testMultipleFunctionExecution.txt index 747430d2d09d..603ae8d110a1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testMultipleFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testMultipleFunctionExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests single-test-execution.bal + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testNonExistingFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testNonExistingFunctionExecution.txt index 004643d4ee8e..0ddb09b90383 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testNonExistingFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testNonExistingFunctionExecution.txt @@ -7,5 +7,8 @@ Running Tests single-test-execution.bal + Test execution time :*****ms + + No tests found \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testSingleFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testSingleFunctionExecution.txt index 6326a4e521ce..e958ae022ad2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testSingleFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SelectedFunctionTest-testSingleFunctionExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests single-test-execution.bal + + Test execution time :*****ms + [pass] testFunc diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterEachFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterEachFails.txt index d41184afba2e..05339646698b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterEachFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterEachFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: afterEach fileName: skip-when-afterEach-fails.bal lineNumber: 30 callableName: afterEach$lambda2$ fileName: skip-when-afterEach-fails.bal lineNumber: 54 + + Test execution time :*****ms + [pass] test1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt index fe7abe96c29f..b2f2404243da 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: afterFunc fileName: skip-when-after-fails.bal lineNumber: 30 callableName: afterFunc$lambda6$ fileName: skip-when-after-fails.bal lineNumber: 35 + + Test execution time :*****ms + [fail] afterSuite[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt index 26929c0503d9..670e1a553bd9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt @@ -13,6 +13,9 @@ Running Tests callableName: beforeEach$lambda1$ fileName: skip-when-beforeEach-fails.bal lineNumber: 53 + Test execution time :*****ms + + 0 passing 0 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt index ac702e2a7322..99fec6d3e0b9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: before fileName: skip-when-before-fails.bal lineNumber: 28 callableName: before$lambda6$ fileName: skip-when-before-fails.bal lineNumber: 32 + + Test execution time :*****ms + [fail] afterSuite[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt index 409c2cfc4c2e..c34e39425793 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: beforeGroupsFunc2 fileName: skip-when-beforeGroups-fails.bal lineNumber: 32 callableName: beforeGroupsFunc2$lambda2$ fileName: skip-when-beforeGroups-fails.bal lineNumber: 85 + + Test execution time :*****ms + [fail] afterSuiteFunc[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt index 0bb1087c54f1..866854a8aca1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt @@ -13,6 +13,9 @@ Running Tests callableName: beforeSuite$lambda1$ fileName: skip-when-beforeSuite-fails.bal lineNumber: 75 + Test execution time :*****ms + + 0 passing 0 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt index ca080c047ee6..b3b80df4f3a1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt @@ -7,6 +7,14 @@ WARNING [dependson-skip-test.bal:(34:5,34:18)] unused variable 'i' Running Tests dependson-skip-test.bal + +**************************************************** +test2 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] test1 [pass] test5 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt index ebefe7d4fb43..80f265611ccb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage defaultModuleSource.module1 + + Test execution time :*****ms + [pass] test3 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped defaultModuleSource.module2 + + Test execution time :*****ms + [pass] test4 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt index d02639b8137f..55748121857d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage sourcelessModulesTest.module1 + + Test execution time :*****ms + [pass] test1 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped sourcelessModulesTest.module2 + + Test execution time :*****ms + [pass] test2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt index 00ceb60d3e50..e888c893e652 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage sourcelessProjectTest + + Test execution time :*****ms + [pass] test7 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped sourcelessProjectTest.module1 + + Test execution time :*****ms + [pass] test5 @@ -20,6 +26,9 @@ Running Tests with Coverage 0 skipped sourcelessProjectTest.module2 + + Test execution time :*****ms + [pass] test6 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt index 458c746edf60..b685ffecd479 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt @@ -6,6 +6,14 @@ WARNING [main.bal:(36:5,36:19)] unused variable 'b' Running Tests foo + +**************************************************** +testMain: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc [fail] testMain: @@ -23,6 +31,9 @@ Running Tests 1 skipped foo.bar.tests + + Test execution time :*****ms + [pass] testBarAdd @@ -31,6 +42,9 @@ Running Tests 0 skipped foo.math + + Test execution time :*****ms + [pass] testFunction1 [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt index 8b322bb73683..35cdd8aa10ae 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt @@ -5,6 +5,14 @@ WARNING [main.bal:(36:5,36:19)] unused variable 'b' Running Tests with Coverage foo + +**************************************************** +testMain: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc [fail] testMain: @@ -22,6 +30,9 @@ Running Tests with Coverage 1 skipped foo.bar.tests + + Test execution time :*****ms + [pass] testBarAdd @@ -30,6 +41,9 @@ Running Tests with Coverage 0 skipped foo.math + + Test execution time :*****ms + [pass] testFunction1 [pass] testFunction2 From f5c59ad275264b9100af78fee102c346ced15248 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 24 May 2023 17:39:22 +0530 Subject: [PATCH 07/37] Modify windows integration test outputs --- .../test/TestparallelizationTest.java | 7 +- .../BasicCasesTest-testAnnotations.txt | 17 +++-- .../BasicCasesTest-testAnydataType.txt | 3 + ...sicCasesTest-testAssertBehavioralTypes.txt | 3 + .../BasicCasesTest-testAssertDiffError.txt | 3 + ...BasicCasesTest-testAssertSequenceTypes.txt | 3 + ...sicCasesTest-testAssertStructuralTypes.txt | 3 + ...sicCasesTest-testAssertionErrorMessage.txt | 3 + .../windows/BasicCasesTest-testAssertions.txt | 3 + .../BasicCasesTest-testAsyncInvocation.txt | 3 + .../BasicCasesTest-testBeforeAfter.txt | 3 + ...BasicCasesTest-testBeforeEachAfterEach.txt | 3 + .../windows/BasicCasesTest-testDependsOn.txt | 3 + .../BasicCasesTest-testIntersectionTypes.txt | 3 + .../BasicCasesTest-testIsolatedFunctions.txt | 5 +- .../BasicCasesTest-testJavaInterops.txt | 3 + .../windows/BasicCasesTest-testRuntimeApi.txt | 3 + ...iderTest-testArrayDataProviderWithFail.txt | 16 ++++ ...viderTest-testArrayDataRerunFailedTest.txt | 16 ++++ ...DataProviderTest-testCodeFragmentKeys0.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys1.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys2.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys3.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys4.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys5.txt | 6 ++ ...DataProviderTest-testCodeFragmentKeys6.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard0.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard1.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard2.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard3.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard4.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard5.txt | 6 ++ ...Test-testCodeFragmentKeysWithWildCard6.txt | 6 ++ ...iderTest-testDataProviderSingleFailure.txt | 23 ++++-- ...iderTest-testDataProviderWithMixedType.txt | 6 ++ ...aProviderTest-testMapValueDataProvider.txt | 11 +++ ...iderTest-testMultiModuleSingleTestExec.txt | 6 ++ .../DataProviderTest-testRerunFailedTest.txt | 16 ++++ ...DataProviderTest-testValidDataProvider.txt | 6 ++ ...ProviderTest-testValidDataProviderCase.txt | 6 ++ ...-testValidDataProviderWithAfterFailing.txt | 6 ++ ...idDataProviderWithBeforeAfterFunctions.txt | 6 ++ ...testValidDataProviderWithBeforeFailing.txt | 6 ++ ...iderTest-testValidDataProviderWithFail.txt | 16 ++++ .../DataProviderTest-testWithSpecialKeys.txt | 6 ++ ...isableTestsTestCase-testDisablingTests.txt | 3 + ...lidationTest-validateFunctionNamesTest.txt | 3 + ...oupingTest-afterGroupsWithDisabledTest.txt | 3 + .../GroupingTest-beforeGroupsAfterGroups1.txt | 3 + .../GroupingTest-beforeGroupsAfterGroups2.txt | 29 +++++++- .../GroupingTest-failedBeforeEachTest.txt | 3 + .../GroupingTest-failedBeforeGroupTest.txt | 3 + ...roupingTest-testMultipleGroupExclusion.txt | 3 + ...roupingTest-testMultipleGroupExecution.txt | 3 + ...pingTest-testNonExistingGroupExclusion.txt | 13 ++++ ...pingTest-testNonExistingGroupInclusion.txt | 3 + .../GroupingTest-testSingleGroupExclusion.txt | 8 ++ .../GroupingTest-testSingleGroupExecution.txt | 3 + .../GroupingTest-testWhenAfterGroupsFails.txt | 3 + .../windows/ImportTest-testImportTest.txt | 3 + ...oviderTestCase-testInvalidDataProvider.txt | 30 ++++---- ...viderTestCase-testInvalidDataProvider2.txt | 30 ++++---- ...rTestCase-testInvalidTupleDataProvider.txt | 30 ++++---- ...stMockingFunctionWithIncompatibleTypes.txt | 8 ++ .../MockTest-testCoverageWithMocking.txt | 6 ++ ...kTest-testFuncMockInMultiModulesWDepen.txt | 6 ++ .../windows/MockTest-testFunctionMocking.txt | 21 ++++++ .../MockTest-testFunctionMockingLegacy.txt | 35 +++++---- ...ockTest-testFunctionMockingModuleLevel.txt | 21 ++++++ ...FunctionMockingThenReturnWithNilRetVal.txt | 8 ++ .../windows/MockTest-testObjectMockDouble.txt | 35 +++++---- .../windows/MockTest-testObjectMocking.txt | 74 ++++++++++++++----- ...kTest-testObjectMocking_NegativeCases1.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases2.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases3.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases4.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases5.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases6.txt | 8 ++ ...kTest-testObjectMocking_NegativeCases7.txt | 8 ++ ...kTest-testObjectMocking_NonPublicField.txt | 8 ++ ...ecutionFlowTest-test-listener-shutdown.txt | 64 ++++++++-------- ...cutionTest-test_DefaultModule_AllTests.txt | 6 ++ ...est-test_DefaultModule_EndWildCardTest.txt | 6 ++ ...-test_DefaultModule_MiddleWildCardTest.txt | 6 ++ ...tionTest-test_DefaultModule_SingleTest.txt | 6 ++ ...t-test_DefaultModule_StartWildCardTest.txt | 6 ++ ...uleExecutionTest-test_Module1_AllTests.txt | 6 ++ ...eExecutionTest-test_Module1_SingleTest.txt | 6 ++ ...xecutionTest-test_Module1_WildCardTest.txt | 6 ++ ...eExecutionTest-test_Module1_WithGroups.txt | 6 ++ .../ModuleExecutionTest-test_WildCardTest.txt | 6 ++ ...cationTest-verifyMissingTestsDirectory.txt | 4 +- ...erificationTest-verifyTestsOutsidePath.txt | 3 + .../windows/RerunFailedTest-testFullTest.txt | 13 ++++ .../RerunFailedTest-testRerunFailedTest.txt | 13 ++++ ...ionTest-testDependentFunctionExecution.txt | 3 + ...tionTest-testDisabledFunctionExecution.txt | 3 + ...tionTest-testMultipleFunctionExecution.txt | 3 + ...nTest-testNonExistingFunctionExecution.txt | 3 + ...nctionTest-testSingleFunctionExecution.txt | 3 + ...stsTestCase-testSkipWhenAfterEachFails.txt | 3 + ...ipTestsTestCase-testSkipWhenAfterFails.txt | 7 +- ...tsTestCase-testSkipWhenBeforeEachFails.txt | 3 + ...pTestsTestCase-testSkipWhenBeforeFails.txt | 7 +- ...TestCase-testSkipWhenBeforeGroupsFails.txt | 3 + ...sTestCase-testSkipWhenBeforeSuiteFails.txt | 3 + ...ase-testSkipWhenDependsOnFunctionFails.txt | 8 ++ ..._DefaultModuleSourceOnly_TestExecution.txt | 6 ++ ...ts-test_SourcelessModule_TestExecution.txt | 6 ++ ...s-test_SourcelessProject_TestExecution.txt | 9 +++ ...tTest-testWarningForCoverageFormatFlag.txt | 14 ++++ ...stReportTest-testWarningForReportTools.txt | 14 ++++ 112 files changed, 860 insertions(+), 150 deletions(-) diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index 8f79004c9090..6bfaebcb54b9 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -2,7 +2,6 @@ import org.ballerinalang.test.context.BMainInstance; import org.ballerinalang.test.context.BallerinaTestException; -import org.ballerinalang.testerina.test.utils.AssertionUtils; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -10,7 +9,9 @@ import java.io.IOException; import java.util.HashMap; -import static org.ballerinalang.testerina.test.utils.CommonUtils.replaceExecutionTime; +/** + * Test class to test the parallel execution of Ballerina tests. + */ public class TestparallelizationTest extends BaseTestCase { @@ -184,7 +185,7 @@ public void testNonParalallelizableMapDataProvider() throws BallerinaTestExcepti private float getTimeForTestExecution(String output) { int firstPos = output.indexOf("Test execution time :") + ("Test execution time :").length(); - int lastPos = output.indexOf("ms"); + int lastPos = output.indexOf("ms", firstPos); String executionTime = output.substring(firstPos, lastPos); return Float.parseFloat(executionTime); } diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt index 12be891a773c..5e50820efd00 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt @@ -4,20 +4,23 @@ Compiling source Running Tests with Coverage annotations + + Test execution time :*****ms + [pass] jsonDataProviderTest#0 - [pass] stringDataProviderTest#0 - [pass] stringDataProviderTest#1 - [pass] stringDataProviderTest#2 - [pass] stringDataProviderTest2#0 [pass] test1 - [pass] testAfter [pass] test2 - [pass] testAfterAlone [pass] test3 [pass] test4 [pass] test5 - [pass] testDependsOn1 [pass] testBefore + [pass] testAfter + [pass] testAfterAlone + [pass] testDependsOn1 + [pass] stringDataProviderTest#0 + [pass] stringDataProviderTest#1 + [pass] stringDataProviderTest#2 + [pass] stringDataProviderTest2#0 14 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnydataType.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnydataType.txt index d5e070afa935..2cdceed3a130 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnydataType.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnydataType.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage anydata_type + + Test execution time :*****ms + [pass] testFn [pass] testFoo diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertBehavioralTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertBehavioralTypes.txt index b4f338ece393..94b26bc33ac2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertBehavioralTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertBehavioralTypes.txt @@ -7,6 +7,9 @@ WARNING [tests\assert-test-behavioral-types.bal:(24:5,24:32)] undocumented field Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertObjectArrayEquals [pass] testAssertObjectArrayEqualsNegative [pass] testAssertObjectArrayNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertDiffError.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertDiffError.txt index f12629270eea..fcdb605c5a5f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertDiffError.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertDiffError.txt @@ -7,6 +7,9 @@ WARNING [tests\assertions-diff-error-messages.bal:(24:5,24:32)] undocumented fie Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertDecimalValues [pass] testAssertIntValues [pass] testAssertJsonArray diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertSequenceTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertSequenceTypes.txt index 520d039ce333..e6de54be39cf 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertSequenceTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertSequenceTypes.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertStringEquals [pass] testAssertStringEqualsNegative [pass] testAssertStringNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertStructuralTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertStructuralTypes.txt index c9a78dddde64..126797d7cdff 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertStructuralTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertStructuralTypes.txt @@ -7,6 +7,9 @@ WARNING [tests\assert-test-structural-types.bal:(24:5,24:18)] undocumented field Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertByteArrayEquals [pass] testAssertByteArrayEqualsNegative [pass] testAssertByteArrayNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertionErrorMessage.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertionErrorMessage.txt index 022809ee2858..30c837278d7b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertionErrorMessage.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertionErrorMessage.txt @@ -7,6 +7,9 @@ WARNING [tests\assertions-error-messages.bal:(24:5,24:32)] undocumented field 'p Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertAnnonymousRecords [pass] testAssertDecimalAndFloat [pass] testAssertDifferentObjects diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertions.txt index a7fd833158c5..e35de1b9c400 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAssertions.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage assertions + + Test execution time :*****ms + [pass] testAssertBooleanEquals [pass] testAssertBooleanEqualsNegative [pass] testAssertBooleanNotEquals diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAsyncInvocation.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAsyncInvocation.txt index 600acd0b5097..3b8b30a804d4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAsyncInvocation.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAsyncInvocation.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage async + + Test execution time :*****ms + [pass] testAsyncRemoteFunctionMain [pass] testAsyncRemoteFunctionTest diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeAfter.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeAfter.txt index d3b9aeb92727..4b7a8ccf30d7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeAfter.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeAfter.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage before_after + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeEachAfterEach.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeEachAfterEach.txt index 6b9060d0040b..af0ac193b699 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeEachAfterEach.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testBeforeEachAfterEach.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage before_each_after_each + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt index 0be84f1e6a28..6fdecbfee824 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage depends_on + + Test execution time :*****ms + [pass] test1 [pass] test2 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIntersectionTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIntersectionTypes.txt index 524f8be388dc..3e8427acf943 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIntersectionTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIntersectionTypes.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage intersection_type + + Test execution time :*****ms + [pass] testIntersectionTypes diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt index 57e855e9ae18..542072415010 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt @@ -4,8 +4,11 @@ Compiling source Running Tests with Coverage isolated_functions - [pass] testCallingIsolatedFunction + + Test execution time :*****ms + [pass] testFunc + [pass] testCallingIsolatedFunction [pass] testIsolatedTestFunction diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testJavaInterops.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testJavaInterops.txt index cebb482b0dc0..cd7f269e0818 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testJavaInterops.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testJavaInterops.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage interops + + Test execution time :*****ms + [pass] testInteropWithRestArgs diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testRuntimeApi.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testRuntimeApi.txt index 68b8e99d2390..fe8c72deaf13 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testRuntimeApi.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testRuntimeApi.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage runtime_api + + Test execution time :*****ms + [pass] testRuntimeApi diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt index 26d372dff248..de1ffa407c52 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt @@ -6,6 +6,19 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +intArrayDataProviderTest:0 has failed. +**************************************************** + + +**************************************************** +intArrayDataProviderTest:1 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] intArrayDataProviderTest#2 [fail] intArrayDataProviderTest#0: @@ -39,6 +52,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt index 670f3ae3e018..31c3862b0acf 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt @@ -7,6 +7,19 @@ Running Tests with Coverage dataproviders +**************************************************** +intArrayDataProviderTest:0 has failed. +**************************************************** + + +**************************************************** +intArrayDataProviderTest:1 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] intArrayDataProviderTest#0: error {ballerina/test:0}TestError ("The sum is not correct @@ -38,6 +51,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys0.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys0.txt index 5ab924174ab8..6f8651238e31 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys0.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys0.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#`'"\a"" @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys1.txt index 1f889fe58687..60e0d69cfaba 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys1.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#"\u{D7FF}"" " @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys2.txt index 0584dab6f042..54a5cd7903e8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys2.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a + b @@ -17,6 +20,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys3.txt index 0d98ffbf5f98..ffb5a94c4084 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys3.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(x * 1) != (y / 3) || (a ^ b) == (b & c) >> (1 % 2) @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys4.txt index c50f70edad0a..6d463567502d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys4.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(1 @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys5.txt index a7684ebe2093..0a3332f49d67 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys5.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a:x(c,d)[]; ^(x|y).ok(); @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys6.txt index cc1a37a1ad1e..120b17a81a66 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeys6.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#map v = { "x": 1 }; @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt index 5ab924174ab8..6f8651238e31 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard0.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#`'"\a"" @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt index 1f889fe58687..60e0d69cfaba 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard1.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#"\u{D7FF}"" " @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt index 0584dab6f042..54a5cd7903e8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard2.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a + b @@ -17,6 +20,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt index 0d98ffbf5f98..ffb5a94c4084 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard3.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(x * 1) != (y / 3) || (a ^ b) == (b & c) >> (1 % 2) @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt index c50f70edad0a..6d463567502d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard4.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#(1 @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt index a7684ebe2093..0a3332f49d67 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard5.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#a:x(c,d)[]; ^(x|y).ok(); @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt index cc1a37a1ad1e..120b17a81a66 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testCodeFragmentKeysWithWildCard6.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction3#map v = { "x": 1 }; @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt index 6648dd1f662b..41b9ee129071 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt @@ -6,6 +6,14 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +testDividingValuesNegative:2 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testDividingValuesNegative#0 [pass] testDividingValuesNegative#1 [pass] testDividingValuesNegative#3 @@ -19,13 +27,9 @@ Running Tests with Coverage callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 17 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 26 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 @@ -35,10 +39,13 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found Generating Test Report data-providers\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderWithMixedType.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderWithMixedType.txt index 4cba42d8ebd5..11170c2436ff 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderWithMixedType.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderWithMixedType.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction1#CaseNew1 [pass] testFunction1#CaseNew2 @@ -16,6 +19,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt index 11783eb40c34..b41ea7d6be37 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt @@ -6,6 +6,14 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +testGetState:1 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testGetState#0 [fail] testGetState#1: @@ -27,6 +35,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMultiModuleSingleTestExec.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMultiModuleSingleTestExec.txt index d124d5f2ae43..8956b08c2f63 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMultiModuleSingleTestExec.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMultiModuleSingleTestExec.txt @@ -7,10 +7,16 @@ Running Tests with Coverage dataproviders + Test execution time :*****ms + + No tests found dataproviders.module1 + + Test execution time :*****ms + [pass] stringDataProviderMod1Test#1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt index 1d29fa076a1a..ef5c275fc4dc 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt @@ -7,6 +7,19 @@ Running Tests with Coverage dataproviders +**************************************************** +intDataProviderTest:Case1 has failed. +**************************************************** + + +**************************************************** +intDataProviderTest:Case2 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] intDataProviderTest#Case1: error {ballerina/test:0}TestError ("The sum is not correct @@ -38,6 +51,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProvider.txt index 050d1546f95d..d5fe21c129c6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProvider.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] jsonDataProviderTest#json1 [pass] jsonDataProviderTest#json2 @@ -16,6 +19,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderCase.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderCase.txt index 41cf260ec3db..24a5a066f383 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderCase.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderCase.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] jsonDataProviderTest#json1 @@ -15,6 +18,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt index 21ad12981bd9..bc34af9cf449 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt @@ -11,6 +11,9 @@ Running Tests with Coverage callableName: afterFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 153 callableName: afterFailsFunction$lambda37$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 + + Test execution time :*****ms + [pass] testDividingValuesWithAfterFailing#0 [pass] testDividingValuesWithAfterFailing#1 [pass] testDividingValuesWithAfterFailing#2 @@ -25,6 +28,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt index 98511368828a..90ed5e32ab0a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeAfterFunctions.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testDividingValues#0 [pass] testDividingValues#1 [pass] testDividingValues#2 @@ -20,6 +23,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt index f655c3414b4e..8cce39612e86 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt @@ -11,6 +11,9 @@ Running Tests with Coverage callableName: beforeFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 121 callableName: beforeFailsFunction$lambda33$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 + + Test execution time :*****ms + [pass] testDividingValuesWithBeforeFailing#0 [pass] testDividingValuesWithBeforeFailing#1 [pass] testDividingValuesWithBeforeFailing#3 @@ -24,6 +27,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt index 51a815d94c6a..090280a726b7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt @@ -6,6 +6,19 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +**************************************************** +intDataProviderTest:Case1 has failed. +**************************************************** + + +**************************************************** +intDataProviderTest:Case2 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] intDataProviderTest#Case3 [fail] intDataProviderTest#Case1: @@ -39,6 +52,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testWithSpecialKeys.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testWithSpecialKeys.txt index f641bd0e5eb3..02b6b8de7b95 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testWithSpecialKeys.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testWithSpecialKeys.txt @@ -6,6 +6,9 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + + Test execution time :*****ms + [pass] testFunction2#Case#1 [pass] testFunction2#Case#2 [pass] testFunction2#Case#3 @@ -17,6 +20,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DisableTestsTestCase-testDisablingTests.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DisableTestsTestCase-testDisablingTests.txt index b73bbf74eb52..6a667ec2b2f4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DisableTestsTestCase-testDisablingTests.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DisableTestsTestCase-testDisablingTests.txt @@ -6,6 +6,9 @@ Compiling source Running Tests disable-test.bal + + Test execution time :*****ms + [pass] testDisableFunc2 [pass] testDisableFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/FunctionNameValidationTest-validateFunctionNamesTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/FunctionNameValidationTest-validateFunctionNamesTest.txt index 20625dcbe7fe..f9868e402aad 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/FunctionNameValidationTest-validateFunctionNamesTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/FunctionNameValidationTest-validateFunctionNamesTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage validate_function_names + + Test execution time :*****ms + [pass] validateFunctionName diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-afterGroupsWithDisabledTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-afterGroupsWithDisabledTest.txt index ec00ccd0b89f..e0beb648ad75 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-afterGroupsWithDisabledTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-afterGroupsWithDisabledTest.txt @@ -6,6 +6,9 @@ Compiling source Running Tests after-groups-with-disabled-test.bal + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups1.txt index fc7173295607..f799c87a3808 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups1.txt @@ -6,6 +6,9 @@ Compiling source Running Tests before-groups-after-groups-test.bal + + Test execution time :*****ms + [pass] testFunction [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt index 913965967b11..f8ab35f6c57b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt @@ -6,13 +6,38 @@ Compiling source Running Tests before-groups-after-groups-test2.bal + + Test execution time :*****ms + + [fail] afterSuiteFunc[after test suite function]: + error {ballerina/test:0}TestError ("Assertion Failed! + + expected: '123456787' + actual : '123467587' + + Diff : + + --- actual + +++ expected + + @@ -1,1 +1,1 @@ + + -123467587 + +123456787 + ") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 + callableName: afterSuiteFunc fileName: before-groups-after-groups-test2.bal lineNumber: 80 + callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 91 + [pass] testFunction [pass] testFunction2 - [pass] testFunction3 [pass] testFunction4 + [pass] testFunction3 [pass] testFunction5 5 passing 0 failing - 0 skipped \ No newline at end of file + 0 skipped +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt index 206b16545759..5b32eba143d4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt @@ -14,6 +14,9 @@ Running Tests callableName: beforeEachFunction$lambda4$ fileName: failed-before-each-with-groups.bal lineNumber: 85 + Test execution time :*****ms + + 0 passing 0 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt index 40b2d4a16f39..ba6b410bfadd 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt @@ -11,6 +11,9 @@ Running Tests callableName: beforeGroupsFunc1 fileName: failed-before-groups-test.bal lineNumber: 28 callableName: beforeGroupsFunc1$lambda1$ fileName: failed-before-groups-test.bal lineNumber: 95 + + Test execution time :*****ms + [pass] testFunction [pass] testFunction4 [pass] testFunction5 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExclusion.txt index 8d2b8ce6757b..f5a8b44bc6d2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExclusion.txt @@ -6,6 +6,9 @@ Compiling source Running Tests groups-test.bal + + Test execution time :*****ms + [pass] testFunc6 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExecution.txt index f1099c274d84..064851e29288 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testMultipleGroupExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests groups-test.bal + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt index 02563d49b53c..73e502cbe060 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt @@ -6,6 +6,19 @@ Compiling source Running Tests groups-test.bal + +**************************************************** +testFunc4: has failed. +**************************************************** + + +**************************************************** +testFunc5: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupInclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupInclusion.txt index 0802625a37c3..4dbe35dc12d1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupInclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupInclusion.txt @@ -7,5 +7,8 @@ Running Tests groups-test.bal + Test execution time :*****ms + + No tests found \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt index 8033347a8fd6..0f83bceb0c9f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt @@ -6,6 +6,14 @@ Compiling source Running Tests groups-test.bal + +**************************************************** +testFunc5: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExecution.txt index f1099c274d84..064851e29288 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests groups-test.bal + + Test execution time :*****ms + [pass] testFunc1 [pass] testFunc2 [pass] testFunc3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt index d460f073ce93..05edf57e97bd 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: afterGroupsFunc1 fileName: failed-after-groups-test.bal lineNumber: 37 callableName: afterGroupsFunc1$lambda3$ fileName: failed-after-groups-test.bal lineNumber: 79 + + Test execution time :*****ms + [pass] testFunction [pass] testFunction2 [pass] testFunction3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ImportTest-testImportTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ImportTest-testImportTest.txt index 3f7a245a8f99..0dc17d06f5f0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ImportTest-testImportTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ImportTest-testImportTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage pre_declared_imports + + Test execution time :*****ms + [pass] testImportTest1 [pass] testImportTest2 [pass] testImportTest3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt index 60ad68ac14ce..3e9799dca306 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -8,30 +8,30 @@ Running Tests invalid-data-provider-test.bal +**************************************************** +testInvalidDataProvider:0 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testInvalidDataProvider#0: [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 343 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index b4d7cacde009..8ac98e6f45c5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -10,30 +10,30 @@ Running Tests invalid-data-provider-test2.bal +**************************************************** +testInvalidDataProvider2:0 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testInvalidDataProvider2#0: [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 343 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 38 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 38 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 756738d859b5..3a3054818661 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -7,30 +7,30 @@ Running Tests invalid-data-provider-test3.bal +**************************************************** +testInvalidTupleDataProvider:0 has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testInvalidTupleDataProvider#0: [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 343 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 35 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 353 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 140 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 127 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 87 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 62 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 46 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 35 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt index f100f93f7345..5822119a1d23 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt @@ -5,6 +5,14 @@ Running Tests incompatible_type_mock +**************************************************** +functionMockingTest has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] functionMockingTest: error {ballerina/test:0}FunctionSignatureMismatchError ("Return type of function stringHello does not match function intAdd") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testCoverageWithMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testCoverageWithMocking.txt index db33a6fe8160..250d29b6bef1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testCoverageWithMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testCoverageWithMocking.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage mocking_coverage.mod2 + + Test execution time :*****ms + [pass] testFloatAdd [pass] testStringAdd @@ -13,6 +16,9 @@ Running Tests with Coverage 0 skipped mocking_coverage + + Test execution time :*****ms + [pass] testFloatAdd [pass] testStringAdd diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFuncMockInMultiModulesWDepen.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFuncMockInMultiModulesWDepen.txt index 1953b54b2dff..49605dcb9368 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFuncMockInMultiModulesWDepen.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFuncMockInMultiModulesWDepen.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage function_mocking_with_dependencies.a_dependency + + Test execution time :*****ms + [pass] test1 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped function_mocking_with_dependencies.b_dependent + + Test execution time :*****ms + [pass] test1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt index 203ceb5918d0..5f53255fe360 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt @@ -4,6 +4,24 @@ Compiling source Running Tests with Coverage function_mocking + +**************************************************** +call_Test3 has failed. +**************************************************** + + +**************************************************** +call_Test4 has failed. +**************************************************** + + +**************************************************** +call_Test5 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] callOriginal_Test1 [pass] callOriginal_Test2 [pass] callOriginal_Test3 @@ -49,6 +67,9 @@ Running Tests with Coverage 0 skipped function_mocking.mock2 + + Test execution time :*****ms + [pass] test1 [pass] test2 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt index 814ae209234f..2b4f2ff65cc9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt @@ -1,16 +1,19 @@ -Compiling source - intg_tests/function_mocking_legacy:0.1.0 - -Running Tests with Coverage - - function_mocking_legacy - [pass] testIntAdd - - - 1 passing - 0 failing - 0 skipped - -Generating Test Report - legacy-function-mocking-tests\target\report\test_results.json - +Compiling source + intg_tests/function_mocking_legacy:0.1.0 + +Running Tests with Coverage + + function_mocking_legacy + + Test execution time :*****ms + + [pass] testIntAdd + + + 1 passing + 0 failing + 0 skipped + +Generating Test Report + legacy-function-mocking-tests\target\report\test_results.json + \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt index 203ceb5918d0..5f53255fe360 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt @@ -4,6 +4,24 @@ Compiling source Running Tests with Coverage function_mocking + +**************************************************** +call_Test3 has failed. +**************************************************** + + +**************************************************** +call_Test4 has failed. +**************************************************** + + +**************************************************** +call_Test5 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] callOriginal_Test1 [pass] callOriginal_Test2 [pass] callOriginal_Test3 @@ -49,6 +67,9 @@ Running Tests with Coverage 0 skipped function_mocking.mock2 + + Test execution time :*****ms + [pass] test1 [pass] test2 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt index d926f246c615..3ed2a7c13827 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt @@ -4,6 +4,14 @@ Compiling source Running Tests with Coverage function_mocking + +**************************************************** +testFunctionMock3 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunctionMock [pass] testFunctionMock2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMockDouble.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMockDouble.txt index 8ae7fd6208c1..e5a6ea4a3f8e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMockDouble.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMockDouble.txt @@ -1,16 +1,19 @@ -Compiling source - intg_tests/object_mocking2:0.0.0 - -Running Tests with Coverage - - object_mocking2 - [pass] testFn - - - 1 passing - 0 failing - 0 skipped - -Generating Test Report - object-mocking-tests2\target\report\test_results.json - +Compiling source + intg_tests/object_mocking2:0.0.0 + +Running Tests with Coverage + + object_mocking2 + + Test execution time :*****ms + + [pass] testFn + + + 1 passing + 0 failing + 0 skipped + +Generating Test Report + object-mocking-tests2\target\report\test_results.json + \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt index 9d50365ab6bf..e6c4e27e4484 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt @@ -6,6 +6,44 @@ WARNING [main.bal:(47:5,47:47)] unused variable 'closeErr' Running Tests with Coverage object_mocking + +**************************************************** +testDefaultMockInvalidFieldName has failed. +**************************************************** + + +**************************************************** +testMockInvalidStream has failed. +**************************************************** + + +**************************************************** +testDefaultIncompatibleArgs has failed. +**************************************************** + + +**************************************************** +testDefaultInvalidMemberReturnValue has failed. +**************************************************** + + +**************************************************** +testDefaultMockInvalidReturnValue has failed. +**************************************************** + + +**************************************************** +testDefaultMockWrongAction has failed. +**************************************************** + + +**************************************************** +testDefaultTooManyArgs has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testDependentlyTypedFunctions_testDouble [pass] testDependentlyTypedFunctions_thenReturn [pass] testMockMemberVariable @@ -16,6 +54,24 @@ Running Tests with Coverage [pass] testProvideErrorReturnValue [pass] testUserDefinedMockObject + [fail] testDefaultMockInvalidFieldName: + + error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") + callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 + callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 + callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 + callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 7 + + + [fail] testMockInvalidStream: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 + callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 + callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 + + [fail] testDefaultIncompatibleArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") @@ -34,15 +90,6 @@ Running Tests with Coverage callableName: testDefaultInvalidMemberReturnValue$lambda5$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 - [fail] testDefaultMockInvalidFieldName: - - error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") - callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 - callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 - callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 - callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 7 - - [fail] testDefaultMockInvalidReturnValue: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") @@ -70,15 +117,6 @@ Running Tests with Coverage callableName: testDefaultTooManyArgs$lambda2$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 5 - [fail] testMockInvalidStream: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 - callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 - callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 - - 9 passing 7 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt index 3e5880c23af2..87e272a2a49e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultIncompatibleArgs has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultIncompatibleArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt index 0e4b696d6ce5..6e325db6849c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultInvalidMemberReturnValue has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultInvalidMemberReturnValue: error {ballerina/test:0}InvalidMemberFieldError ("return value provided does not match the type of 'url'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt index e66503b42849..61c656198a94 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultMockInvalidFieldName has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultMockInvalidFieldName: error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt index a47f259dc490..ba591fceb7fe 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultMockInvalidReturnValue has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultMockInvalidReturnValue: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt index 361aa64260ac..8a90d37888a2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultMockWrongAction has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultMockWrongAction: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt index 4145d4ac9f53..9bfd9b95c3ee 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testDefaultTooManyArgs has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testDefaultTooManyArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("too many argument provided to mock the function 'get()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt index faef9c5c75d1..7eb5c6abefd6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt @@ -7,6 +7,14 @@ Running Tests with Coverage object_mocking +**************************************************** +testMockInvalidStream has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testMockInvalidStream: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt index 74482cc5d556..cdf5260c2476 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt @@ -5,6 +5,14 @@ Running Tests with Coverage non_public_field_mock +**************************************************** +testNonPublicMemberFieldMock has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testNonPublicMemberFieldMock: error("NonPublicMemberFieldError",message="member field should be public to be mocked. The provided field 'url' is not public") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt index 43796ff5625e..b61511a5b41c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt @@ -1,29 +1,35 @@ -Compiling source - wso2/moduleExecutionFlow:0.0.0 - -Running Tests - - moduleExecutionFlow -Calling init for 'moduleA listener' -Calling init for 'current module listener' -Calling start for 'moduleA listener' -Calling start for 'current module listener' - [pass] main_test1 - - - 1 passing - 0 failing - 0 skipped -Calling stop for 'current module listener' -Calling stop for 'moduleA listener' - - moduleExecutionFlow.moduleA -Calling init for 'moduleA listener' -Calling start for 'moduleA listener' - [pass] test1 - - - 1 passing - 0 failing - 0 skipped -Calling stop for 'moduleA listener' +Compiling source + wso2/moduleExecutionFlow:0.0.0 + +Running Tests + + moduleExecutionFlow +Calling init for 'moduleA listener' +Calling init for 'current module listener' +Calling start for 'moduleA listener' +Calling start for 'current module listener' + + Test execution time :*****ms + + [pass] main_test1 + + + 1 passing + 0 failing + 0 skipped +Calling stop for 'current module listener' +Calling stop for 'moduleA listener' + + moduleExecutionFlow.moduleA +Calling init for 'moduleA listener' +Calling start for 'moduleA listener' + + Test execution time :*****ms + + [pass] test1 + + + 1 passing + 0 failing + 0 skipped +Calling stop for 'moduleA listener' \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_AllTests.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_AllTests.txt index 15c2a081f427..1eab01cb8b25 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_AllTests.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_AllTests.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] commonTest [pass] main_test1 [pass] main_test2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt index 884fcfcf1d84..3384fffea4c0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_EndWildCardTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] main_test1 [pass] main_test2 [pass] main_test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt index 884fcfcf1d84..3384fffea4c0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_MiddleWildCardTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] main_test1 [pass] main_test2 [pass] main_test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_SingleTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_SingleTest.txt index 5f40d869bba8..5f0bb38075e0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_SingleTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_SingleTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] main_test1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt index 4bd66fd24c74..086050fd2f93 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_DefaultModule_StartWildCardTest.txt @@ -5,10 +5,16 @@ Running Tests with Coverage moduleExecution.Module1 + Test execution time :*****ms + + No tests found moduleExecution + + Test execution time :*****ms + [pass] commonTest diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_AllTests.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_AllTests.txt index 99680375ccc3..7e1b0682791b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_AllTests.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_AllTests.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] commonTest_Module1 [pass] module1_test1 [pass] module1_test2 @@ -15,6 +18,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_SingleTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_SingleTest.txt index e3b2822f0a62..271299c71d4c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_SingleTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_SingleTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] module1_test1 @@ -13,6 +16,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WildCardTest.txt index f68fec87add1..f6f43a526ff0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WildCardTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] module1_test1 [pass] module1_test2 @@ -14,6 +17,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WithGroups.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WithGroups.txt index 90b49c3f933a..805a6fab92c7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WithGroups.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_Module1_WithGroups.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] module1_test2 @@ -13,6 +16,9 @@ Running Tests with Coverage moduleExecution + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_WildCardTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_WildCardTest.txt index c45128679656..34856beee756 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_WildCardTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionTest-test_WildCardTest.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage moduleExecution.Module1 + + Test execution time :*****ms + [pass] commonTest_Module1 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped moduleExecution + + Test execution time :*****ms + [pass] commonTest diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyMissingTestsDirectory.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyMissingTestsDirectory.txt index 2c951a872358..431b63d5a448 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyMissingTestsDirectory.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyMissingTestsDirectory.txt @@ -1 +1,3 @@ -Compiling source intg_tests/missing_tests_dir:0.0.0 No tests found \ No newline at end of file +Compiling source + intg_tests/missing_tests_dir:0.0.0 + No tests found \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyTestsOutsidePath.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyTestsOutsidePath.txt index 1c0ac4b57c6f..fa4872558181 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyTestsOutsidePath.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/PathVerificationTest-verifyTestsOutsidePath.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage path_verification + + Test execution time :*****ms + [pass] testFunction [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt index a5f70b79f76b..b36a7836a4b4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt @@ -4,6 +4,19 @@ Compiling source Running Tests with Coverage rerun_failed + +**************************************************** +testFunctionFail1: has failed. +**************************************************** + + +**************************************************** +testFunctionFail2: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunctionPass1 [pass] testFunctionPass2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt index 18e3e4a677e0..58cd5e6bdcf6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt @@ -5,6 +5,19 @@ Running Tests with Coverage rerun_failed +**************************************************** +testFunctionFail1: has failed. +**************************************************** + + +**************************************************** +testFunctionFail2: has failed. +**************************************************** + + + Test execution time :*****ms + + [fail] testFunctionFail1: error {ballerina/test:0}TestError ("Failed!") diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDependentFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDependentFunctionExecution.txt index c89d17ea9b9a..dcbd24715486 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDependentFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDependentFunctionExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests single-test-execution.bal + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDisabledFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDisabledFunctionExecution.txt index 87854717a900..2de4a92543bb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDisabledFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testDisabledFunctionExecution.txt @@ -7,5 +7,8 @@ Running Tests single-test-execution.bal + Test execution time :*****ms + + No tests found \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testMultipleFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testMultipleFunctionExecution.txt index c89d17ea9b9a..dcbd24715486 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testMultipleFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testMultipleFunctionExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests single-test-execution.bal + + Test execution time :*****ms + [pass] testFunc [pass] testFunc2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testNonExistingFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testNonExistingFunctionExecution.txt index 87854717a900..2de4a92543bb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testNonExistingFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testNonExistingFunctionExecution.txt @@ -7,5 +7,8 @@ Running Tests single-test-execution.bal + Test execution time :*****ms + + No tests found \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testSingleFunctionExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testSingleFunctionExecution.txt index ff9dd2d99739..d9c90a241c48 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testSingleFunctionExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SelectedFunctionTest-testSingleFunctionExecution.txt @@ -6,6 +6,9 @@ Compiling source Running Tests single-test-execution.bal + + Test execution time :*****ms + [pass] testFunc diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt index e317c04d2033..71330369fd71 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: afterEach fileName: skip-when-afterEach-fails.bal lineNumber: 30 callableName: afterEach$lambda2$ fileName: skip-when-afterEach-fails.bal lineNumber: 54 + + Test execution time :*****ms + [pass] test1 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt index 09e49bdb3359..5713e0c31136 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -12,11 +12,14 @@ Running Tests callableName: afterFunc fileName: skip-when-after-fails.bal lineNumber: 30 callableName: afterFunc$lambda6$ fileName: skip-when-after-fails.bal lineNumber: 35 + + Test execution time :*****ms + [fail] afterSuite[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! expected: 'beforetestafterEachtestafterEach' - actual : 'beforetestafterEachafterEachtestafterEach' + actual : 'beforetestafterEachtestafterEachafterEach' Diff : @@ -25,7 +28,7 @@ Running Tests @@ -1,1 +1,1 @@ - -beforetestafterEachafterEachtestafterEach + -beforetestafterEachtestafterEachafterEach +beforetestafterEachtestafterEach ") callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt index e1599365b3c1..2ccebae7c9b7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt @@ -13,6 +13,9 @@ Running Tests callableName: beforeEach$lambda1$ fileName: skip-when-beforeEach-fails.bal lineNumber: 53 + Test execution time :*****ms + + 0 passing 0 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt index 157b6a68414e..a6826274fa2e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -12,11 +12,14 @@ Running Tests callableName: before fileName: skip-when-before-fails.bal lineNumber: 28 callableName: before$lambda6$ fileName: skip-when-before-fails.bal lineNumber: 32 + + Test execution time :*****ms + [fail] afterSuite[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! expected: 'beforetest3afterEach' - actual : 'beforeafterEachafterEachtest3afterEach' + actual : 'beforeafterEachtest3afterEachafterEach' Diff : @@ -25,7 +28,7 @@ Running Tests @@ -1,1 +1,1 @@ - -beforeafterEachafterEachtest3afterEach + -beforeafterEachtest3afterEachafterEach +beforetest3afterEach ") callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt index c08982e6fecc..4056a8f27753 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt @@ -12,6 +12,9 @@ Running Tests callableName: beforeGroupsFunc2 fileName: skip-when-beforeGroups-fails.bal lineNumber: 32 callableName: beforeGroupsFunc2$lambda2$ fileName: skip-when-beforeGroups-fails.bal lineNumber: 85 + + Test execution time :*****ms + [fail] afterSuiteFunc[after test suite function]: error {ballerina/test:0}TestError ("Assertion Failed! diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt index 6c44d59d706c..6d057aaa5dec 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt @@ -13,6 +13,9 @@ Running Tests callableName: beforeSuite$lambda1$ fileName: skip-when-beforeSuite-fails.bal lineNumber: 75 + Test execution time :*****ms + + 0 passing 0 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt index ec5b1cbe0cf3..c9450e41180a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt @@ -7,6 +7,14 @@ WARNING [dependson-skip-test.bal:(34:5,34:18)] unused variable 'i' Running Tests dependson-skip-test.bal + +**************************************************** +test2 has failed. +**************************************************** + + + Test execution time :*****ms + [pass] test1 [pass] test5 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt index f89b9051d724..8324fc0637bc 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_DefaultModuleSourceOnly_TestExecution.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage defaultModuleSource.module1 + + Test execution time :*****ms + [pass] test3 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped defaultModuleSource.module2 + + Test execution time :*****ms + [pass] test4 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt index b80264eb6c3f..3db8dae7f160 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessModule_TestExecution.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage sourcelessModulesTest.module1 + + Test execution time :*****ms + [pass] test1 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped sourcelessModulesTest.module2 + + Test execution time :*****ms + [pass] test2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt index c9f902aab511..37d486749e44 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SourcelessTestExecutionTests-test_SourcelessProject_TestExecution.txt @@ -4,6 +4,9 @@ Compiling source Running Tests with Coverage sourcelessProjectTest + + Test execution time :*****ms + [pass] test7 @@ -12,6 +15,9 @@ Running Tests with Coverage 0 skipped sourcelessProjectTest.module1 + + Test execution time :*****ms + [pass] test5 @@ -20,6 +26,9 @@ Running Tests with Coverage 0 skipped sourcelessProjectTest.module2 + + Test execution time :*****ms + [pass] test6 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt index b999edd68ca0..ccdab90581f5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt @@ -6,6 +6,14 @@ WARNING [main.bal:(36:5,36:19)] unused variable 'b' Running Tests foo + +**************************************************** +testMain: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc [fail] testMain: @@ -23,6 +31,9 @@ Running Tests 1 skipped foo.bar.tests + + Test execution time :*****ms + [pass] testBarAdd @@ -31,6 +42,9 @@ Running Tests 0 skipped foo.math + + Test execution time :*****ms + [pass] testFunction1 [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt index 435763fa5b12..09b1b2b772bc 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt @@ -5,6 +5,14 @@ WARNING [main.bal:(36:5,36:19)] unused variable 'b' Running Tests with Coverage foo + +**************************************************** +testMain: has failed. +**************************************************** + + + Test execution time :*****ms + [pass] testFunc [fail] testMain: @@ -22,6 +30,9 @@ Running Tests with Coverage 1 skipped foo.bar.tests + + Test execution time :*****ms + [pass] testBarAdd @@ -30,6 +41,9 @@ Running Tests with Coverage 0 skipped foo.math + + Test execution time :*****ms + [pass] testFunction1 [pass] testFunction2 From c78a5e9365e1759e5a04664205020402652ffeb7 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 24 May 2023 17:56:23 +0530 Subject: [PATCH 08/37] Fix checkstyle failure --- .../testerina/natives/CommonUtils.java | 28 +++++++++++++++---- .../test/TestparallelizationTest.java | 17 +++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java index 600450995eed..4c7064c9c3cb 100644 --- a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java +++ b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java @@ -1,3 +1,20 @@ +// Copyright (c) 2023 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. +// +// WSO2 Inc. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + package org.ballerinalang.testerina.natives; import io.ballerina.runtime.api.types.FunctionType; @@ -9,12 +26,13 @@ import io.ballerina.runtime.api.values.BFunctionPointer; import io.ballerina.runtime.internal.util.exceptions.BLangExceptionHelper; import io.ballerina.runtime.internal.util.exceptions.RuntimeErrors; -import org.wso2.ballerinalang.compiler.semantics.model.symbols.Symbols; -import org.wso2.ballerinalang.compiler.semantics.model.types.BType; -import org.wso2.ballerinalang.compiler.semantics.model.types.BUnionType; -import org.wso2.ballerinalang.compiler.util.TypeTags; -import org.wso2.ballerinalang.util.Flags; + +/** + * Common utility functions for the Testerina module. + * + * @since 2201.7.0 + */ public class CommonUtils { public static Object sleep(BDecimal seconds) { try { diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index 6bfaebcb54b9..391bb69d1100 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -1,3 +1,20 @@ +// Copyright (c) 2023 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. +// +// WSO2 Inc. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + package org.ballerinalang.testerina.test; import org.ballerinalang.test.context.BMainInstance; From 2d252b68d43ff5f8c79c57c6582d5a608c842528 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Fri, 2 Jun 2023 09:23:21 +0530 Subject: [PATCH 09/37] Add comments --- .../main/ballerina/annotation_processor.bal | 4 ++++ .../src/main/ballerina/execute.bal | 22 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index b402cbef0777..6a33257cc3f6 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -46,6 +46,7 @@ public function registerTest(string name, function f) { function processConfigAnnotation(string name, function f) returns boolean { TestConfig? config = (typeof f).@Config; if config != () { + // Evaluate the test function to determine the parallelizability of the test function. boolean isTestFunctionIsolated = f is isolated function; boolean isDataProviderIsolated = true; boolean isTestFunctionParamSafe = true; @@ -68,6 +69,7 @@ function processConfigAnnotation(string name, function f) returns boolean { } } + // Register the reason for serial execution. if !isTestFunctionIsolated { reasonToSerialExecution.push("non-isolated test function"); } @@ -79,6 +81,8 @@ function processConfigAnnotation(string name, function f) returns boolean { if !isTestFunctionParamSafe { reasonToSerialExecution.push("unsafe test parameters"); } + + // If the test function is not parallelizable, then print the reason for serial execution. if !isSatisfiedParallelizableConditions && !config.serialExecution && (testWorkers > 1) { println("WARNING : Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 142b56065974..f0df60e29d86 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -69,6 +69,7 @@ function exitOnError() { function executeTests() returns error? { decimal startTime = currentTimeInMillis(); + // Add intial independant tests to the execution queue based on parallelizable condition foreach TestFunction testFunction in testRegistry.getFunctions() { if testFunction.parallelizable { parallelTestExecutionList.push(testFunction); @@ -94,29 +95,35 @@ function executeTests() returns error? { break; } + //Execute tests if there are available workers if (getAvailableWorkerCount() != 0) { + //If there are no tests to execute, wait for tests to be added to the execution queue if paralalTestExecutionListLength == 0 && serialTestExecutionListLength == 0 { runtime:sleep(0.0001); continue; } + //Execute serial tests if there are no test in execution process if (serialTestExecutionListLength != 0 && getAvailableWorkerCount() == testWorkers) { lock { testFunction = serialTestExecutionList.remove(0); } - + // wait until the test is complete execution if testFunction is TestFunction { allocateWorker(); future serialWaiter = start executeTest(testFunction); any _ = check wait serialWaiter; } + // Execute parallel tests if there are no serial tests in execution queue } else if paralalTestExecutionListLength != 0 && serialTestExecutionListLength == 0 { lock { testFunction = parallelTestExecutionList.remove(0); } + + // wait for the data driven tests to allocate workers if testFunction is TestFunction { allocateWorker(); future<(error?)> parallelWaiter = start executeTest(testFunction); @@ -133,10 +140,13 @@ function executeTests() returns error? { } function executeTest(TestFunction testFunction) returns error? { + // release worker if test is not enabled if !testFunction.enabled { releaseWorker(); return; } + + // relese worker if test has diagnostic errors error? diagnoseError = testFunction.diagnostics; if diagnoseError is error { lock { @@ -180,6 +190,7 @@ function executeTest(TestFunction testFunction) returns error? { releaseWorker(); } +//Reduce the dependents count and add to the execution queue if all the dependencies are executed function checkExecutionReadiness(TestFunction testFunction) { lock { testFunction.dependsOnCount -= 1; @@ -217,20 +228,25 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { boolean isIntialJob = true; while true { + //Break if there is no keys to execute or all keys are executed if keys.length() == 0 { break; } + //Reuse the already assigned worker for 1st data driven test or + // if there are available workers assign a worker for the next data driven test if isIntialJob || getAvailableWorkerCount() > 0 { string key = keys.remove(0); AnyOrError[] value = values.remove(0); + // Allocate worker from 2nd data driven test onwards if !isIntialJob { allocateWorker(); } future<()> serialWaiter = start prepareDataDrivenTest(testFunction, key, value, testType); + //Wait for the test to complete if the test is not parallelizable if !testFunction.parallelizable { any _ = check wait serialWaiter; } @@ -239,6 +255,7 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { isIntialJob = false; + //Wait until at least one worker is available if getAvailableWorkerCount() == 0 { runtime:sleep(0.0001); continue; @@ -246,6 +263,7 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { runtime:sleep(0.0001); } + //Wait until at least one worker is available while true { if getAvailableWorkerCount() > 0 { break; @@ -253,11 +271,13 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { runtime:sleep(0.0001); } + // Allocate the worker for the remaining processing of data driven test if !isIntialJob { allocateWorker(); } } +// Execute the data driven test and release the worker function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { boolean beforeFailed = executeBeforeFunction(testFunction); if (beforeFailed) { From c342bcf8178853f092b52d5cc8fa877ecd72e280 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Mon, 5 Jun 2023 17:50:41 +0530 Subject: [PATCH 10/37] Add necessary class --- .../main/ballerina/annotation_processor.bal | 2 +- .../src/main/ballerina/execute.bal | 90 +++++++-------- .../src/main/ballerina/filter.bal | 1 + .../src/main/ballerina/register.bal | 109 ++++++++++++++++++ 4 files changed, 153 insertions(+), 49 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 6a33257cc3f6..2462715da306 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -84,7 +84,7 @@ function processConfigAnnotation(string name, function f) returns boolean { // If the test function is not parallelizable, then print the reason for serial execution. if !isSatisfiedParallelizableConditions && !config.serialExecution && (testWorkers > 1) { - println("WARNING : Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); + println("WARNING: Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); } boolean enabled = config.enable && (filterGroups.length() == 0 ? true : hasGroup(config.groups, filterGroups)) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index f0df60e29d86..422ce15c6786 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -23,6 +23,7 @@ isolated int exitCode = 0; TestFunction[] parallelTestExecutionList = []; TestFunction[] serialTestExecutionList = []; isolated int unAllocatedTestWorkers = 1; +ConcurrentExecutionManager concurrentMgmr = new (1); public function startSuite() { // exit if setTestOptions has failed @@ -68,74 +69,72 @@ function exitOnError() { } function executeTests() returns error? { + TestFunction[] testsInExecution = []; decimal startTime = currentTimeInMillis(); // Add intial independant tests to the execution queue based on parallelizable condition foreach TestFunction testFunction in testRegistry.getFunctions() { - if testFunction.parallelizable { - parallelTestExecutionList.push(testFunction); - } else { - serialTestExecutionList.push(testFunction); - } - + _ = testFunction.parallelizable ? parallelTestExecutionList.push(testFunction) : serialTestExecutionList.push(testFunction); } - int paralalTestExecutionListLength = parallelTestExecutionList.length(); - int serialTestExecutionListLength = serialTestExecutionList.length(); - while true { - TestFunction? testFunction = (); - lock { - paralalTestExecutionListLength = parallelTestExecutionList.length(); - serialTestExecutionListLength = serialTestExecutionList.length(); - } //Exit from the loop if there are no tests to execute and all jobs are released - if (paralalTestExecutionListLength == 0 && getAvailableWorkerCount() == testWorkers - && serialTestExecutionListLength == 0) { + if parallelTestExecutionList.length() == 0 && getAvailableWorkerCount() == testWorkers + && serialTestExecutionList.length() == 0 && testsInExecution.length() == 0 { break; } + int i = 0; + + while i < testsInExecution.length() { + TestFunction testInProgress = testsInExecution[i]; + if testInProgress.isExecutionDone { + testInProgress.dependents.forEach(dependent => checkExecutionReadiness(dependent)); + _ = testsInExecution.remove(i); + } else { + i = i + 1; + } + } + //Execute tests if there are available workers if (getAvailableWorkerCount() != 0) { //If there are no tests to execute, wait for tests to be added to the execution queue - if paralalTestExecutionListLength == 0 && serialTestExecutionListLength == 0 { - runtime:sleep(0.0001); + if parallelTestExecutionList.length() == 0 && serialTestExecutionList.length() == 0 { + runtime:sleep(0.0001); // Check async waits continue; } //Execute serial tests if there are no test in execution process - if (serialTestExecutionListLength != 0 && getAvailableWorkerCount() == testWorkers) { - lock { - testFunction = serialTestExecutionList.remove(0); - } + if (serialTestExecutionList.length() != 0 && getAvailableWorkerCount() == testWorkers) { + TestFunction testFunction = serialTestExecutionList.remove(0); // wait until the test is complete execution - if testFunction is TestFunction { - allocateWorker(); - future serialWaiter = start executeTest(testFunction); - any _ = check wait serialWaiter; - } + testsInExecution.push(testFunction); + allocateWorker(); + future serialWaiter = start executeTest(testFunction); + any _ = check wait serialWaiter; // Execute parallel tests if there are no serial tests in execution queue - } else if paralalTestExecutionListLength != 0 && serialTestExecutionListLength == 0 { - lock { - testFunction = parallelTestExecutionList.remove(0); - } + } + else if parallelTestExecutionList.length() != 0 && serialTestExecutionList.length() == 0 { + TestFunction testFunction = parallelTestExecutionList.remove(0); // wait for the data driven tests to allocate workers - if testFunction is TestFunction { - allocateWorker(); - future<(error?)> parallelWaiter = start executeTest(testFunction); - if isDataDrivenTest(testFunction) { - any _ = check wait parallelWaiter; - } + + testsInExecution.push(testFunction); + allocateWorker(); + future<(error?)> parallelWaiter = start executeTest(testFunction); + if isDataDrivenTest(testFunction) { + any _ = check wait parallelWaiter; } } + } runtime:sleep(0.0001); } + println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n"); } @@ -186,22 +185,16 @@ function executeTest(TestFunction testFunction) returns error? { } }); } - testFunction.dependents.forEach(dependent => checkExecutionReadiness(dependent)); + testFunction.isExecutionDone = true; releaseWorker(); } //Reduce the dependents count and add to the execution queue if all the dependencies are executed function checkExecutionReadiness(TestFunction testFunction) { - lock { - testFunction.dependsOnCount -= 1; - if testFunction.dependsOnCount == 0 && testFunction.isInExecutionQueue != true { - testFunction.isInExecutionQueue = true; - if testFunction.parallelizable { - parallelTestExecutionList.push(testFunction); - } else { - serialTestExecutionList.push(testFunction); - } - } + testFunction.dependsOnCount -= 1; + if testFunction.dependsOnCount == 0 && testFunction.isInExecutionQueue != true { + testFunction.isInExecutionQueue = true; + _ = testFunction.parallelizable ? parallelTestExecutionList.push(testFunction) : serialTestExecutionList.push(testFunction); } } @@ -301,6 +294,7 @@ function executeDataDrivenTest(TestFunction testFunction, string suffix, TestTyp lock { reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + "]\n" + getErrorMessage(err), testType = testType); + //Remove *** && new line from the below line to print the error stack trace println("\n****************************************************\n" + testFunction.name + ":" + suffix + " has failed.\n****************************************************\n"); enableExit(); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index a0a7c3734fe7..7bc71b48a2d8 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -41,6 +41,7 @@ public function setTestOptions(string inTargetPath, string inPackageName, string boolean codeCoverage = parseBooleanInput(inCoverage, "code-coverage"); listGroups = parseBooleanInput(inListGroups, "list-groups"); testWorkers = parseIntegerInput(inTestWorkers, "testWorkers"); + concurrentMgmr = new (testWorkers); lock { unAllocatedTestWorkers = testWorkers; } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 769ead307b8b..6641052d3f55 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -42,8 +42,117 @@ type TestFunction record {| boolean isInExecutionQueue = false; boolean parallelizable = true; TestConfig? config = (); + boolean isExecutionDone = false; |}; +class ConcurrentExecutionManager { + private TestFunction[] parallelTestExecutionList = []; + private TestFunction[] serialTestExecutionList = []; + private TestFunction[] testsInExecution = []; + private int unAllocatedTestWorkers = 1; + private final int intialWorkers; + + function init(int workers) { + self.intialWorkers = workers; + self.unAllocatedTestWorkers = workers; + } + + function addParallelTest(TestFunction testFunction) { + self.parallelTestExecutionList.push(testFunction); + } + + function addSerialTest(TestFunction testFunction) { + self.serialTestExecutionList.push(testFunction); + } + + function addTestInExecution(TestFunction testFunction) { + self.testsInExecution.push(testFunction); + } + + function getConfiguredWorkers() returns int { + return self.intialWorkers; + } + + function getSerialQueueLength() returns int { + return self.serialTestExecutionList.length(); + } + + function getParallelQueueLength() returns int { + return self.parallelTestExecutionList.length(); + } + + function isExecutionDone() returns boolean { + return self.parallelTestExecutionList.length() == 0 && + self.getAvailableWorkers() == testWorkers && + self.serialTestExecutionList.length() == 0 && + self.testsInExecution.length() == 0; + + } + + isolated function allocateWorker() { + lock { + self.unAllocatedTestWorkers -= 1; + } + } + + isolated function releaseWorker() { + lock { + self.unAllocatedTestWorkers += 1; + } + } + + isolated function getAvailableWorkers() returns int { + lock { + return self.unAllocatedTestWorkers; + } + } + + function getParallelTest() returns TestFunction? { + if self.parallelTestExecutionList.length() > 0 { + return self.parallelTestExecutionList.remove(0); + } + return; + } + + function getSerialTest() returns TestFunction? { + if self.serialTestExecutionList.length() > 0 { + return self.serialTestExecutionList.remove(0); + } + return; + } + + function waitUntilEmptyQueueFilled() { + while parallelTestExecutionList.length() == 0 && serialTestExecutionList.length() == 0 { + self.populateExecutionQueues(); + if self.isExecutionDone() { + break; + } + } + } + + function populateExecutionQueues() { + int i = 0; + while i < self.testsInExecution.length() { + TestFunction testInProgress = self.testsInExecution[i]; + if testInProgress.isExecutionDone { + testInProgress.dependents.forEach(dependent => checkExecutionReadiness(dependent)); + _ = self.testsInExecution.remove(i); + } else { + i = i + 1; + } + } + } + + private function checkExecutionReadiness(TestFunction testFunction) { + testFunction.dependsOnCount -= 1; + if testFunction.dependsOnCount == 0 && testFunction.isInExecutionQueue != true { + testFunction.isInExecutionQueue = true; + _ = testFunction.parallelizable ? self.addParallelTest(testFunction) : self.addSerialTest(testFunction); + } + } + +} + class TestRegistry { private final TestFunction[] rootRegistry = []; private final TestFunction[] dependentRegistry = []; From 42c91ae40cdfbed5afed04d3d8e2a5a6be1adb21 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 28 Jun 2023 14:29:49 +0530 Subject: [PATCH 11/37] Utilise the concurrent manager --- .../src/main/ballerina/execute.bal | 145 ++++-------------- .../src/main/ballerina/filter.bal | 5 +- .../src/main/ballerina/register.bal | 36 +++-- .../test/TestparallelizationTest.java | 6 +- 4 files changed, 56 insertions(+), 136 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 422ce15c6786..4a1737388008 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -20,10 +20,7 @@ import ballerina/lang.runtime; isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; isolated int exitCode = 0; -TestFunction[] parallelTestExecutionList = []; -TestFunction[] serialTestExecutionList = []; -isolated int unAllocatedTestWorkers = 1; -ConcurrentExecutionManager concurrentMgmr = new (1); +ConcurrentExecutionManager conMgr = new (1); public function startSuite() { // exit if setTestOptions has failed @@ -69,61 +66,29 @@ function exitOnError() { } function executeTests() returns error? { - TestFunction[] testsInExecution = []; decimal startTime = currentTimeInMillis(); - // Add intial independant tests to the execution queue based on parallelizable condition foreach TestFunction testFunction in testRegistry.getFunctions() { - _ = testFunction.parallelizable ? parallelTestExecutionList.push(testFunction) : serialTestExecutionList.push(testFunction); + _ = testFunction.parallelizable ? conMgr.addParallelTest(testFunction) : conMgr.addSerialTest(testFunction); } - while true { + while !conMgr.isExecutionDone() { - //Exit from the loop if there are no tests to execute and all jobs are released - if parallelTestExecutionList.length() == 0 && getAvailableWorkerCount() == testWorkers - && serialTestExecutionList.length() == 0 && testsInExecution.length() == 0 { - break; - } - - int i = 0; - - while i < testsInExecution.length() { - TestFunction testInProgress = testsInExecution[i]; - if testInProgress.isExecutionDone { - testInProgress.dependents.forEach(dependent => checkExecutionReadiness(dependent)); - _ = testsInExecution.remove(i); - } else { - i = i + 1; - } - } - - //Execute tests if there are available workers - if (getAvailableWorkerCount() != 0) { - - //If there are no tests to execute, wait for tests to be added to the execution queue - if parallelTestExecutionList.length() == 0 && serialTestExecutionList.length() == 0 { - runtime:sleep(0.0001); // Check async waits - continue; + if conMgr.getAvailableWorkers() != 0 { + conMgr.waitUntilEmptyQueueFilled(); - } - - //Execute serial tests if there are no test in execution process - if (serialTestExecutionList.length() != 0 && getAvailableWorkerCount() == testWorkers) { - TestFunction testFunction = serialTestExecutionList.remove(0); - // wait until the test is complete execution - testsInExecution.push(testFunction); - allocateWorker(); + if conMgr.getSerialQueueLength() != 0 && conMgr.getAvailableWorkers() == testWorkers { + TestFunction testFunction = conMgr.getSerialTest(); + conMgr.addTestInExecution(testFunction); + conMgr.allocateWorker(); future serialWaiter = start executeTest(testFunction); any _ = check wait serialWaiter; - // Execute parallel tests if there are no serial tests in execution queue } - else if parallelTestExecutionList.length() != 0 && serialTestExecutionList.length() == 0 { - TestFunction testFunction = parallelTestExecutionList.remove(0); - // wait for the data driven tests to allocate workers - - testsInExecution.push(testFunction); - allocateWorker(); + else if conMgr.getParallelQueueLength() != 0 && conMgr.getSerialQueueLength() == 0 { + TestFunction testFunction = conMgr.getParallelTest(); + conMgr.addTestInExecution(testFunction); + conMgr.allocateWorker(); future<(error?)> parallelWaiter = start executeTest(testFunction); if isDataDrivenTest(testFunction) { any _ = check wait parallelWaiter; @@ -132,28 +97,26 @@ function executeTests() returns error? { } } - runtime:sleep(0.0001); + runtime:sleep(0.0001); // sleep is added to yield the strand } println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n"); } function executeTest(TestFunction testFunction) returns error? { - // release worker if test is not enabled if !testFunction.enabled { - releaseWorker(); + conMgr.releaseWorker(); return; } - // relese worker if test has diagnostic errors error? diagnoseError = testFunction.diagnostics; if diagnoseError is error { lock { reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); - println("\n****************************************************\n" + testFunction.name + " has failed.\n****************************************************\n"); + println("\n" + testFunction.name + " has failed.\n"); } enableExit(); - releaseWorker(); + conMgr.releaseWorker(); return; } @@ -185,17 +148,10 @@ function executeTest(TestFunction testFunction) returns error? { } }); } - testFunction.isExecutionDone = true; - releaseWorker(); -} - -//Reduce the dependents count and add to the execution queue if all the dependencies are executed -function checkExecutionReadiness(TestFunction testFunction) { - testFunction.dependsOnCount -= 1; - if testFunction.dependsOnCount == 0 && testFunction.isInExecutionQueue != true { - testFunction.isInExecutionQueue = true; - _ = testFunction.parallelizable ? parallelTestExecutionList.push(testFunction) : serialTestExecutionList.push(testFunction); + lock { + testFunction.isExecutionDone = true; } + conMgr.releaseWorker(); } function executeDataDrivenTestSet(TestFunction testFunction) returns error? { @@ -220,26 +176,18 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { boolean isIntialJob = true; - while true { - //Break if there is no keys to execute or all keys are executed - if keys.length() == 0 { - break; - } + while keys.length() != 0 { - //Reuse the already assigned worker for 1st data driven test or - // if there are available workers assign a worker for the next data driven test - if isIntialJob || getAvailableWorkerCount() > 0 { + if isIntialJob || conMgr.getAvailableWorkers() > 0 { string key = keys.remove(0); AnyOrError[] value = values.remove(0); - // Allocate worker from 2nd data driven test onwards if !isIntialJob { - allocateWorker(); + conMgr.allocateWorker(); } future<()> serialWaiter = start prepareDataDrivenTest(testFunction, key, value, testType); - //Wait for the test to complete if the test is not parallelizable if !testFunction.parallelizable { any _ = check wait serialWaiter; } @@ -247,30 +195,16 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { } isIntialJob = false; - - //Wait until at least one worker is available - if getAvailableWorkerCount() == 0 { - runtime:sleep(0.0001); - continue; - } - runtime:sleep(0.0001); + conMgr.waitForWorkers(); } - //Wait until at least one worker is available - while true { - if getAvailableWorkerCount() > 0 { - break; - } - runtime:sleep(0.0001); - } + conMgr.waitForWorkers(); - // Allocate the worker for the remaining processing of data driven test if !isIntialJob { - allocateWorker(); + conMgr.allocateWorker(); } } -// Execute the data driven test and release the worker function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { boolean beforeFailed = executeBeforeFunction(testFunction); if (beforeFailed) { @@ -281,7 +215,7 @@ function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError executeDataDrivenTest(testFunction, key, testType, value); var _ = executeAfterFunction(testFunction); } - releaseWorker(); + conMgr.releaseWorker(); } function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { @@ -294,8 +228,7 @@ function executeDataDrivenTest(TestFunction testFunction, string suffix, TestTyp lock { reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + "]\n" + getErrorMessage(err), testType = testType); - //Remove *** && new line from the below line to print the error stack trace - println("\n****************************************************\n" + testFunction.name + ":" + suffix + " has failed.\n****************************************************\n"); + println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); enableExit(); } } @@ -316,7 +249,7 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { failed = true; lock { reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); - println("\n****************************************************\n" + testFunction.name + " has failed.\n****************************************************\n"); + println("\n" + testFunction.name + " has failed.\n"); } } @@ -508,7 +441,7 @@ function executeTestFunction(TestFunction testFunction, string suffix, TestType enableExit(); lock { reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); - println("\n****************************************************\n" + testFunction.name + ":" + suffix + " has failed.\n****************************************************\n"); + println("\n" + testFunction.name + ":" + suffix + " has failed\n"); } return true; } else if output is any { @@ -608,24 +541,6 @@ function nestedEnabledDependentsAvailable(TestFunction[] dependents) returns boo return nestedEnabledDependentsAvailable(queue); } -isolated function allocateWorker() { - lock { - unAllocatedTestWorkers -= 1; - } -} - -isolated function releaseWorker() { - lock { - unAllocatedTestWorkers += 1; - } -} - -isolated function getAvailableWorkerCount() returns int { - lock { - return unAllocatedTestWorkers; - } -} - isolated function enableShouldSkip() { lock { shouldSkip = true; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index 7bc71b48a2d8..e0b25111436e 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -41,10 +41,7 @@ public function setTestOptions(string inTargetPath, string inPackageName, string boolean codeCoverage = parseBooleanInput(inCoverage, "code-coverage"); listGroups = parseBooleanInput(inListGroups, "list-groups"); testWorkers = parseIntegerInput(inTestWorkers, "testWorkers"); - concurrentMgmr = new (testWorkers); - lock { - unAllocatedTestWorkers = testWorkers; - } + conMgr = new (testWorkers); if rerunFailed { error? err = parseRerunJson(); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 6641052d3f55..0027a74004b2 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -1,3 +1,5 @@ +import ballerina/lang.runtime; + // Copyright (c) 2022 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. // // WSO2 Inc. licenses this file to you under the Apache License, @@ -101,29 +103,31 @@ class ConcurrentExecutionManager { } } + isolated function waitForWorkers() { + while self.getAvailableWorkers() < 1 { + runtime:sleep(0.0001); // sleep is added to yield the strand + } + } + isolated function getAvailableWorkers() returns int { lock { return self.unAllocatedTestWorkers; } } - function getParallelTest() returns TestFunction? { - if self.parallelTestExecutionList.length() > 0 { - return self.parallelTestExecutionList.remove(0); - } - return; + function getParallelTest() returns TestFunction { + return self.parallelTestExecutionList.remove(0); } - function getSerialTest() returns TestFunction? { - if self.serialTestExecutionList.length() > 0 { - return self.serialTestExecutionList.remove(0); - } - return; + function getSerialTest() returns TestFunction { + return self.serialTestExecutionList.remove(0); + } function waitUntilEmptyQueueFilled() { - while parallelTestExecutionList.length() == 0 && serialTestExecutionList.length() == 0 { + while self.parallelTestExecutionList.length() == 0 && self.serialTestExecutionList.length() == 0 { self.populateExecutionQueues(); + runtime:sleep(0.0001); // sleep is added to yield the strand if self.isExecutionDone() { break; } @@ -132,10 +136,14 @@ class ConcurrentExecutionManager { function populateExecutionQueues() { int i = 0; + boolean isExecutionDone = false; while i < self.testsInExecution.length() { TestFunction testInProgress = self.testsInExecution[i]; - if testInProgress.isExecutionDone { - testInProgress.dependents.forEach(dependent => checkExecutionReadiness(dependent)); + lock { + isExecutionDone = testInProgress.isExecutionDone; + } + if isExecutionDone { + testInProgress.dependents.forEach(dependent => self.checkExecutionReadiness(dependent)); _ = self.testsInExecution.remove(i); } else { i = i + 1; @@ -147,7 +155,7 @@ class ConcurrentExecutionManager { testFunction.dependsOnCount -= 1; if testFunction.dependsOnCount == 0 && testFunction.isInExecutionQueue != true { testFunction.isInExecutionQueue = true; - _ = testFunction.parallelizable ? self.addParallelTest(testFunction) : self.addSerialTest(testFunction); + _ = testFunction.parallelizable ? self.parallelTestExecutionList.push(testFunction) : self.serialTestExecutionList.push(testFunction); } } diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index 391bb69d1100..ecf1c24b2514 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -123,7 +123,7 @@ public void testNonParalallelizableTupleDataProvider() throws BallerinaTestExcep @Test public void testNonIsolatedTestFunction() throws BallerinaTestException, IOException { - String warningDiagnostics = "WARNING : Test function 'testAssertEquals*' cannot be parallelized due to " + + String warningDiagnostics = "WARNING: Test function 'testAssertEquals*' cannot be parallelized due to " + "non-isolated test function"; String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-tests"}); String output = balClient.runMainAndReadStdOut("test", args, @@ -146,7 +146,7 @@ public void testNonIsolatedTestFunction() throws BallerinaTestException, IOExcep @Test public void testNonIsolatedTestParameter() throws BallerinaTestException, IOException { - String warningDiagnostics = "WARNING : Test function 'mapDataProviderTest' cannot be parallelized due to " + + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized due to " + "unsafe test parameters"; String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-test-params"}); String output = balClient.runMainAndReadStdOut("test", args, @@ -166,7 +166,7 @@ public void testNonIsolatedTestParameter() throws BallerinaTestException, IOExce @Test public void testNonIsolatedDataProvider() throws BallerinaTestException, IOException { - String warningDiagnostics = "WARNING : Test function 'mapDataProviderTest' cannot be parallelized due to " + + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized due to " + "non-isolated data-provider function"; String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, From 397ff57a148893134c0408cfd8f25172e70ebba4 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 28 Jun 2023 15:38:05 +0530 Subject: [PATCH 12/37] Fix build failure --- .../testerina-core/src/main/ballerina/external.bal | 5 ----- .../ballerinalang/testerina/natives/CommonUtils.java | 12 ------------ 2 files changed, 17 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal index a7bd808bab81..b6fc907f193f 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal @@ -88,11 +88,6 @@ isolated function decode(string str, string charset) returns string|error = @jav 'class: "org.ballerinalang.testerina.natives.io.StringUtils" } external; -isolated function sleep(decimal seconds) returns error? = @java:Method { - name: "sleep", - 'class: "org.ballerinalang.testerina.natives.CommonUtils" -} external; - isolated function getBallerinaType((any|error) value) returns string = @java:Method { name: "getBallerinaType", 'class: "org.ballerinalang.testerina.core.BallerinaTypeCheck" diff --git a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java index 4c7064c9c3cb..85f7565eb22b 100644 --- a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java +++ b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java @@ -24,8 +24,6 @@ import io.ballerina.runtime.api.types.UnionType; import io.ballerina.runtime.api.values.BDecimal; import io.ballerina.runtime.api.values.BFunctionPointer; -import io.ballerina.runtime.internal.util.exceptions.BLangExceptionHelper; -import io.ballerina.runtime.internal.util.exceptions.RuntimeErrors; /** @@ -34,16 +32,6 @@ * @since 2201.7.0 */ public class CommonUtils { - public static Object sleep(BDecimal seconds) { - try { - Thread.sleep(seconds.intValue() * 1000); - } catch (InterruptedException e) { - return BLangExceptionHelper.getRuntimeException( - RuntimeErrors.OPERATION_NOT_SUPPORTED_ERROR, "Invalid duration: " + e.getMessage()); - } - return null; - } - public static BDecimal currentTimeInMillis() { long currentTime = System.currentTimeMillis(); return BDecimal.valueOf(currentTime); From e49136e1b9adfe6e7443ee3b36196456fcb2da6c Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 29 Jun 2023 15:40:30 +0530 Subject: [PATCH 13/37] Fix hanging issue --- .../testerina-core/src/main/ballerina/execute.bal | 8 +++++++- .../unix/BasicCasesTest-testAnnotations.txt | 6 +++--- .../unix/BasicCasesTest-testDependsOn.txt | 8 ++++---- ...aProviderTest-testArrayDataProviderWithFail.txt | 8 ++------ ...taProviderTest-testArrayDataRerunFailedTest.txt | 8 ++------ ...aProviderTest-testDataProviderSingleFailure.txt | 8 +++----- .../DataProviderTest-testMapValueDataProvider.txt | 4 +--- .../unix/DataProviderTest-testRerunFailedTest.txt | 8 ++------ ...aProviderTest-testValidDataProviderWithFail.txt | 8 ++------ .../GroupingTest-testNonExistingGroupExclusion.txt | 8 ++------ .../unix/GroupingTest-testSingleGroupExclusion.txt | 4 +--- ...ataProviderTestCase-testInvalidDataProvider.txt | 14 ++++++-------- ...taProviderTestCase-testInvalidDataProvider2.txt | 14 ++++++-------- ...oviderTestCase-testInvalidTupleDataProvider.txt | 14 ++++++-------- ...se-testMockingFunctionWithIncompatibleTypes.txt | 2 -- .../unix/MockTest-testFunctionMocking.txt | 6 ------ .../MockTest-testFunctionMockingModuleLevel.txt | 6 ------ ...-testFunctionMockingThenReturnWithNilRetVal.txt | 2 -- .../unix/MockTest-testObjectMocking.txt | 14 -------------- .../MockTest-testObjectMocking_NegativeCases1.txt | 2 -- .../MockTest-testObjectMocking_NegativeCases2.txt | 2 -- .../MockTest-testObjectMocking_NegativeCases3.txt | 2 -- .../MockTest-testObjectMocking_NegativeCases4.txt | 2 -- .../MockTest-testObjectMocking_NegativeCases5.txt | 2 -- .../MockTest-testObjectMocking_NegativeCases6.txt | 2 -- .../MockTest-testObjectMocking_NegativeCases7.txt | 2 -- .../MockTest-testObjectMocking_NonPublicField.txt | 2 -- .../unix/RerunFailedTest-testFullTest.txt | 8 ++------ .../unix/RerunFailedTest-testRerunFailedTest.txt | 8 ++------ ...TestCase-testSkipWhenDependsOnFunctionFails.txt | 2 -- ...ReportTest-testWarningForCoverageFormatFlag.txt | 4 +--- .../TestReportTest-testWarningForReportTools.txt | 4 +--- 32 files changed, 53 insertions(+), 139 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 130d5bb4a5f6..8f44ef8c529c 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -105,6 +105,9 @@ function executeTests() returns error? { function executeTest(TestFunction testFunction) returns error? { if !testFunction.enabled { + lock { + testFunction.isExecutionDone = true; + } conMgr.releaseWorker(); return; } @@ -116,6 +119,9 @@ function executeTest(TestFunction testFunction) returns error? { println("\n" + testFunction.name + " has failed.\n"); } enableExit(); + lock { + testFunction.isExecutionDone = true; + } conMgr.releaseWorker(); return; } @@ -345,7 +351,7 @@ function executeAfterGroupFunctions(TestFunction testFunction) { TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { ExecutionError? err = executeFunctions(afterGroupFunctions, - getShouldSkip()|| groupStatusRegistry.getSkipAfterGroup('group)); + getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); if err is ExecutionError { enableExit(); printExecutionError(err, "after test group function for the test"); diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt index 3c43d99f9a39..09f57e52f0aa 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt @@ -14,13 +14,13 @@ Running Tests with Coverage [pass] test4 [pass] test5 [pass] testBefore - [pass] testAfter - [pass] testAfterAlone - [pass] testDependsOn1 [pass] stringDataProviderTest#0 [pass] stringDataProviderTest#1 [pass] stringDataProviderTest#2 [pass] stringDataProviderTest2#0 + [pass] testAfter + [pass] testAfterAlone + [pass] testDependsOn1 14 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt index 157431325eb7..1cf19cef0acf 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt @@ -8,17 +8,17 @@ Running Tests with Coverage Test execution time :*****ms [pass] test1 + [pass] testAddingValues#0 + [pass] testAddingValues#1 + [pass] testAddingValues#2 [pass] test2 + [pass] testAddingValuesDependant [pass] test3 [pass] test4 [pass] testWithBefore1 [pass] testWithBefore2 [pass] testWithBefore3 [pass] testWithBefore4 - [pass] testAddingValues#0 - [pass] testAddingValues#1 - [pass] testAddingValues#2 - [pass] testAddingValuesDependant 12 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt index b15d0858176b..4954398b5e3b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intArrayDataProviderTest:0 has failed. -**************************************************** +intArrayDataProviderTest:0 has failed -**************************************************** -intArrayDataProviderTest:1 has failed. -**************************************************** +intArrayDataProviderTest:1 has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt index 7322a902d51b..847e362d6f9a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intArrayDataProviderTest:0 has failed. -**************************************************** +intArrayDataProviderTest:0 has failed -**************************************************** -intArrayDataProviderTest:1 has failed. -**************************************************** +intArrayDataProviderTest:1 has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt index 6108305ae478..03f3c93514d0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt @@ -7,9 +7,7 @@ Running Tests with Coverage dataproviders -**************************************************** testDividingValuesNegative:2 has failed. -**************************************************** Test execution time :*****ms @@ -27,9 +25,9 @@ testDividingValuesNegative:2 has failed. callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 17 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 460 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 232 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 221 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt index cab0fb57f9b5..2e4824b864b0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt @@ -7,9 +7,7 @@ Running Tests with Coverage dataproviders -**************************************************** -testGetState:1 has failed. -**************************************************** +testGetState:1 has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt index 6647c78558fd..e5e37f5788d2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intDataProviderTest:Case1 has failed. -**************************************************** +intDataProviderTest:Case1 has failed -**************************************************** -intDataProviderTest:Case2 has failed. -**************************************************** +intDataProviderTest:Case2 has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt index 8148cd806279..125ec3ddf3e6 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intDataProviderTest:Case1 has failed. -**************************************************** +intDataProviderTest:Case1 has failed -**************************************************** -intDataProviderTest:Case2 has failed. -**************************************************** +intDataProviderTest:Case2 has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt index bf3cc441adae..9b177f1d6b68 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt @@ -7,14 +7,10 @@ Running Tests groups-test.bal -**************************************************** -testFunc4: has failed. -**************************************************** +testFunc4: has failed -**************************************************** -testFunc5: has failed. -**************************************************** +testFunc5: has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt index c4316eef0276..a8e871f315ee 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt @@ -7,9 +7,7 @@ Running Tests groups-test.bal -**************************************************** -testFunc5: has failed. -**************************************************** +testFunc5: has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt index 691b426e44af..2b306b99c345 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -8,9 +8,7 @@ Running Tests invalid-data-provider-test.bal -**************************************************** testInvalidDataProvider:0 has failed. -**************************************************** Test execution time :*****ms @@ -21,13 +19,13 @@ testInvalidDataProvider:0 has failed. [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 445 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 232 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 221 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 460 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 232 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 221 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index 89a83384cf0b..c7c19a73214a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -10,9 +10,7 @@ Running Tests invalid-data-provider-test2.bal -**************************************************** testInvalidDataProvider2:0 has failed. -**************************************************** Test execution time :*****ms @@ -23,13 +21,13 @@ testInvalidDataProvider2:0 has failed. [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 445 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 232 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 221 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 460 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 232 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 221 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 4ee3c92d555b..55a9a4f1dd80 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -7,9 +7,7 @@ Running Tests invalid-data-provider-test3.bal -**************************************************** testInvalidTupleDataProvider:0 has failed. -**************************************************** Test execution time :*****ms @@ -20,13 +18,13 @@ testInvalidTupleDataProvider:0 has failed. [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 492 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 445 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 232 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 221 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 507 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 279 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 268 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 460 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 232 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 221 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt index be2b6d68d317..f32e25a93c5f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt @@ -5,9 +5,7 @@ Running Tests incompatible_type_mock -**************************************************** functionMockingTest has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt index 6ed66acea85b..03759611000a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMocking.txt @@ -5,19 +5,13 @@ Running Tests with Coverage function_mocking -**************************************************** call_Test3 has failed. -**************************************************** -**************************************************** call_Test4 has failed. -**************************************************** -**************************************************** call_Test5 has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt index 6ed66acea85b..03759611000a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingModuleLevel.txt @@ -5,19 +5,13 @@ Running Tests with Coverage function_mocking -**************************************************** call_Test3 has failed. -**************************************************** -**************************************************** call_Test4 has failed. -**************************************************** -**************************************************** call_Test5 has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt index 69633db91d42..0751baeefec1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt @@ -5,9 +5,7 @@ Running Tests with Coverage function_mocking -**************************************************** testFunctionMock3 has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt index 7a3607262fc6..ef365df2795c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt @@ -7,39 +7,25 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultMockInvalidFieldName has failed. -**************************************************** -**************************************************** testMockInvalidStream has failed. -**************************************************** -**************************************************** testDefaultIncompatibleArgs has failed. -**************************************************** -**************************************************** testDefaultInvalidMemberReturnValue has failed. -**************************************************** -**************************************************** testDefaultMockInvalidReturnValue has failed. -**************************************************** -**************************************************** testDefaultMockWrongAction has failed. -**************************************************** -**************************************************** testDefaultTooManyArgs has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt index 47513eec2f5c..34a4fe5a3511 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases1.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultIncompatibleArgs has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt index 30d67ded5139..6a6383e31fe9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases2.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultInvalidMemberReturnValue has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt index 06dbd63af55f..fd676f1ff64d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases3.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultMockInvalidFieldName has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt index 132e141ca51c..b4c5586d3d00 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases4.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultMockInvalidReturnValue has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt index f4663714fbfb..4bcdafc5519e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases5.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultMockWrongAction has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt index 0f201695e411..2350b5969f7e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases6.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultTooManyArgs has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt index 34cebaf70957..8f3cc16837c8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NegativeCases7.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testMockInvalidStream has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt index cbe0eff07abb..b8397dcd79de 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking_NonPublicField.txt @@ -5,9 +5,7 @@ Running Tests with Coverage non_public_field_mock -**************************************************** testNonPublicMemberFieldMock has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt index 5f30cf1614dc..5a9660146a84 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt @@ -5,14 +5,10 @@ Running Tests with Coverage rerun_failed -**************************************************** -testFunctionFail1: has failed. -**************************************************** +testFunctionFail1: has failed -**************************************************** -testFunctionFail2: has failed. -**************************************************** +testFunctionFail2: has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt index 0e0ce8f98171..7542c2be35f4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt @@ -5,14 +5,10 @@ Running Tests with Coverage rerun_failed -**************************************************** -testFunctionFail1: has failed. -**************************************************** +testFunctionFail1: has failed -**************************************************** -testFunctionFail2: has failed. -**************************************************** +testFunctionFail2: has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt index b3b80df4f3a1..e73b617172eb 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt @@ -8,9 +8,7 @@ Running Tests dependson-skip-test.bal -**************************************************** test2 has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt index b685ffecd479..4f4e4608e1c1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt @@ -7,9 +7,7 @@ Running Tests foo -**************************************************** -testMain: has failed. -**************************************************** +testMain: has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt index 35cdd8aa10ae..195c28b821d5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt @@ -6,9 +6,7 @@ Running Tests with Coverage foo -**************************************************** -testMain: has failed. -**************************************************** +testMain: has failed Test execution time :*****ms From d69ef3092c83e7874b02f78082e9e2589f9f12dc Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Tue, 4 Jul 2023 15:41:17 +0530 Subject: [PATCH 14/37] Fix StrandDumpTest failure --- .../testOutputs/balTestStrandDumpRegEx.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt b/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt index e60dca503a85..914eab9e0ff6 100644 --- a/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt +++ b/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt @@ -2,20 +2,22 @@ Ballerina Strand Dump \[\d*/\d*/\d* \d*:\d*:\d*\] =========================================== Total strand group count \t:\t3 -Total strand count \t:\t8 +Total strand count \t:\t9 Active strand group count\t:\t1 -Active strand count \t:\t8 +Active strand count \t:\t9 -group \d* \[QUEUED\]: \[8\] -\tstrand \d* "main" \[testOrg.testPackageWithModules.0:main\] \[BLOCKED\]: +group \d* \[QUEUED\]: \[9\] +\tstrand \d* "main" \[testOrg.testPackageWithModules.0:main\] \[WAITING\]: +\t\tat\tballerina.test.\d*.\d*.\d*:executeTests\(execute.bal:\d*\) +\t\t \tballerina.test.\d*.\d*.\d*:startSuite\(execute.bal:\d*\) +\t\t \ttestOrg.testPackageWithModules.0.1.0:__execute__\(tests/test_execute-generated_1.bal:\d*\) +\t\t \t\$moduleExecute + +\tstrand \d* "serialWaiter" \[ballerina.test.\d*:executeTests\]\[\d*\] \[BLOCKED\]: \t\tat\tballerina.lang.function.\d*.\d*.\d*:call\(function.bal:\d*\) \t\t \tballerina.test.\d*.\d*.\d*:executeTestFunction\(execute.bal:\d*\) \t\t \tballerina.test.\d*.\d*.\d*:executeNonDataDrivenTest\(execute.bal:\d*\) \t\t \tballerina.test.\d*.\d*.\d*:executeTest\(execute.bal:\d*\) -\t\t \tballerina.test.\d*.\d*.\d*:executeTests\(execute.bal:\d*\) -\t\t \tballerina.test.\d*.\d*.\d*:startSuite\(execute.bal:\d*\) -\t\t \ttestOrg.testPackageWithModules.0.1.0:__execute__\(tests/test_execute-generated_1.bal:\d*\) -\t\t \t\$moduleExecute \tstrand \d* \[ballerina.lang.function.\d*.\d*.\d*:call\]\[\d*\] \[WAITING\]: \t\tat\ttestOrg.testPackageWithModules.0.1.0:bar\(main.bal:52\) From 2170c859f446536dcd0013621bf4fc347f22ad56 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Mon, 11 Dec 2023 16:10:31 +0530 Subject: [PATCH 15/37] Add lock to exit code --- .../src/main/ballerina/execute.bal | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index ae81ddfb573a..d6429c53a5f5 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -13,7 +13,6 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - import ballerina/lang.'error as langError; import ballerina/lang.runtime; @@ -24,9 +23,12 @@ ConcurrentExecutionManager conMgr = new (1); public function startSuite() returns int { // exit if setTestOptions has failed - if exitCode > 0 { - return exitCode; + lock { + if exitCode > 0 { + return exitCode; + } } + if listGroups { string[] groupsList = groupStatusRegistry.getGroupsList(); if groupsList.length() == 0 { @@ -38,7 +40,10 @@ public function startSuite() returns int { } else { if testRegistry.getFunctions().length() == 0 && testRegistry.getDependentFunctions().length() == 0 { println("\tNo tests found"); - return exitCode; + lock { + return exitCode; + } + } error? err = orderTests(); @@ -56,7 +61,9 @@ public function startSuite() returns int { reportGenerators.forEach(reportGen => reportGen(reportData)); } } - return exitCode; + lock { + return exitCode; + } } function executeTests() returns error? { From 4b9ee0a1c8b7de59fbd8a52a82a096ba29d7494a Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Mon, 11 Dec 2023 17:58:53 +0530 Subject: [PATCH 16/37] Modify integration test outputs --- ...iderTest-testDataProviderSingleFailure.txt | 19 +++--- ...-testValidDataProviderWithAfterFailing.txt | 2 +- ...testValidDataProviderWithBeforeFailing.txt | 2 +- .../GroupingTest-beforeGroupsAfterGroups2.txt | 2 +- ...oviderTestCase-testInvalidDataProvider.txt | 26 ++++---- ...viderTestCase-testInvalidDataProvider2.txt | 26 ++++---- ...rTestCase-testInvalidTupleDataProvider.txt | 26 ++++---- .../unix/MockTest-testObjectMocking.txt | 60 +++++++++++++------ .../ModuleExecutionWithInitStartFailures.txt | 8 ++- ...racefulStopTest-test-listener-shutdown.txt | 17 +++++- 10 files changed, 116 insertions(+), 72 deletions(-) diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt index 48139cc61cf4..9a4e2a518840 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt @@ -6,6 +6,12 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +testDividingValuesNegative:2 has failed. + + + Test execution time :*****ms + [pass] testDividingValuesNegative#0 [pass] testDividingValuesNegative#1 [pass] testDividingValuesNegative#3 @@ -19,13 +25,9 @@ Running Tests with Coverage callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 18 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 27 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 @@ -35,6 +37,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt index ad1050cea3ea..ac01995d5ef0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt @@ -9,7 +9,7 @@ Running Tests with Coverage [fail] [after test function for the test]: error("{ballerina}DivisionByZero",message=" / by zero") callableName: afterFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 153 - callableName: afterFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 + callableName: afterFailsFunction$lambda84$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt index b45267d98a5d..8f3a0bf59c7a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt @@ -9,7 +9,7 @@ Running Tests with Coverage [fail] [before test function for the test]: error("{ballerina}DivisionByZero",message=" / by zero") callableName: beforeFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 121 - callableName: beforeFailsFunction$lambda62$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 + callableName: beforeFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt index 101b23b28c97..6fadb5466f67 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt @@ -28,7 +28,7 @@ Running Tests callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuiteFunc fileName: before-groups-after-groups-test2.bal lineNumber: 80 - callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 91 + callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 92 [pass] testFunction [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt index a27ffae1b440..525766123d68 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -8,26 +8,24 @@ Running Tests invalid-data-provider-test.bal +testInvalidDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider#0: [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 339 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index 921212fb9f70..092f0ecf5fad 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -10,26 +10,24 @@ Running Tests invalid-data-provider-test2.bal +testInvalidDataProvider2:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider2#0: [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 339 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 068ebba3b51e..03f75dcf1607 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -7,26 +7,24 @@ Running Tests invalid-data-provider-test3.bal +testInvalidTupleDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidTupleDataProvider#0: [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 339 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt index 787b738aafb2..e43e502b57ee 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt @@ -6,6 +6,30 @@ WARNING [main.bal:(47:5,47:47)] unused variable 'closeErr' Running Tests with Coverage object_mocking + +testDefaultMockInvalidFieldName has failed. + + +testMockInvalidStream has failed. + + +testDefaultIncompatibleArgs has failed. + + +testDefaultInvalidMemberReturnValue has failed. + + +testDefaultMockInvalidReturnValue has failed. + + +testDefaultMockWrongAction has failed. + + +testDefaultTooManyArgs has failed. + + + Test execution time :*****ms + [pass] testDependentlyTypedFunctions_testDouble [pass] testDependentlyTypedFunctions_thenReturn [pass] testMockMemberVariable @@ -16,6 +40,24 @@ Running Tests with Coverage [pass] testProvideErrorReturnValue [pass] testUserDefinedMockObject + [fail] testDefaultMockInvalidFieldName: + + error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") + callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 + callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 + callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 + callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 + + + [fail] testMockInvalidStream: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 + callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 + callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 + + [fail] testDefaultIncompatibleArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") @@ -34,15 +76,6 @@ Running Tests with Coverage callableName: testDefaultInvalidMemberReturnValue$lambda5$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 - [fail] testDefaultMockInvalidFieldName: - - error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") - callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 - callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 - callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 - callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 - - [fail] testDefaultMockInvalidReturnValue: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") @@ -70,15 +103,6 @@ Running Tests with Coverage callableName: testDefaultTooManyArgs$lambda2$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 6 - [fail] testMockInvalidStream: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 - callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 - callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 - - 9 passing 7 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt index b135d95d38d8..d465ba49e9d7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt @@ -16,6 +16,12 @@ error: Error from start of moduleC at wso2.moduleExecutionInitStartFailure.moduleC.0.Listener:start(main.bal:32) moduleExecutionInitStartFailure.moduleB + +test2: has failed + + + Test execution time :*****ms + [pass] test1 [pass] test3 @@ -35,4 +41,4 @@ error: Error from start of moduleC 2 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt index 5b2bf742b2b1..25a55f23f2ee 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt @@ -8,6 +8,15 @@ Calling init for moduleA listener Calling init for default module listener Calling start for moduleA listener Calling start for default module listener + +main_negative_test1: has failed + + +main_negative_test2: has failed + + + Test execution time :*****ms + [pass] main_test1 [pass] main_test2 @@ -45,6 +54,12 @@ Calling stop for moduleA listener moduleGracefulStopTest.moduleA Calling init for moduleA listener Calling start for moduleA listener + +negative_test1: has failed + + + Test execution time :*****ms + [pass] test1 [fail] negative_test1: @@ -64,4 +79,4 @@ Calling start for moduleA listener 1 failing 0 skipped Calling stop for moduleA listener -error: there are test failures +error: there are test failures \ No newline at end of file From e4eed51df9552f31f3ccdb8a84a2d85a2aacafa4 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Tue, 12 Dec 2023 00:07:08 -0800 Subject: [PATCH 17/37] Fix build failures --- .../project_b_100/resources/expectedDeps.toml | 12 ++- .../BasicCasesTest-testAnnotations.txt | 6 +- .../windows/BasicCasesTest-testDependsOn.txt | 8 +- ...iderTest-testArrayDataProviderWithFail.txt | 10 +-- ...viderTest-testArrayDataRerunFailedTest.txt | 10 +-- ...iderTest-testDataProviderSingleFailure.txt | 21 ++++-- ...aProviderTest-testMapValueDataProvider.txt | 6 +- .../DataProviderTest-testRerunFailedTest.txt | 10 +-- ...-testValidDataProviderWithAfterFailing.txt | 74 ++++++++++--------- ...testValidDataProviderWithBeforeFailing.txt | 72 +++++++++--------- ...iderTest-testValidDataProviderWithFail.txt | 10 +-- .../GroupingTest-beforeGroupsAfterGroups2.txt | 2 +- .../GroupingTest-failedBeforeEachTest.txt | 2 +- .../GroupingTest-failedBeforeGroupTest.txt | 2 +- ...pingTest-testNonExistingGroupExclusion.txt | 10 +-- .../GroupingTest-testSingleGroupExclusion.txt | 6 +- .../GroupingTest-testWhenAfterGroupsFails.txt | 2 +- ...oviderTestCase-testInvalidDataProvider.txt | 28 ++++--- ...viderTestCase-testInvalidDataProvider2.txt | 28 ++++--- ...rTestCase-testInvalidTupleDataProvider.txt | 28 ++++--- ...stMockingFunctionWithIncompatibleTypes.txt | 2 - .../windows/MockTest-testFunctionMocking.txt | 8 +- .../MockTest-testFunctionMockingLegacy.txt | 37 +++++----- ...ockTest-testFunctionMockingModuleLevel.txt | 8 +- ...FunctionMockingThenReturnWithNilRetVal.txt | 2 - .../windows/MockTest-testObjectMocking.txt | 62 +++++++++++----- ...kTest-testObjectMocking_NegativeCases1.txt | 4 +- ...kTest-testObjectMocking_NegativeCases2.txt | 4 +- ...kTest-testObjectMocking_NegativeCases3.txt | 4 +- ...kTest-testObjectMocking_NegativeCases4.txt | 4 +- ...kTest-testObjectMocking_NegativeCases5.txt | 4 +- ...kTest-testObjectMocking_NegativeCases6.txt | 4 +- ...kTest-testObjectMocking_NegativeCases7.txt | 4 +- ...kTest-testObjectMocking_NonPublicField.txt | 4 +- ...ecutionFlowTest-test-listener-shutdown.txt | 8 +- .../ModuleExecutionWithInitStartFailures.txt | 8 +- ...racefulStopTest-test-listener-shutdown.txt | 17 ++++- .../windows/RerunFailedTest-testFullTest.txt | 8 +- .../RerunFailedTest-testRerunFailedTest.txt | 8 +- ...stsTestCase-testSkipWhenAfterEachFails.txt | 2 +- ...ipTestsTestCase-testSkipWhenAfterFails.txt | 2 +- ...tsTestCase-testSkipWhenBeforeEachFails.txt | 2 +- ...pTestsTestCase-testSkipWhenBeforeFails.txt | 2 +- ...TestCase-testSkipWhenBeforeGroupsFails.txt | 2 +- ...sTestCase-testSkipWhenBeforeSuiteFails.txt | 2 +- ...ase-testSkipWhenDependsOnFunctionFails.txt | 4 +- ...tTest-testWarningForCoverageFormatFlag.txt | 6 +- ...stReportTest-testWarningForReportTools.txt | 6 +- 48 files changed, 290 insertions(+), 285 deletions(-) diff --git a/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml b/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml index 9aa444443339..703d373b52a3 100644 --- a/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml +++ b/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml @@ -22,6 +22,15 @@ dependencies = [ {org = "ballerina", name = "jballerina.java"} ] +[[package]] +org = "ballerina" +name = "lang.runtime" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + [[package]] org = "ballerina" name = "test" @@ -29,7 +38,8 @@ version = "0.0.0" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.error"} + {org = "ballerina", name = "lang.error"}, + {org = "ballerina", name = "lang.runtime"} ] modules = [ {org = "ballerina", packageName = "test", moduleName = "test"} diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt index 5e50820efd00..4564b4cc2821 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt @@ -14,13 +14,13 @@ Running Tests with Coverage [pass] test4 [pass] test5 [pass] testBefore - [pass] testAfter - [pass] testAfterAlone - [pass] testDependsOn1 [pass] stringDataProviderTest#0 [pass] stringDataProviderTest#1 [pass] stringDataProviderTest#2 [pass] stringDataProviderTest2#0 + [pass] testAfter + [pass] testAfterAlone + [pass] testDependsOn1 14 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt index 6fdecbfee824..7f183c35f4f9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt @@ -8,17 +8,17 @@ Running Tests with Coverage Test execution time :*****ms [pass] test1 + [pass] testAddingValues#0 + [pass] testAddingValues#1 + [pass] testAddingValues#2 [pass] test2 + [pass] testAddingValuesDependant [pass] test3 [pass] test4 [pass] testWithBefore1 [pass] testWithBefore2 [pass] testWithBefore3 [pass] testWithBefore4 - [pass] testAddingValues#0 - [pass] testAddingValues#1 - [pass] testAddingValues#2 - [pass] testAddingValuesDependant 12 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt index 95edeab6a4a5..d6b9021f01d0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intArrayDataProviderTest:0 has failed. -**************************************************** +intArrayDataProviderTest:0 has failed -**************************************************** -intArrayDataProviderTest:1 has failed. -**************************************************** +intArrayDataProviderTest:1 has failed Test execution time :*****ms @@ -61,4 +57,4 @@ intArrayDataProviderTest:1 has failed. Generating Test Report data-providers\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt index ee581dc93e55..ca6b41a8bd3e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intArrayDataProviderTest:0 has failed. -**************************************************** +intArrayDataProviderTest:0 has failed -**************************************************** -intArrayDataProviderTest:1 has failed. -**************************************************** +intArrayDataProviderTest:1 has failed Test execution time :*****ms @@ -60,4 +56,4 @@ intArrayDataProviderTest:1 has failed. Generating Test Report data-providers\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt index c63c5bc8af00..f70be3d2fcb0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt @@ -6,6 +6,12 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +testDividingValuesNegative:2 has failed. + + + Test execution time :*****ms + [pass] testDividingValuesNegative#0 [pass] testDividingValuesNegative#1 [pass] testDividingValuesNegative#3 @@ -19,13 +25,9 @@ Running Tests with Coverage callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 18 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 27 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 @@ -35,10 +37,13 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found Generating Test Report data-providers\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt index 90fe48776b40..0ce50e4c8357 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt @@ -7,9 +7,7 @@ Running Tests with Coverage dataproviders -**************************************************** -testGetState:1 has failed. -**************************************************** +testGetState:1 has failed Test execution time :*****ms @@ -44,4 +42,4 @@ testGetState:1 has failed. Generating Test Report data-providers\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt index ffd9b46bcca8..326e653ce3c4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intDataProviderTest:Case1 has failed. -**************************************************** +intDataProviderTest:Case1 has failed -**************************************************** -intDataProviderTest:Case2 has failed. -**************************************************** +intDataProviderTest:Case2 has failed Test execution time :*****ms @@ -60,4 +56,4 @@ intDataProviderTest:Case2 has failed. Generating Test Report data-providers\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt index fe59becf0e30..850bf54956e4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt @@ -1,34 +1,40 @@ -Compiling source - intg_tests/dataproviders:0.0.0 -WARNING [tests\new-data-provider-tests.bal:(121:9,121:21)] unused variable 'a' -WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' - -Running Tests with Coverage - - dataproviders - [fail] [after test function for the test]: - error("{ballerina}DivisionByZero",message=" / by zero") - callableName: afterFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 153 - callableName: afterFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 - - [pass] testDividingValuesWithAfterFailing#0 - [pass] testDividingValuesWithAfterFailing#1 - [pass] testDividingValuesWithAfterFailing#2 - [pass] testDividingValuesWithAfterFailing#3 - [pass] testDividingValuesWithAfterFailing#4 - [pass] testExecutionOfAfterFailing - - - 6 passing - 0 failing - 0 skipped - - dataproviders.module1 - - - No tests found - -Generating Test Report - data-providers\target\report\test_results.json - -error: there are test failures +Compiling source + intg_tests/dataproviders:0.0.0 +WARNING [tests\new-data-provider-tests.bal:(121:9,121:21)] unused variable 'a' +WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' + +Running Tests with Coverage + + dataproviders + [fail] [after test function for the test]: + error("{ballerina}DivisionByZero",message=" / by zero") + callableName: afterFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 153 + callableName: afterFailsFunction$lambda84$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 + + + Test execution time :*****ms + + [pass] testDividingValuesWithAfterFailing#0 + [pass] testDividingValuesWithAfterFailing#1 + [pass] testDividingValuesWithAfterFailing#2 + [pass] testDividingValuesWithAfterFailing#3 + [pass] testDividingValuesWithAfterFailing#4 + [pass] testExecutionOfAfterFailing + + + 6 passing + 0 failing + 0 skipped + + dataproviders.module1 + + Test execution time :*****ms + + + + No tests found + +Generating Test Report + data-providers\target\report\test_results.json + +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt index 4a194c293db7..b4bee5dcaf75 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt @@ -1,33 +1,39 @@ -Compiling source - intg_tests/dataproviders:0.0.0 -WARNING [tests\new-data-provider-tests.bal:(121:9,121:21)] unused variable 'a' -WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' - -Running Tests with Coverage - - dataproviders - [fail] [before test function for the test]: - error("{ballerina}DivisionByZero",message=" / by zero") - callableName: beforeFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 121 - callableName: beforeFailsFunction$lambda62$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 - - [pass] testDividingValuesWithBeforeFailing#0 - [pass] testDividingValuesWithBeforeFailing#1 - [pass] testDividingValuesWithBeforeFailing#3 - [pass] testDividingValuesWithBeforeFailing#4 - [pass] testExecutionOfBeforeFailing - - - 5 passing - 0 failing - 1 skipped - - dataproviders.module1 - - - No tests found - -Generating Test Report - data-providers\target\report\test_results.json - -error: there are test failures +Compiling source + intg_tests/dataproviders:0.0.0 +WARNING [tests\new-data-provider-tests.bal:(121:9,121:21)] unused variable 'a' +WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' + +Running Tests with Coverage + + dataproviders + [fail] [before test function for the test]: + error("{ballerina}DivisionByZero",message=" / by zero") + callableName: beforeFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 121 + callableName: beforeFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 + + + Test execution time :*****ms + + [pass] testDividingValuesWithBeforeFailing#0 + [pass] testDividingValuesWithBeforeFailing#1 + [pass] testDividingValuesWithBeforeFailing#3 + [pass] testDividingValuesWithBeforeFailing#4 + [pass] testExecutionOfBeforeFailing + + + 5 passing + 0 failing + 1 skipped + + dataproviders.module1 + + Test execution time :*****ms + + + + No tests found + +Generating Test Report + data-providers\target\report\test_results.json + +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt index b3d279948692..69a7549a761e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt @@ -7,14 +7,10 @@ Running Tests with Coverage dataproviders -**************************************************** -intDataProviderTest:Case1 has failed. -**************************************************** +intDataProviderTest:Case1 has failed -**************************************************** -intDataProviderTest:Case2 has failed. -**************************************************** +intDataProviderTest:Case2 has failed Test execution time :*****ms @@ -61,4 +57,4 @@ intDataProviderTest:Case2 has failed. Generating Test Report data-providers\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt index f8ab35f6c57b..d0e32fb03a9d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt @@ -28,7 +28,7 @@ Running Tests callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuiteFunc fileName: before-groups-after-groups-test2.bal lineNumber: 80 - callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 91 + callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 92 [pass] testFunction [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt index dfb50bc452f8..d0d2e9f86cbf 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeEachTest.txt @@ -21,4 +21,4 @@ Running Tests 0 passing 0 failing 4 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt index a4ad8ef356bd..99a38226f9af 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-failedBeforeGroupTest.txt @@ -22,4 +22,4 @@ Running Tests 3 passing 0 failing 2 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt index 55aab4e7e440..105744904750 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt @@ -7,14 +7,10 @@ Running Tests groups-test.bal -**************************************************** -testFunc4: has failed. -**************************************************** +testFunc4: has failed -**************************************************** -testFunc5: has failed. -**************************************************** +testFunc5: has failed Test execution time :*****ms @@ -46,4 +42,4 @@ testFunc5: has failed. 4 passing 2 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt index 7e87c0d7fae8..fb70d13eab0f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt @@ -7,9 +7,7 @@ Running Tests groups-test.bal -**************************************************** -testFunc5: has failed. -**************************************************** +testFunc5: has failed Test execution time :*****ms @@ -32,4 +30,4 @@ testFunc5: has failed. 4 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt index c2d09f5e5bb6..e2c83a31876e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testWhenAfterGroupsFails.txt @@ -25,4 +25,4 @@ Running Tests 5 passing 0 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt index ba73eb492805..8bf3862f950f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -8,30 +8,28 @@ Running Tests invalid-data-provider-test.bal +testInvalidDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider#0: [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 339 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index 95f28ab15b5a..cbad83abbcd3 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -10,30 +10,28 @@ Running Tests invalid-data-provider-test2.bal +testInvalidDataProvider2:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider2#0: [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 339 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 43fb04a46c97..6b452602fee8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -7,30 +7,28 @@ Running Tests invalid-data-provider-test3.bal +testInvalidTupleDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidTupleDataProvider#0: [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 339 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 - callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt index b2bbfdbcd39a..10e9590c3018 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidFunctionMockingTestCase-testMockingFunctionWithIncompatibleTypes.txt @@ -5,9 +5,7 @@ Running Tests incompatible_type_mock -**************************************************** functionMockingTest has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt index 3aae0db98904..0acaa3a3666b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMocking.txt @@ -5,19 +5,13 @@ Running Tests with Coverage function_mocking -**************************************************** call_Test3 has failed. -**************************************************** -**************************************************** call_Test4 has failed. -**************************************************** -**************************************************** call_Test5 has failed. -**************************************************** Test execution time :*****ms @@ -82,4 +76,4 @@ call_Test5 has failed. Generating Test Report function-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt index 184ca4ac709c..a8ca32afc8c5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingLegacy.txt @@ -1,17 +1,20 @@ -Compiling source - intg_tests/function_mocking_legacy:0.1.0 - -Running Tests with Coverage - - function_mocking_legacy - [pass] testIntAdd - [pass] testIntMul - - - 2 passing - 0 failing - 0 skipped - -Generating Test Report - legacy-function-mocking-tests\target\report\test_results.json - +Compiling source + intg_tests/function_mocking_legacy:0.1.0 + +Running Tests with Coverage + + function_mocking_legacy + + Test execution time :*****ms + + [pass] testIntAdd + [pass] testIntMul + + + 2 passing + 0 failing + 0 skipped + +Generating Test Report + legacy-function-mocking-tests\target\report\test_results.json + \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt index 3aae0db98904..0acaa3a3666b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingModuleLevel.txt @@ -5,19 +5,13 @@ Running Tests with Coverage function_mocking -**************************************************** call_Test3 has failed. -**************************************************** -**************************************************** call_Test4 has failed. -**************************************************** -**************************************************** call_Test5 has failed. -**************************************************** Test execution time :*****ms @@ -82,4 +76,4 @@ call_Test5 has failed. Generating Test Report function-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt index 5b34492579e0..976aea0736de 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testFunctionMockingThenReturnWithNilRetVal.txt @@ -5,9 +5,7 @@ Running Tests with Coverage function_mocking -**************************************************** testFunctionMock3 has failed. -**************************************************** Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt index 629ee60cd9f9..c4f3c2f6bb5e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt @@ -6,6 +6,30 @@ WARNING [main.bal:(47:5,47:47)] unused variable 'closeErr' Running Tests with Coverage object_mocking + +testDefaultMockInvalidFieldName has failed. + + +testMockInvalidStream has failed. + + +testDefaultIncompatibleArgs has failed. + + +testDefaultInvalidMemberReturnValue has failed. + + +testDefaultMockInvalidReturnValue has failed. + + +testDefaultMockWrongAction has failed. + + +testDefaultTooManyArgs has failed. + + + Test execution time :*****ms + [pass] testDependentlyTypedFunctions_testDouble [pass] testDependentlyTypedFunctions_thenReturn [pass] testMockMemberVariable @@ -16,6 +40,24 @@ Running Tests with Coverage [pass] testProvideErrorReturnValue [pass] testUserDefinedMockObject + [fail] testDefaultMockInvalidFieldName: + + error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") + callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 + callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 + callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 + callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 + + + [fail] testMockInvalidStream: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 + callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 + callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 + + [fail] testDefaultIncompatibleArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") @@ -34,15 +76,6 @@ Running Tests with Coverage callableName: testDefaultInvalidMemberReturnValue$lambda5$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 - [fail] testDefaultMockInvalidFieldName: - - error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") - callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 - callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 - callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 - callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 - - [fail] testDefaultMockInvalidReturnValue: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") @@ -70,15 +103,6 @@ Running Tests with Coverage callableName: testDefaultTooManyArgs$lambda2$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 6 - [fail] testMockInvalidStream: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 - callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 - callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 - - 9 passing 7 failing @@ -87,4 +111,4 @@ Running Tests with Coverage Generating Test Report object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt index ea886dbcbd4a..cc1bb608ced5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases1.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultIncompatibleArgs has failed. -**************************************************** Test execution time :*****ms @@ -31,4 +29,4 @@ testDefaultIncompatibleArgs has failed. Generating Test Report*****project-based-tests\object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt index 99ae6fbdfae9..3109f412b758 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases2.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultInvalidMemberReturnValue has failed. -**************************************************** Test execution time :*****ms @@ -31,4 +29,4 @@ testDefaultInvalidMemberReturnValue has failed. Generating Test Report*****project-based-tests\object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt index 090216c319ca..e5e7042e4e56 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases3.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultMockInvalidFieldName has failed. -**************************************************** Test execution time :*****ms @@ -31,4 +29,4 @@ testDefaultMockInvalidFieldName has failed. Generating Test Report*****project-based-tests\object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt index 13385b30ea43..b32d9661cbaf 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases4.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultMockInvalidReturnValue has failed. -**************************************************** Test execution time :*****ms @@ -31,4 +29,4 @@ testDefaultMockInvalidReturnValue has failed. Generating Test Report*****project-based-tests\object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt index bc7dfcc30fee..e6c1ba886705 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases5.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultMockWrongAction has failed. -**************************************************** Test execution time :*****ms @@ -31,4 +29,4 @@ testDefaultMockWrongAction has failed. Generating Test Report*****project-based-tests\object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt index 43bc87872e9a..ef30fc128ac9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases6.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testDefaultTooManyArgs has failed. -**************************************************** Test execution time :*****ms @@ -31,4 +29,4 @@ testDefaultTooManyArgs has failed. Generating Test Report*****project-based-tests\object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt index e288f8c9b7bf..9d25a830fd9c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NegativeCases7.txt @@ -7,9 +7,7 @@ Running Tests with Coverage object_mocking -**************************************************** testMockInvalidStream has failed. -**************************************************** Test execution time :*****ms @@ -31,4 +29,4 @@ testMockInvalidStream has failed. Generating Test Report*****project-based-tests\object-mocking-tests\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt index 6ddac17d7ad2..0c95be6a7562 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking_NonPublicField.txt @@ -5,9 +5,7 @@ Running Tests with Coverage non_public_field_mock -**************************************************** testNonPublicMemberFieldMock has failed. -**************************************************** Test execution time :*****ms @@ -29,4 +27,4 @@ testNonPublicMemberFieldMock has failed. Generating Test Report*****project-based-tests\non-public-field-mock\target\report\test_results.json -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt index efd47ae865c1..b61511a5b41c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionFlowTest-test-listener-shutdown.txt @@ -8,6 +8,9 @@ Calling init for 'moduleA listener' Calling init for 'current module listener' Calling start for 'moduleA listener' Calling start for 'current module listener' + + Test execution time :*****ms + [pass] main_test1 @@ -20,10 +23,13 @@ Calling stop for 'moduleA listener' moduleExecutionFlow.moduleA Calling init for 'moduleA listener' Calling start for 'moduleA listener' + + Test execution time :*****ms + [pass] test1 1 passing 0 failing 0 skipped -Calling stop for 'moduleA listener' +Calling stop for 'moduleA listener' \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt index 37923df2e9be..87656dd15c94 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt @@ -16,6 +16,12 @@ error: Error from start of moduleC at wso2.moduleExecutionInitStartFailure.moduleC.0.Listener:start(main.bal:32) moduleExecutionInitStartFailure.moduleB + +test2: has failed + + + Test execution time :*****ms + [pass] test1 [pass] test3 @@ -35,4 +41,4 @@ error: Error from start of moduleC 2 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt index 08ff88c19732..419a5894b033 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt @@ -8,6 +8,15 @@ Calling init for moduleA listener Calling init for default module listener Calling start for moduleA listener Calling start for default module listener + +main_negative_test1: has failed + + +main_negative_test2: has failed + + + Test execution time :*****ms + [pass] main_test1 [pass] main_test2 @@ -45,6 +54,12 @@ Calling stop for moduleA listener moduleGracefulStopTest.moduleA Calling init for moduleA listener Calling start for moduleA listener + +negative_test1: has failed + + + Test execution time :*****ms + [pass] test1 [fail] negative_test1: @@ -64,4 +79,4 @@ Calling start for moduleA listener 1 failing 0 skipped Calling stop for moduleA listener -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt index a7851bd08fb6..4b19adc087f0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt @@ -5,14 +5,10 @@ Running Tests with Coverage rerun_failed -**************************************************** -testFunctionFail1: has failed. -**************************************************** +testFunctionFail1: has failed -**************************************************** -testFunctionFail2: has failed. -**************************************************** +testFunctionFail2: has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt index 6f1f2eda4498..b97de420aca9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt @@ -5,14 +5,10 @@ Running Tests with Coverage rerun_failed -**************************************************** -testFunctionFail1: has failed. -**************************************************** +testFunctionFail1: has failed -**************************************************** -testFunctionFail2: has failed. -**************************************************** +testFunctionFail2: has failed Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt index 764dffb06515..af7992316543 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterEachFails.txt @@ -21,4 +21,4 @@ Running Tests 1 passing 0 failing 2 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt index b3f310542481..6cffd58b26cf 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -43,4 +43,4 @@ Running Tests 2 passing 0 failing 1 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt index 800ef7ae3ff8..ed9a844e3a99 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeEachFails.txt @@ -20,4 +20,4 @@ Running Tests 0 passing 0 failing 3 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt index f47a1b554c92..6492d54bc5f2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -42,4 +42,4 @@ Running Tests 1 passing 0 failing 2 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt index ea4014c6d7f7..007a069073c9 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt @@ -43,4 +43,4 @@ Running Tests 2 passing 0 failing 3 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt index 03fd2e9f6e8f..c966c747064b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeSuiteFails.txt @@ -20,4 +20,4 @@ Running Tests 0 passing 0 failing 3 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt index 86c508ee2815..eb480d0b4625 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenDependsOnFunctionFails.txt @@ -8,9 +8,7 @@ Running Tests dependson-skip-test.bal -**************************************************** test2 has failed. -**************************************************** Test execution time :*****ms @@ -29,4 +27,4 @@ test2 has failed. 2 passing 1 failing 2 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt index efb57dd57d84..f935e359e798 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt @@ -7,9 +7,7 @@ Running Tests foo -**************************************************** -testMain: has failed. -**************************************************** +testMain: has failed Test execution time :*****ms @@ -52,4 +50,4 @@ testMain: has failed. 2 passing 0 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt index b8fdcc037add..3752a560a25c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt @@ -6,9 +6,7 @@ Running Tests with Coverage foo -**************************************************** -testMain: has failed. -**************************************************** +testMain: has failed Test execution time :*****ms @@ -55,4 +53,4 @@ testMain: has failed. Generating Test Report*****project-based-tests\test-report-tests\target\report\test_results.json warning: Could not find the required HTML report tools for code coverage at \lib\tools\coverage\report.zip -error: there are test failures +error: there are test failures \ No newline at end of file From f2ae536aac2f82a0e4b014b17705838e9ad11a9e Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 14 Dec 2023 11:06:36 +0530 Subject: [PATCH 18/37] Restructure serial test --- .../testerina-core/src/main/ballerina/annotation_processor.bal | 2 +- .../modules/testerina-core/src/main/ballerina/register.bal | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index c9c2f45b4710..5eb8d0b93613 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -93,7 +93,7 @@ function processConfigAnnotation(string name, function f) returns boolean { testRegistry.addFunction(name = name, executableFunction = f, params = params, before = config.before, after = config.after, groups = config.groups, diagnostics = diagnostics, dependsOn = config.dependsOn, - enabled = enabled, dependsOnCount = config.dependsOn.length(), parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions), config = config); + enabled = enabled, dependsOnCount = config.dependsOn.length(), parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (testWorkers > 1)), config = config); return true; } return false; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index b66739cfd690..577ade8df01d 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -13,7 +13,6 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - import ballerina/lang.runtime; final TestRegistry testRegistry = new (); @@ -120,7 +119,7 @@ class ConcurrentExecutionManager { } function getSerialTest() returns TestFunction { - return self.serialTestExecutionList.remove(0); + return self.serialTestExecutionList.pop(); } From 7557e0d55af1bbed2654782a21756d01f7e65c05 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 14 Dec 2023 16:54:10 +0530 Subject: [PATCH 19/37] Add functions to create initial test array --- .../testerina-core/src/main/ballerina/execute.bal | 5 +++-- .../testerina-core/src/main/ballerina/external.bal | 1 - .../testerina-core/src/main/ballerina/register.bal | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index d6429c53a5f5..5cdc20f22d41 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -69,7 +69,7 @@ public function startSuite() returns int { function executeTests() returns error? { decimal startTime = currentTimeInMillis(); foreach TestFunction testFunction in testRegistry.getFunctions() { - _ = testFunction.parallelizable ? conMgr.addParallelTest(testFunction) : conMgr.addSerialTest(testFunction); + _ = testFunction.parallelizable ? conMgr.addInitialParallelTest(testFunction) : conMgr.addInitialSerialTest(testFunction); } while !conMgr.isExecutionDone() { @@ -148,6 +148,7 @@ function executeTest(TestFunction testFunction) returns error? { executeAfterEachFunctions(); executeAfterGroupFunctions(testFunction); + println(testFunction.name + " on execution"); if shouldSkipDependents { testFunction.dependents.forEach(function(TestFunction dependent) { lock { @@ -352,7 +353,7 @@ function executeAfterGroupFunctions(TestFunction testFunction) { TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { ExecutionError? err = executeFunctions(afterGroupFunctions, - getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); + getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); if err is ExecutionError { enableExit(); printExecutionError(err, "after test group function for the test"); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal index b6fc907f193f..8a7300adea7f 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal @@ -13,7 +13,6 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License.` ` - import ballerina/jballerina.java; handle outStreamObj = outStream(); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 577ade8df01d..7a3fb3349b35 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -66,6 +66,14 @@ class ConcurrentExecutionManager { self.serialTestExecutionList.push(testFunction); } + function addInitialParallelTest(TestFunction testFunction) { + self.parallelTestExecutionList.unshift(testFunction); + } + + function addInitialSerialTest(TestFunction testFunction) { + self.serialTestExecutionList.unshift(testFunction); + } + function addTestInExecution(TestFunction testFunction) { self.testsInExecution.push(testFunction); } @@ -124,6 +132,7 @@ class ConcurrentExecutionManager { } function waitUntilEmptyQueueFilled() { + self.populateExecutionQueues(); while self.parallelTestExecutionList.length() == 0 && self.serialTestExecutionList.length() == 0 { self.populateExecutionQueues(); runtime:sleep(0.0001); // sleep is added to yield the strand From 4c29dbb74d513e707995d83fce15690d07be644a Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Fri, 15 Dec 2023 10:15:52 +0530 Subject: [PATCH 20/37] Reverse the dependents to preserve existing behaviour --- .../modules/testerina-core/src/main/ballerina/execute.bal | 1 - .../modules/testerina-core/src/main/ballerina/register.bal | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 5cdc20f22d41..ccd84157f970 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -148,7 +148,6 @@ function executeTest(TestFunction testFunction) returns error? { executeAfterEachFunctions(); executeAfterGroupFunctions(testFunction); - println(testFunction.name + " on execution"); if shouldSkipDependents { testFunction.dependents.forEach(function(TestFunction dependent) { lock { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 7a3fb3349b35..9af3f824f5f5 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -151,7 +151,7 @@ class ConcurrentExecutionManager { isExecutionDone = testInProgress.isExecutionDone; } if isExecutionDone { - testInProgress.dependents.forEach(dependent => self.checkExecutionReadiness(dependent)); + testInProgress.dependents.reverse().forEach(dependent => self.checkExecutionReadiness(dependent)); _ = self.testsInExecution.remove(i); } else { i = i + 1; From 8be91a9a30fbe995116328fb652693a4c649ba9a Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Fri, 15 Dec 2023 01:12:04 -0800 Subject: [PATCH 21/37] Fix build failures --- .../unix/BasicCasesTest-testAnnotations.txt | 12 +- .../unix/BasicCasesTest-testDependsOn.txt | 8 +- .../BasicCasesTest-testIsolatedFunctions.txt | 2 +- .../GroupingTest-beforeGroupsAfterGroups2.txt | 26 +- .../unix/MockTest-testObjectMocking.txt | 48 ++-- ...ipTestsTestCase-testSkipWhenAfterFails.txt | 4 +- ...pTestsTestCase-testSkipWhenBeforeFails.txt | 4 +- .../BasicCasesTest-testAnnotations.txt | 64 ++--- .../windows/BasicCasesTest-testDependsOn.txt | 60 ++--- .../BasicCasesTest-testIsolatedFunctions.txt | 42 ++-- .../GroupingTest-beforeGroupsAfterGroups2.txt | 64 ++--- .../windows/MockTest-testObjectMocking.txt | 228 +++++++++--------- ...ipTestsTestCase-testSkipWhenAfterFails.txt | 92 +++---- ...pTestsTestCase-testSkipWhenBeforeFails.txt | 90 +++---- 14 files changed, 350 insertions(+), 394 deletions(-) diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt index 09f57e52f0aa..196f776e269f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testAnnotations.txt @@ -8,19 +8,19 @@ Running Tests with Coverage Test execution time :*****ms [pass] jsonDataProviderTest#0 - [pass] test1 - [pass] test2 - [pass] test3 - [pass] test4 - [pass] test5 - [pass] testBefore [pass] stringDataProviderTest#0 [pass] stringDataProviderTest#1 [pass] stringDataProviderTest#2 [pass] stringDataProviderTest2#0 + [pass] test1 [pass] testAfter + [pass] test2 [pass] testAfterAlone + [pass] test3 + [pass] test4 + [pass] test5 [pass] testDependsOn1 + [pass] testBefore 14 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt index 1cf19cef0acf..157431325eb7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testDependsOn.txt @@ -8,17 +8,17 @@ Running Tests with Coverage Test execution time :*****ms [pass] test1 - [pass] testAddingValues#0 - [pass] testAddingValues#1 - [pass] testAddingValues#2 [pass] test2 - [pass] testAddingValuesDependant [pass] test3 [pass] test4 [pass] testWithBefore1 [pass] testWithBefore2 [pass] testWithBefore3 [pass] testWithBefore4 + [pass] testAddingValues#0 + [pass] testAddingValues#1 + [pass] testAddingValues#2 + [pass] testAddingValuesDependant 12 passing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt index 970ff353a1a7..6dcd59fdbfc8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/BasicCasesTest-testIsolatedFunctions.txt @@ -7,8 +7,8 @@ Running Tests with Coverage Test execution time :*****ms - [pass] testFunc [pass] testCallingIsolatedFunction + [pass] testFunc [pass] testIsolatedTestFunction diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt index 6fadb5466f67..b2fed38010f5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-beforeGroupsAfterGroups2.txt @@ -9,35 +9,13 @@ Running Tests Test execution time :*****ms - [fail] afterSuiteFunc[after test suite function]: - error {ballerina/test:0}TestError ("Assertion Failed! - - expected: '123456787' - actual : '123467587' - - Diff : - - --- actual - +++ expected - - @@ -1,1 +1,1 @@ - - -123467587 - +123456787 - ") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 - callableName: afterSuiteFunc fileName: before-groups-after-groups-test2.bal lineNumber: 80 - callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 92 - [pass] testFunction [pass] testFunction2 - [pass] testFunction4 [pass] testFunction3 + [pass] testFunction4 [pass] testFunction5 5 passing 0 failing - 0 skipped -error: there are test failures \ No newline at end of file + 0 skipped \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt index e43e502b57ee..a12aa0625b95 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/MockTest-testObjectMocking.txt @@ -7,18 +7,15 @@ Running Tests with Coverage object_mocking -testDefaultMockInvalidFieldName has failed. - - -testMockInvalidStream has failed. - - testDefaultIncompatibleArgs has failed. testDefaultInvalidMemberReturnValue has failed. +testDefaultMockInvalidFieldName has failed. + + testDefaultMockInvalidReturnValue has failed. @@ -28,6 +25,9 @@ testDefaultMockWrongAction has failed. testDefaultTooManyArgs has failed. +testMockInvalidStream has failed. + + Test execution time :*****ms [pass] testDependentlyTypedFunctions_testDouble @@ -40,24 +40,6 @@ testDefaultTooManyArgs has failed. [pass] testProvideErrorReturnValue [pass] testUserDefinedMockObject - [fail] testDefaultMockInvalidFieldName: - - error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") - callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 - callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 - callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 - callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 - - - [fail] testMockInvalidStream: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 - callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 - callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 - - [fail] testDefaultIncompatibleArgs: error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") @@ -76,6 +58,15 @@ testDefaultTooManyArgs has failed. callableName: testDefaultInvalidMemberReturnValue$lambda5$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 + [fail] testDefaultMockInvalidFieldName: + + error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") + callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 + callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 + callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 + callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 + + [fail] testDefaultMockInvalidReturnValue: error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") @@ -103,6 +94,15 @@ testDefaultTooManyArgs has failed. callableName: testDefaultTooManyArgs$lambda2$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 6 + [fail] testMockInvalidStream: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 + callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 + callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 + + 9 passing 7 failing diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt index 1d438430ca46..7b344e2d8571 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -19,7 +19,7 @@ Running Tests error {ballerina/test:0}TestError ("Assertion Failed! expected: 'beforetestafterEachtestafterEach' - actual : 'beforetestafterEachtestafterEachafterEach' + actual : 'beforetestafterEachafterEachtestafterEach' Diff : @@ -28,7 +28,7 @@ Running Tests @@ -1,1 +1,1 @@ - -beforetestafterEachtestafterEachafterEach + -beforetestafterEachafterEachtestafterEach +beforetestafterEachtestafterEach ") callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt index 5a60efba0f7f..cba1fc4e65d5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -19,7 +19,7 @@ Running Tests error {ballerina/test:0}TestError ("Assertion Failed! expected: 'beforetest3afterEach' - actual : 'beforeafterEachtest3afterEachafterEach' + actual : 'beforeafterEachafterEachtest3afterEach' Diff : @@ -28,7 +28,7 @@ Running Tests @@ -1,1 +1,1 @@ - -beforeafterEachtest3afterEachafterEach + -beforeafterEachafterEachtest3afterEach +beforetest3afterEach ") callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt index 4564b4cc2821..65052b8028ec 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testAnnotations.txt @@ -1,32 +1,32 @@ -Compiling source - intg_tests/annotations:0.0.0 - -Running Tests with Coverage - - annotations - - Test execution time :*****ms - - [pass] jsonDataProviderTest#0 - [pass] test1 - [pass] test2 - [pass] test3 - [pass] test4 - [pass] test5 - [pass] testBefore - [pass] stringDataProviderTest#0 - [pass] stringDataProviderTest#1 - [pass] stringDataProviderTest#2 - [pass] stringDataProviderTest2#0 - [pass] testAfter - [pass] testAfterAlone - [pass] testDependsOn1 - - - 14 passing - 0 failing - 0 skipped - -Generating Test Report - annotations\target\report\test_results.json - \ No newline at end of file +Compiling source + intg_tests/annotations:0.0.0 + +Running Tests with Coverage + + annotations + + Test execution time :*****ms + + [pass] jsonDataProviderTest#0 + [pass] stringDataProviderTest#0 + [pass] stringDataProviderTest#1 + [pass] stringDataProviderTest#2 + [pass] stringDataProviderTest2#0 + [pass] test1 + [pass] testAfter + [pass] test2 + [pass] testAfterAlone + [pass] test3 + [pass] test4 + [pass] test5 + [pass] testDependsOn1 + [pass] testBefore + + + 14 passing + 0 failing + 0 skipped + +Generating Test Report + annotations\target\report\test_results.json + diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt index 7f183c35f4f9..8c8d6838de4b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testDependsOn.txt @@ -1,30 +1,30 @@ -Compiling source - intg_tests/depends_on:0.0.0 - -Running Tests with Coverage - - depends_on - - Test execution time :*****ms - - [pass] test1 - [pass] testAddingValues#0 - [pass] testAddingValues#1 - [pass] testAddingValues#2 - [pass] test2 - [pass] testAddingValuesDependant - [pass] test3 - [pass] test4 - [pass] testWithBefore1 - [pass] testWithBefore2 - [pass] testWithBefore3 - [pass] testWithBefore4 - - - 12 passing - 0 failing - 0 skipped - -Generating Test Report - depends-on\target\report\test_results.json - \ No newline at end of file +Compiling source + intg_tests/depends_on:0.0.0 + +Running Tests with Coverage + + depends_on + + Test execution time :*****ms + + [pass] test1 + [pass] test2 + [pass] test3 + [pass] test4 + [pass] testWithBefore1 + [pass] testWithBefore2 + [pass] testWithBefore3 + [pass] testWithBefore4 + [pass] testAddingValues#0 + [pass] testAddingValues#1 + [pass] testAddingValues#2 + [pass] testAddingValuesDependant + + + 12 passing + 0 failing + 0 skipped + +Generating Test Report + depends-on\target\report\test_results.json + diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt index 542072415010..575ddaf13b7b 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/BasicCasesTest-testIsolatedFunctions.txt @@ -1,21 +1,21 @@ -Compiling source - intg_tests/isolated_functions:0.0.0 - -Running Tests with Coverage - - isolated_functions - - Test execution time :*****ms - - [pass] testFunc - [pass] testCallingIsolatedFunction - [pass] testIsolatedTestFunction - - - 3 passing - 0 failing - 0 skipped - -Generating Test Report - isolated-functions\target\report\test_results.json - \ No newline at end of file +Compiling source + intg_tests/isolated_functions:0.0.0 + +Running Tests with Coverage + + isolated_functions + + Test execution time :*****ms + + [pass] testCallingIsolatedFunction + [pass] testFunc + [pass] testIsolatedTestFunction + + + 3 passing + 0 failing + 0 skipped + +Generating Test Report + isolated-functions\target\report\test_results.json + diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt index d0e32fb03a9d..bb5abf79a67f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-beforeGroupsAfterGroups2.txt @@ -1,43 +1,21 @@ -Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... -warning: ignoring --includes flag since code coverage is not enabled -Compiling source - before-groups-after-groups-test2.bal - -Running Tests - - before-groups-after-groups-test2.bal - - Test execution time :*****ms - - [fail] afterSuiteFunc[after test suite function]: - error {ballerina/test:0}TestError ("Assertion Failed! - - expected: '123456787' - actual : '123467587' - - Diff : - - --- actual - +++ expected - - @@ -1,1 +1,1 @@ - - -123467587 - +123456787 - ") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 - callableName: afterSuiteFunc fileName: before-groups-after-groups-test2.bal lineNumber: 80 - callableName: afterSuiteFunc$lambda9$ fileName: before-groups-after-groups-test2.bal lineNumber: 92 - - [pass] testFunction - [pass] testFunction2 - [pass] testFunction4 - [pass] testFunction3 - [pass] testFunction5 - - - 5 passing - 0 failing - 0 skipped -error: there are test failures \ No newline at end of file +Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... +warning: ignoring --includes flag since code coverage is not enabled +Compiling source + before-groups-after-groups-test2.bal + +Running Tests + + before-groups-after-groups-test2.bal + + Test execution time :*****ms + + [pass] testFunction + [pass] testFunction2 + [pass] testFunction3 + [pass] testFunction4 + [pass] testFunction5 + + + 5 passing + 0 failing + 0 skipped diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt index c4f3c2f6bb5e..57e57935403a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/MockTest-testObjectMocking.txt @@ -1,114 +1,114 @@ -Compiling source - intg_tests/object_mocking:0.0.0 -WARNING [modules\TestHttpClient\main.bal:(54:45,54:82)] this function should explicitly return a value -WARNING [main.bal:(47:5,47:47)] unused variable 'closeErr' - -Running Tests with Coverage - - object_mocking - -testDefaultMockInvalidFieldName has failed. - - -testMockInvalidStream has failed. - - -testDefaultIncompatibleArgs has failed. - - -testDefaultInvalidMemberReturnValue has failed. - - -testDefaultMockInvalidReturnValue has failed. - - -testDefaultMockWrongAction has failed. - - -testDefaultTooManyArgs has failed. - - - Test execution time :*****ms - - [pass] testDependentlyTypedFunctions_testDouble - [pass] testDependentlyTypedFunctions_thenReturn - [pass] testMockMemberVariable - [pass] testMockStreamSuccess - [pass] testProvideAReturnSequence - [pass] testProvideAReturnValue - [pass] testProvideAReturnValueBasedOnInput - [pass] testProvideErrorReturnValue - [pass] testUserDefinedMockObject - - [fail] testDefaultMockInvalidFieldName: - - error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") - callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 - callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 - callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 - callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 - - - [fail] testMockInvalidStream: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 - callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 - callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 - - - [fail] testDefaultIncompatibleArgs: - - error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") - callableName: validateArgumentsExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 346 - callableName: withArguments moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 132 - callableName: testDefaultIncompatibleArgs moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 56 - callableName: testDefaultIncompatibleArgs$lambda3$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 7 - - - [fail] testDefaultInvalidMemberReturnValue: - - error {ballerina/test:0}InvalidMemberFieldError ("return value provided does not match the type of 'url'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberVariableStub fileName: mock.bal lineNumber: 212 - callableName: testDefaultInvalidMemberReturnValue moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 74 - callableName: testDefaultInvalidMemberReturnValue$lambda5$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 - - - [fail] testDefaultMockInvalidReturnValue: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 - callableName: testDefaultMockInvalidReturnValue moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 35 - callableName: testDefaultMockInvalidReturnValue$lambda0$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 4 - - - [fail] testDefaultMockWrongAction: - - error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") - callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 - callableName: doNothing moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 180 - callableName: testDefaultMockWrongAction moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 42 - callableName: testDefaultMockWrongAction$lambda1$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 5 - - - [fail] testDefaultTooManyArgs: - - error {ballerina/test:0}FunctionSignatureMismatchError ("too many argument provided to mock the function 'get()'") - callableName: validateArgumentsExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 346 - callableName: withArguments moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 132 - callableName: testDefaultTooManyArgs moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 49 - callableName: testDefaultTooManyArgs$lambda2$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 6 - - - - 9 passing - 7 failing - 0 skipped - -Generating Test Report - object-mocking-tests\target\report\test_results.json - -error: there are test failures \ No newline at end of file +Compiling source + intg_tests/object_mocking:0.0.0 +WARNING [modules\TestHttpClient\main.bal:(54:45,54:82)] this function should explicitly return a value +WARNING [main.bal:(47:5,47:47)] unused variable 'closeErr' + +Running Tests with Coverage + + object_mocking + +testDefaultIncompatibleArgs has failed. + + +testDefaultInvalidMemberReturnValue has failed. + + +testDefaultMockInvalidFieldName has failed. + + +testDefaultMockInvalidReturnValue has failed. + + +testDefaultMockWrongAction has failed. + + +testDefaultTooManyArgs has failed. + + +testMockInvalidStream has failed. + + + Test execution time :*****ms + + [pass] testDependentlyTypedFunctions_testDouble + [pass] testDependentlyTypedFunctions_thenReturn + [pass] testMockMemberVariable + [pass] testMockStreamSuccess + [pass] testProvideAReturnSequence + [pass] testProvideAReturnValue + [pass] testProvideAReturnValueBasedOnInput + [pass] testProvideErrorReturnValue + [pass] testUserDefinedMockObject + + [fail] testDefaultIncompatibleArgs: + + error {ballerina/test:0}FunctionSignatureMismatchError ("incorrect type of argument provided at position '1' to mock the function 'get()'") + callableName: validateArgumentsExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 346 + callableName: withArguments moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 132 + callableName: testDefaultIncompatibleArgs moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 56 + callableName: testDefaultIncompatibleArgs$lambda3$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 7 + + + [fail] testDefaultInvalidMemberReturnValue: + + error {ballerina/test:0}InvalidMemberFieldError ("return value provided does not match the type of 'url'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberVariableStub fileName: mock.bal lineNumber: 212 + callableName: testDefaultInvalidMemberReturnValue moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 74 + callableName: testDefaultInvalidMemberReturnValue$lambda5$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 9 + + + [fail] testDefaultMockInvalidFieldName: + + error {ballerina/test:0}InvalidMemberFieldError ("invalid field name 'invalidField' provided") + callableName: validateFieldNameExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 337 + callableName: getMember moduleName: ballerina.test.0.MockObject fileName: mock.bal lineNumber: 95 + callableName: testDefaultMockInvalidFieldName moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 65 + callableName: testDefaultMockInvalidFieldName$lambda4$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 8 + + + [fail] testDefaultMockInvalidReturnValue: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 + callableName: testDefaultMockInvalidReturnValue moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 35 + callableName: testDefaultMockInvalidReturnValue$lambda0$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 4 + + + [fail] testDefaultMockWrongAction: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: doNothing moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 180 + callableName: testDefaultMockWrongAction moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 42 + callableName: testDefaultMockWrongAction$lambda1$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 5 + + + [fail] testDefaultTooManyArgs: + + error {ballerina/test:0}FunctionSignatureMismatchError ("too many argument provided to mock the function 'get()'") + callableName: validateArgumentsExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 346 + callableName: withArguments moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 132 + callableName: testDefaultTooManyArgs moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 49 + callableName: testDefaultTooManyArgs$lambda2$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 6 + + + [fail] testMockInvalidStream: + + error {ballerina/test:0}FunctionSignatureMismatchError ("return value provided does not match the return type of function 'get_stream()'") + callableName: thenReturnExt moduleName: ballerina.test.0 fileName: mock.bal lineNumber: 355 + callableName: thenReturn moduleName: ballerina.test.0.MemberFunctionStub fileName: mock.bal lineNumber: 148 + callableName: testMockInvalidStream moduleName: intg_tests.object_mocking$test.0.tests.main_error_test fileName: tests/main_error_test.bal lineNumber: 81 + callableName: testMockInvalidStream$lambda6$ moduleName: intg_tests.object_mocking$test.0.tests.test_execute-generated_*****lineNumber: 10 + + + + 9 passing + 7 failing + 0 skipped + +Generating Test Report + object-mocking-tests\target\report\test_results.json + +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt index 6cffd58b26cf..caf282f71d35 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -1,46 +1,46 @@ -Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... -warning: ignoring --includes flag since code coverage is not enabled -Compiling source - skip-when-after-fails.bal -WARNING [skip-when-after-fails.bal:(30:5,30:18)] unused variable 'i' - -Running Tests - - skip-when-after-fails.bal - [fail] [after test function for the test]: - error("{ballerina}DivisionByZero",message=" / by zero") - callableName: afterFunc fileName: skip-when-after-fails.bal lineNumber: 30 - callableName: afterFunc$lambda6$ fileName: skip-when-after-fails.bal lineNumber: 35 - - - Test execution time :*****ms - - [fail] afterSuite[after test suite function]: - error {ballerina/test:0}TestError ("Assertion Failed! - - expected: 'beforetestafterEachtestafterEach' - actual : 'beforetestafterEachtestafterEachafterEach' - - Diff : - - --- actual - +++ expected - - @@ -1,1 +1,1 @@ - - -beforetestafterEachtestafterEachafterEach - +beforetestafterEachtestafterEach - ") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 - callableName: afterSuite fileName: skip-when-after-fails.bal lineNumber: 57 - callableName: afterSuite$lambda5$ fileName: skip-when-after-fails.bal lineNumber: 65 - - [pass] test1 - [pass] test3 - - - 2 passing - 0 failing - 1 skipped -error: there are test failures \ No newline at end of file +Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... +warning: ignoring --includes flag since code coverage is not enabled +Compiling source + skip-when-after-fails.bal +WARNING [skip-when-after-fails.bal:(30:5,30:18)] unused variable 'i' + +Running Tests + + skip-when-after-fails.bal + [fail] [after test function for the test]: + error("{ballerina}DivisionByZero",message=" / by zero") + callableName: afterFunc fileName: skip-when-after-fails.bal lineNumber: 30 + callableName: afterFunc$lambda6$ fileName: skip-when-after-fails.bal lineNumber: 35 + + + Test execution time :*****ms + + [fail] afterSuite[after test suite function]: + error {ballerina/test:0}TestError ("Assertion Failed! + + expected: 'beforetestafterEachtestafterEach' + actual : 'beforetestafterEachafterEachtestafterEach' + + Diff : + + --- actual + +++ expected + + @@ -1,1 +1,1 @@ + + -beforetestafterEachafterEachtestafterEach + +beforetestafterEachtestafterEach + ") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 + callableName: afterSuite fileName: skip-when-after-fails.bal lineNumber: 57 + callableName: afterSuite$lambda5$ fileName: skip-when-after-fails.bal lineNumber: 65 + + [pass] test1 + [pass] test3 + + + 2 passing + 0 failing + 1 skipped +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt index 6492d54bc5f2..6aadf6d5847f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -1,45 +1,45 @@ -Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... -warning: ignoring --includes flag since code coverage is not enabled -Compiling source - skip-when-before-fails.bal -WARNING [skip-when-before-fails.bal:(28:5,28:18)] unused variable 'i' - -Running Tests - - skip-when-before-fails.bal - [fail] [before test function for the test]: - error("{ballerina}DivisionByZero",message=" / by zero") - callableName: before fileName: skip-when-before-fails.bal lineNumber: 28 - callableName: before$lambda6$ fileName: skip-when-before-fails.bal lineNumber: 32 - - - Test execution time :*****ms - - [fail] afterSuite[after test suite function]: - error {ballerina/test:0}TestError ("Assertion Failed! - - expected: 'beforetest3afterEach' - actual : 'beforeafterEachtest3afterEachafterEach' - - Diff : - - --- actual - +++ expected - - @@ -1,1 +1,1 @@ - - -beforeafterEachtest3afterEachafterEach - +beforetest3afterEach - ") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 - callableName: afterSuite fileName: skip-when-before-fails.bal lineNumber: 54 - callableName: afterSuite$lambda5$ fileName: skip-when-before-fails.bal lineNumber: 62 - - [pass] test3 - - - 1 passing - 0 failing - 2 skipped -error: there are test failures \ No newline at end of file +Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... +warning: ignoring --includes flag since code coverage is not enabled +Compiling source + skip-when-before-fails.bal +WARNING [skip-when-before-fails.bal:(28:5,28:18)] unused variable 'i' + +Running Tests + + skip-when-before-fails.bal + [fail] [before test function for the test]: + error("{ballerina}DivisionByZero",message=" / by zero") + callableName: before fileName: skip-when-before-fails.bal lineNumber: 28 + callableName: before$lambda6$ fileName: skip-when-before-fails.bal lineNumber: 32 + + + Test execution time :*****ms + + [fail] afterSuite[after test suite function]: + error {ballerina/test:0}TestError ("Assertion Failed! + + expected: 'beforetest3afterEach' + actual : 'beforeafterEachafterEachtest3afterEach' + + Diff : + + --- actual + +++ expected + + @@ -1,1 +1,1 @@ + + -beforeafterEachafterEachtest3afterEach + +beforetest3afterEach + ") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 + callableName: afterSuite fileName: skip-when-before-fails.bal lineNumber: 54 + callableName: afterSuite$lambda5$ fileName: skip-when-before-fails.bal lineNumber: 62 + + [pass] test3 + + + 1 passing + 0 failing + 2 skipped +error: there are test failures From 19ad2847cd14169a7116538ab4eeace9824808df Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Tue, 2 Jan 2024 12:46:52 +0530 Subject: [PATCH 22/37] Make ConcurrentExecutionManager isolated --- .../main/ballerina/annotation_processor.bal | 10 +- .../src/main/ballerina/execute.bal | 40 +-- .../src/main/ballerina/filter.bal | 2 +- .../src/main/ballerina/register.bal | 242 ++++++++++++++---- 4 files changed, 214 insertions(+), 80 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 5eb8d0b93613..bdd315ff1eca 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -53,7 +53,7 @@ function processConfigAnnotation(string name, function f) returns boolean { boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated; string[] reasonToSerialExecution = []; - DataProviderReturnType? params = (); + DataProviderReturnType? & readonly params = (); error? diagnostics = (); if config.dataProvider != () { var providerFn = config.dataProvider; @@ -63,7 +63,7 @@ function processConfigAnnotation(string name, function f) returns boolean { isTestFunctionParamSafe = isFunctionParamConcurrencySafe(f); isSatisfiedParallelizableConditions = isTestFunctionIsolated && isDataProviderIsolated && isTestFunctionParamSafe; DataProviderReturnType providerOutput = providerFn(); - params = providerOutput; + params = providerOutput; } else { diagnostics = error("Failed to execute the data provider"); } @@ -92,9 +92,9 @@ function processConfigAnnotation(string name, function f) returns boolean { config.groups.forEach('group => groupStatusRegistry.incrementTotalTest('group, enabled)); testRegistry.addFunction(name = name, executableFunction = f, params = params, before = config.before, - after = config.after, groups = config.groups, diagnostics = diagnostics, dependsOn = config.dependsOn, - enabled = enabled, dependsOnCount = config.dependsOn.length(), parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (testWorkers > 1)), config = config); - return true; + after = config.after, groups = config.groups.cloneReadOnly(), diagnostics = diagnostics, dependsOn = config.dependsOn.cloneReadOnly(), + parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (testWorkers > 1)), config = config.cloneReadOnly()); + conMgr.createTestFunctionMetaData(functionName = name, dependsOnCount = config.dependsOn.length(), enabled = enabled); } return false; } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index ccd84157f970..35cc29ab7533 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -19,7 +19,7 @@ import ballerina/lang.runtime; isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; isolated int exitCode = 0; -ConcurrentExecutionManager conMgr = new (1); +isolated final ConcurrentExecutionManager conMgr = new (); public function startSuite() returns int { // exit if setTestOptions has failed @@ -105,9 +105,9 @@ function executeTests() returns error? { } function executeTest(TestFunction testFunction) returns error? { - if !testFunction.enabled { + if !conMgr.isEnabled(testFunction.name) { lock { - testFunction.isExecutionDone = true; + conMgr.setExecutionDone(testFunction.name); } conMgr.releaseWorker(); return; @@ -121,7 +121,7 @@ function executeTest(TestFunction testFunction) returns error? { } enableExit(); lock { - testFunction.isExecutionDone = true; + conMgr.setExecutionDone(testFunction.name); } conMgr.releaseWorker(); return; @@ -131,7 +131,7 @@ function executeTest(TestFunction testFunction) returns error? { executeBeforeEachFunctions(); boolean shouldSkipDependents = false; - if !testFunction.skip && !getShouldSkip() { + if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { if (isDataDrivenTest(testFunction)) { check executeDataDrivenTestSet(testFunction); } else { @@ -149,14 +149,14 @@ function executeTest(TestFunction testFunction) returns error? { executeAfterGroupFunctions(testFunction); if shouldSkipDependents { - testFunction.dependents.forEach(function(TestFunction dependent) { + conMgr.getDependents(testFunction.name).forEach(function(TestFunction dependent) { lock { - dependent.skip = true; + conMgr.setSkip(dependent.name); } }); } lock { - testFunction.isExecutionDone = true; + conMgr.setExecutionDone(testFunction.name); } conMgr.releaseWorker(); } @@ -245,7 +245,7 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { boolean failed = false; boolean beforeFailed = executeBeforeFunction(testFunction); if (beforeFailed) { - testFunction.skip = true; + conMgr.setSkip(testFunction.name); lock { reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); } @@ -308,7 +308,7 @@ function executeAfterEachFunctions() { function executeBeforeFunction(TestFunction testFunction) returns boolean { boolean failed = false; - if testFunction.before is function && !getShouldSkip() && !testFunction.skip { + if testFunction.before is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { ExecutionError? err = executeFunction(testFunction.before); if err is ExecutionError { enableExit(); @@ -321,7 +321,7 @@ function executeBeforeFunction(TestFunction testFunction) returns boolean { function executeAfterFunction(TestFunction testFunction) returns boolean { boolean failed = false; - if testFunction.after is function && !getShouldSkip() && !testFunction.skip { + if testFunction.after is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { ExecutionError? err = executeFunction(testFunction.after); if err is ExecutionError { enableExit(); @@ -338,7 +338,7 @@ function executeBeforeGroupFunctions(TestFunction testFunction) { if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { ExecutionError? err = executeFunctions(beforeGroupFunctions, getShouldSkip()); if err is ExecutionError { - testFunction.skip = true; + conMgr.setSkip(testFunction.name); groupStatusRegistry.setSkipAfterGroup('group); enableExit(); printExecutionError(err, "before test group function for the test"); @@ -366,7 +366,7 @@ function skipDataDrivenTest(TestFunction testFunction, string suffix, TestType t if (!hasFilteredTests) { return false; } - TestFunction[] dependents = testFunction.dependents; + TestFunction[] dependents = conMgr.getDependents(functionName); // if a dependent in a below level is enabled, this test should run if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { @@ -484,7 +484,7 @@ function orderTests() returns error? { string[] descendants = []; foreach TestFunction testFunction in testRegistry.getDependentFunctions() { - if !testFunction.visited && testFunction.enabled { + if !conMgr.isVisited(testFunction.name) && conMgr.isEnabled(testFunction.name) { check restructureTest(testFunction, descendants); } } @@ -506,7 +506,7 @@ function restructureTest(TestFunction testFunction, string[] descendants) return return error(errMsg); } - dependsOnTestFunction.dependents.push(testFunction); + conMgr.addDependent(dependsOnTestFunction.name, testFunction); // Contains cyclic dependencies int? startIndex = descendants.indexOf(dependsOnTestFunction.name); @@ -514,13 +514,13 @@ function restructureTest(TestFunction testFunction, string[] descendants) return string[] newCycle = descendants.slice(startIndex); newCycle.push(dependsOnTestFunction.name); return error("Cyclic test dependencies detected: " + string:'join(" -> ", ...newCycle)); - } else if !dependsOnTestFunction.visited { + } else if !conMgr.isVisited(dependsOnTestFunction.name) { check restructureTest(dependsOnTestFunction, descendants); } } - testFunction.enabled = true; - testFunction.visited = true; + conMgr.setEnabled(testFunction.name); + conMgr.setVisited(testFunction.name); _ = descendants.pop(); } @@ -540,10 +540,10 @@ function nestedEnabledDependentsAvailable(TestFunction[] dependents) returns boo } TestFunction[] queue = []; foreach TestFunction dependent in dependents { - if (dependent.enabled) { + if (conMgr.isEnabled(dependent.name)) { return true; } - dependent.dependents.forEach((superDependent) => queue.push(superDependent)); + conMgr.getDependents(dependent.name).forEach((superDependent) => queue.push(superDependent)); } return nestedEnabledDependentsAvailable(queue); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index e0b25111436e..fb6df02c739a 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -41,7 +41,7 @@ public function setTestOptions(string inTargetPath, string inPackageName, string boolean codeCoverage = parseBooleanInput(inCoverage, "code-coverage"); listGroups = parseBooleanInput(inListGroups, "list-groups"); testWorkers = parseIntegerInput(inTestWorkers, "testWorkers"); - conMgr = new (testWorkers); + conMgr.setIntialWorkers(testWorkers); if rerunFailed { error? err = parseRerunJson(); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 9af3f824f5f5..bdae52b7df65 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -28,74 +28,189 @@ final GroupStatusRegistry groupStatusRegistry = new (); type TestFunction record {| string name; function executableFunction; - boolean enabled = true; DataProviderReturnType? params = (); function? before = (); function? after = (); boolean alwaysRun = false; string[] groups = []; - boolean skip = false; error? diagnostics = (); function[] dependsOn = []; + boolean parallelizable = true; + TestConfig? config = (); +|} & readonly; + +type TestFunctionMetaData record {| + boolean enabled = true; + boolean skip = false; int dependsOnCount = 0; TestFunction[] dependents = []; boolean visited = false; boolean isInExecutionQueue = false; - boolean parallelizable = true; - TestConfig? config = (); boolean isExecutionDone = false; |}; -class ConcurrentExecutionManager { +isolated class ConcurrentExecutionManager { private TestFunction[] parallelTestExecutionList = []; private TestFunction[] serialTestExecutionList = []; private TestFunction[] testsInExecution = []; private int unAllocatedTestWorkers = 1; - private final int intialWorkers; + private int intialWorkers = 1; + private final map testMetaData = {}; + + isolated function createTestFunctionMetaData(string functionName, *TestFunctionMetaData intialMetaData) { + lock { + self.testMetaData[functionName] = intialMetaData.clone(); + } + } - function init(int workers) { - self.intialWorkers = workers; - self.unAllocatedTestWorkers = workers; + isolated function setExecutionDone(string functionName) { + lock { + self.testMetaData[functionName].isExecutionDone = true; + } } - function addParallelTest(TestFunction testFunction) { - self.parallelTestExecutionList.push(testFunction); + isolated function setDisabled(string functionName) { + lock { + self.testMetaData[functionName].enabled = false; + } + } + + isolated function setEnabled(string functionName) { + lock { + self.testMetaData[functionName].enabled = true; + } + } + + isolated function isEnabled(string functionName) returns boolean { + lock { + TestFunctionMetaData? unionResult = self.testMetaData[functionName]; + if unionResult is TestFunctionMetaData { + return unionResult.enabled; + } else { + return false; + } + + } } - function addSerialTest(TestFunction testFunction) { - self.serialTestExecutionList.push(testFunction); + isolated function setVisited(string functionName) { + lock { + self.testMetaData[functionName].visited = true; + } } - function addInitialParallelTest(TestFunction testFunction) { - self.parallelTestExecutionList.unshift(testFunction); + isolated function isVisited(string functionName) returns boolean { + lock { + TestFunctionMetaData? unionResult = self.testMetaData[functionName]; + if unionResult is TestFunctionMetaData { + return unionResult.visited; + } else { + return false; + } + + } } - function addInitialSerialTest(TestFunction testFunction) { - self.serialTestExecutionList.unshift(testFunction); + isolated function isSkip(string functionName) returns boolean { + lock { + TestFunctionMetaData? unionResult = self.testMetaData[functionName]; + if unionResult is TestFunctionMetaData { + return unionResult.skip; + } else { + return false; + } + + } } - function addTestInExecution(TestFunction testFunction) { - self.testsInExecution.push(testFunction); + isolated function setSkip(string functionName) { + lock { + self.testMetaData[functionName].skip = true; + } } - function getConfiguredWorkers() returns int { - return self.intialWorkers; + isolated function addDependent(string functionName, TestFunction dependent) { + lock { + TestFunctionMetaData? unionResult = self.testMetaData[functionName]; + if unionResult is TestFunctionMetaData { + unionResult.dependents.push(dependent); + } + } } - function getSerialQueueLength() returns int { - return self.serialTestExecutionList.length(); + isolated function getDependents(string functionName) returns TestFunction[] { + lock { + TestFunctionMetaData? unionResult = self.testMetaData[functionName]; + if unionResult is TestFunctionMetaData { + return unionResult.dependents.clone(); + } else { + return []; + } + } } - function getParallelQueueLength() returns int { - return self.parallelTestExecutionList.length(); + isolated function setIntialWorkers(int workers) { + lock { + self.intialWorkers = workers; + self.unAllocatedTestWorkers = workers; + } } - function isExecutionDone() returns boolean { - return self.parallelTestExecutionList.length() == 0 && - self.getAvailableWorkers() == testWorkers && + isolated function addParallelTest(TestFunction testFunction) { + lock { + self.parallelTestExecutionList.push(testFunction); + } + } + + isolated function addSerialTest(TestFunction testFunction) { + lock { + self.serialTestExecutionList.push(testFunction); + } + } + + isolated function addInitialParallelTest(TestFunction testFunction) { + lock { + self.parallelTestExecutionList.unshift(testFunction); + } + } + + isolated function addInitialSerialTest(TestFunction testFunction) { + lock { + self.serialTestExecutionList.unshift(testFunction); + } + } + + isolated function addTestInExecution(TestFunction testFunction) { + lock { + self.testsInExecution.push(testFunction); + } + } + + isolated function getConfiguredWorkers() returns int { + lock { + return self.intialWorkers; + } + } + + isolated function getSerialQueueLength() returns int { + lock { + return self.serialTestExecutionList.length(); + } + } + + isolated function getParallelQueueLength() returns int { + lock { + return self.parallelTestExecutionList.length(); + } + } + + isolated function isExecutionDone() returns boolean { + lock { + return self.parallelTestExecutionList.length() == 0 && + self.getAvailableWorkers() == self.intialWorkers && self.serialTestExecutionList.length() == 0 && self.testsInExecution.length() == 0; - + } } isolated function allocateWorker() { @@ -122,18 +237,27 @@ class ConcurrentExecutionManager { } } - function getParallelTest() returns TestFunction { - return self.parallelTestExecutionList.remove(0); + isolated function getParallelTest() returns TestFunction { + lock { + return self.parallelTestExecutionList.remove(0); + } } - function getSerialTest() returns TestFunction { - return self.serialTestExecutionList.pop(); - + isolated function getSerialTest() returns TestFunction { + lock { + return self.serialTestExecutionList.pop(); + } } - function waitUntilEmptyQueueFilled() { + isolated function waitUntilEmptyQueueFilled() { + int parallelQueueLength = 0; + int serialQueueLength = 0; self.populateExecutionQueues(); - while self.parallelTestExecutionList.length() == 0 && self.serialTestExecutionList.length() == 0 { + lock { + parallelQueueLength = self.parallelTestExecutionList.length(); + serialQueueLength = self.serialTestExecutionList.length(); + } + while parallelQueueLength == 0 && serialQueueLength == 0 { self.populateExecutionQueues(); runtime:sleep(0.0001); // sleep is added to yield the strand if self.isExecutionDone() { @@ -142,28 +266,38 @@ class ConcurrentExecutionManager { } } - function populateExecutionQueues() { - int i = 0; - boolean isExecutionDone = false; - while i < self.testsInExecution.length() { - TestFunction testInProgress = self.testsInExecution[i]; - lock { - isExecutionDone = testInProgress.isExecutionDone; - } - if isExecutionDone { - testInProgress.dependents.reverse().forEach(dependent => self.checkExecutionReadiness(dependent)); - _ = self.testsInExecution.remove(i); - } else { - i = i + 1; + isolated function populateExecutionQueues() { + lock { + int i = 0; + boolean isExecutionDone = false; + while i < self.testsInExecution.length() { + TestFunction testInProgress = self.testsInExecution[i]; + TestFunctionMetaData? inProgressTestMetaData = self.testMetaData[testInProgress.name]; + if inProgressTestMetaData == () { + return; + } + isExecutionDone = inProgressTestMetaData.isExecutionDone; + if isExecutionDone { + inProgressTestMetaData.dependents.reverse().forEach(dependent => self.checkExecutionReadiness(dependent)); + _ = self.testsInExecution.remove(i); + } else { + i = i + 1; + } } } } - private function checkExecutionReadiness(TestFunction testFunction) { - testFunction.dependsOnCount -= 1; - if testFunction.dependsOnCount == 0 && testFunction.isInExecutionQueue != true { - testFunction.isInExecutionQueue = true; - _ = testFunction.parallelizable ? self.parallelTestExecutionList.push(testFunction) : self.serialTestExecutionList.push(testFunction); + private isolated function checkExecutionReadiness(TestFunction testFunction) { + lock { + TestFunctionMetaData? unionResult = self.testMetaData[testFunction.name]; + if unionResult is TestFunctionMetaData { + unionResult.dependsOnCount -= 1; + if unionResult.dependsOnCount == 0 && unionResult.isInExecutionQueue != true { + unionResult.isInExecutionQueue = true; + _ = testFunction.parallelizable ? self.parallelTestExecutionList.push(testFunction) : self.serialTestExecutionList.push(testFunction); + } + + } } } From 0c827c4b34c45a12cda353265b51ae3761b3026d Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 4 Jan 2024 13:33:06 +0530 Subject: [PATCH 23/37] Create isolated functions for execution --- .../main/ballerina/annotation_processor.bal | 16 +- .../src/main/ballerina/execute.bal | 384 +----------------- .../src/main/ballerina/external.bal | 12 +- .../src/main/ballerina/filter.bal | 53 ++- .../src/main/ballerina/isolateExecuter.bal | 334 +++++++++++++++ .../src/main/ballerina/nonIsolateExecuter.bal | 321 +++++++++++++++ .../src/main/ballerina/register.bal | 213 ++++++++-- .../src/main/ballerina/report.bal | 150 +++++-- 8 files changed, 1009 insertions(+), 474 deletions(-) create mode 100644 misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal create mode 100644 misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index bdd315ff1eca..734b34421b51 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -83,7 +83,7 @@ function processConfigAnnotation(string name, function f) returns boolean { } // If the test function is not parallelizable, then print the reason for serial execution. - if !isSatisfiedParallelizableConditions && !config.serialExecution && (testWorkers > 1) { + if !isSatisfiedParallelizableConditions && !config.serialExecution && (conMgr.getConfiguredWorkers() > 1) { println("WARNING: Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); } @@ -93,7 +93,7 @@ function processConfigAnnotation(string name, function f) returns boolean { testRegistry.addFunction(name = name, executableFunction = f, params = params, before = config.before, after = config.after, groups = config.groups.cloneReadOnly(), diagnostics = diagnostics, dependsOn = config.dependsOn.cloneReadOnly(), - parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (testWorkers > 1)), config = config.cloneReadOnly()); + parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (conMgr.getConfiguredWorkers() > 1)), config = config.cloneReadOnly()); conMgr.createTestFunctionMetaData(functionName = name, dependsOnCount = config.dependsOn.length(), enabled = enabled); } return false; @@ -164,12 +164,12 @@ function hasGroup(string[] groups, string[] filter) returns boolean { return false; } -function hasTest(string name) returns boolean { - if hasFilteredTests { +isolated function hasTest(string name) returns boolean { + if testOptions.getHasFilteredTests() { string testName = name; - int? testIndex = filterTests.indexOf(testName); + int? testIndex = testOptions.getFilterTestIndex(testName); if testIndex == () { - foreach string filter in filterTests { + foreach string filter in testOptions.getFilterTests() { if (filter.includes(WILDCARD)) { boolean|error wildCardMatch = matchWildcard(testName, filter); if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { @@ -186,7 +186,7 @@ function hasTest(string name) returns boolean { return true; } -function matchModuleName(string testName) returns boolean { - string? filterModule = filterTestModules[testName]; +isolated function matchModuleName(string testName) returns boolean { + string? filterModule = testOptions.getFilterTestModule(testName); return filterModule == () ? true : filterModule == getFullModuleName(); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 35cc29ab7533..2350411bf1af 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -77,7 +77,7 @@ function executeTests() returns error? { if conMgr.getAvailableWorkers() != 0 { conMgr.waitUntilEmptyQueueFilled(); - if conMgr.getSerialQueueLength() != 0 && conMgr.getAvailableWorkers() == testWorkers { + if conMgr.getSerialQueueLength() != 0 && conMgr.getAvailableWorkers() == conMgr.getConfiguredWorkers() { TestFunction testFunction = conMgr.getSerialTest(); conMgr.addTestInExecution(testFunction); conMgr.allocateWorker(); @@ -104,172 +104,6 @@ function executeTests() returns error? { println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n"); } -function executeTest(TestFunction testFunction) returns error? { - if !conMgr.isEnabled(testFunction.name) { - lock { - conMgr.setExecutionDone(testFunction.name); - } - conMgr.releaseWorker(); - return; - } - - error? diagnoseError = testFunction.diagnostics; - if diagnoseError is error { - lock { - reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); - println("\n" + testFunction.name + " has failed.\n"); - } - enableExit(); - lock { - conMgr.setExecutionDone(testFunction.name); - } - conMgr.releaseWorker(); - return; - } - - executeBeforeGroupFunctions(testFunction); - executeBeforeEachFunctions(); - - boolean shouldSkipDependents = false; - if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { - if (isDataDrivenTest(testFunction)) { - check executeDataDrivenTestSet(testFunction); - } else { - shouldSkipDependents = executeNonDataDrivenTest(testFunction); - } - } else { - lock { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); - } - shouldSkipDependents = true; - } - - testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); - executeAfterEachFunctions(); - executeAfterGroupFunctions(testFunction); - - if shouldSkipDependents { - conMgr.getDependents(testFunction.name).forEach(function(TestFunction dependent) { - lock { - conMgr.setSkip(dependent.name); - } - }); - } - lock { - conMgr.setExecutionDone(testFunction.name); - } - conMgr.releaseWorker(); -} - -function executeDataDrivenTestSet(TestFunction testFunction) returns error? { - DataProviderReturnType? params = testFunction.params; - string[] keys = []; - AnyOrError[][] values = []; - TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; - if params is map { - foreach [string, AnyOrError[]] entry in params.entries() { - keys.push(entry[0]); - values.push(entry[1]); - } - } else if params is AnyOrError[][] { - testType = DATA_DRIVEN_TUPLE_OF_TUPLE; - int i = 0; - foreach AnyOrError[] entry in params { - keys.push(i.toString()); - values.push(entry); - i += 1; - } - } - - boolean isIntialJob = true; - - while keys.length() != 0 { - - if isIntialJob || conMgr.getAvailableWorkers() > 0 { - string key = keys.remove(0); - AnyOrError[] value = values.remove(0); - - if !isIntialJob { - conMgr.allocateWorker(); - } - - future<()> serialWaiter = start prepareDataDrivenTest(testFunction, key, value, testType); - - if !testFunction.parallelizable { - any _ = check wait serialWaiter; - } - - } - - isIntialJob = false; - conMgr.waitForWorkers(); - } - - conMgr.waitForWorkers(); - - if !isIntialJob { - conMgr.allocateWorker(); - } -} - -function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { - boolean beforeFailed = executeBeforeFunction(testFunction); - if (beforeFailed) { - lock { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); - } - } else { - executeDataDrivenTest(testFunction, key, testType, value); - var _ = executeAfterFunction(testFunction); - } - conMgr.releaseWorker(); -} - -function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { - if (skipDataDrivenTest(testFunction, suffix, testType)) { - return; - } - - ExecutionError|boolean err = executeTestFunction(testFunction, suffix, testType, params); - if err is ExecutionError { - lock { - reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name - + "]\n" + getErrorMessage(err), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); - enableExit(); - } - } -} - -function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { - boolean failed = false; - boolean beforeFailed = executeBeforeFunction(testFunction); - if (beforeFailed) { - conMgr.setSkip(testFunction.name); - lock { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); - } - return true; - } - ExecutionError|boolean output = executeTestFunction(testFunction, "", GENERAL_TEST); - if output is ExecutionError { - failed = true; - lock { - reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); - println("\n" + testFunction.name + " has failed.\n"); - } - } - - else if output { - failed = true; - } - boolean afterFailed = executeAfterFunction(testFunction); - if (afterFailed) { - return true; - } - return failed; -} - function executeBeforeSuiteFunctions() { ExecutionError? err = executeFunctions(beforeSuiteRegistry.getFunctions()); if err is ExecutionError { @@ -288,198 +122,6 @@ function executeAfterSuiteFunctions() { } } -function executeBeforeEachFunctions() { - ExecutionError? err = executeFunctions(beforeEachRegistry.getFunctions(), getShouldSkip()); - if err is ExecutionError { - enableShouldSkip(); - enableExit(); - printExecutionError(err, "before each test function for the test"); - } -} - -function executeAfterEachFunctions() { - ExecutionError? err = executeFunctions(afterEachRegistry.getFunctions(), getShouldSkip()); - if err is ExecutionError { - enableShouldSkip(); - enableExit(); - printExecutionError(err, "after each test function for the test"); - } -} - -function executeBeforeFunction(TestFunction testFunction) returns boolean { - boolean failed = false; - if testFunction.before is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { - ExecutionError? err = executeFunction(testFunction.before); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "before test function for the test"); - failed = true; - } - } - return failed; -} - -function executeAfterFunction(TestFunction testFunction) returns boolean { - boolean failed = false; - if testFunction.after is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { - ExecutionError? err = executeFunction(testFunction.after); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "after test function for the test"); - failed = true; - } - } - return failed; -} - -function executeBeforeGroupFunctions(TestFunction testFunction) { - foreach string 'group in testFunction.groups { - TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); - if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { - ExecutionError? err = executeFunctions(beforeGroupFunctions, getShouldSkip()); - if err is ExecutionError { - conMgr.setSkip(testFunction.name); - groupStatusRegistry.setSkipAfterGroup('group); - enableExit(); - printExecutionError(err, "before test group function for the test"); - } - } - } -} - -function executeAfterGroupFunctions(TestFunction testFunction) { - foreach string 'group in testFunction.groups { - TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); - if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { - ExecutionError? err = executeFunctions(afterGroupFunctions, - getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "after test group function for the test"); - } - } - } -} - -function skipDataDrivenTest(TestFunction testFunction, string suffix, TestType testType) returns boolean { - string functionName = testFunction.name; - if (!hasFilteredTests) { - return false; - } - TestFunction[] dependents = conMgr.getDependents(functionName); - - // if a dependent in a below level is enabled, this test should run - if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { - return false; - } - string functionKey = functionName; - - // check if prefix matches directly - boolean prefixMatch = filterSubTests.hasKey(functionName); - - // if prefix matches to a wildcard - if (!prefixMatch && hasTest(functionName)) { - - // get the matching wildcard - prefixMatch = true; - foreach string filter in filterTests { - if (filter.includes(WILDCARD)) { - boolean|error wildCardMatch = matchWildcard(functionKey, filter); - if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { - functionKey = filter; - break; - } - } - } - } - - // check if no filterSubTests found for a given prefix - boolean suffixMatch = !filterSubTests.hasKey(functionKey); - - // if a subtest is found specified - if (!suffixMatch) { - string[] subTests = filterSubTests.get(functionKey); - foreach string subFilter in subTests { - - string updatedSubFilter = subFilter; - if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { - if (subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE)) { - updatedSubFilter = subFilter.substring(1, subFilter.length() - 1); - } else { - continue; - } - } - string|error decodedSubFilter = escapeSpecialCharacters(updatedSubFilter); - updatedSubFilter = decodedSubFilter is string ? decodedSubFilter : updatedSubFilter; - string|error decodedSuffix = escapeSpecialCharacters(suffix); - string updatedSuffix = decodedSuffix is string ? decodedSuffix : suffix; - - boolean wildCardMatchBoolean = false; - if (updatedSubFilter.includes(WILDCARD)) { - boolean|error wildCardMatch = matchWildcard(updatedSuffix, updatedSubFilter); - wildCardMatchBoolean = wildCardMatch is boolean && wildCardMatch; - } - if ((updatedSubFilter == updatedSuffix) || wildCardMatchBoolean) { - suffixMatch = true; - break; - } - } - } - - // do not skip iff both matches - return !(prefixMatch && suffixMatch); -} - -function printExecutionError(ExecutionError err, string functionSuffix) - => println("\t[fail] " + err.detail().functionName + "[" + functionSuffix + "]" + ":\n\t " + formatFailedError(err.message(), 2)); - -function executeFunctions(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { - foreach TestFunction testFunction in testFunctions { - if !skip || testFunction.alwaysRun { - check executeFunction(testFunction); - } - } -} - -function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { - any|error output = params == () ? trap function:call(testFunction.executableFunction) - : trap function:call(testFunction.executableFunction, ...params); - if output is TestError { - enableExit(); - lock { - reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed\n"); - } - return true; - } else if output is any { - lock { - reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); - } - return false; - } else { - enableExit(); - return error(getErrorMessage(output), functionName = testFunction.name); - } -} - -function executeFunction(TestFunction|function testFunction) returns ExecutionError? { - any|error output = trap function:call(testFunction is function ? testFunction : testFunction.executableFunction); - if output is error { - enableExit(); - return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); - } -} - -function getErrorMessage(error err) returns string { - string message = err.toBalString(); - - string accumulatedTrace = ""; - foreach langError:StackFrame stackFrame in err.stackTrace() { - accumulatedTrace = accumulatedTrace + "\t" + stackFrame.toString() + "\n"; - } - return message + "\n" + accumulatedTrace; -} - function orderTests() returns error? { string[] descendants = []; @@ -524,7 +166,20 @@ function restructureTest(TestFunction testFunction, string[] descendants) return _ = descendants.pop(); } -function getTestType(TestFunction testFunction) returns TestType { +isolated function printExecutionError(ExecutionError err, string functionSuffix) + => println("\t[fail] " + err.detail().functionName + "[" + functionSuffix + "]" + ":\n\t " + formatFailedError(err.message(), 2)); + +isolated function getErrorMessage(error err) returns string { + string message = err.toBalString(); + + string accumulatedTrace = ""; + foreach langError:StackFrame stackFrame in err.stackTrace() { + accumulatedTrace = accumulatedTrace + "\t" + stackFrame.toString() + "\n"; + } + return message + "\n" + accumulatedTrace; +} + +isolated function getTestType(TestFunction testFunction) returns TestType { DataProviderReturnType? params = testFunction.params; if (params is map) { return DATA_DRIVEN_MAP_OF_TUPLE; @@ -534,7 +189,7 @@ function getTestType(TestFunction testFunction) returns TestType { return GENERAL_TEST; } -function nestedEnabledDependentsAvailable(TestFunction[] dependents) returns boolean { +isolated function nestedEnabledDependentsAvailable(TestFunction[] dependents) returns boolean { if (dependents.length() == 0) { return false; } @@ -543,11 +198,16 @@ function nestedEnabledDependentsAvailable(TestFunction[] dependents) returns boo if (conMgr.isEnabled(dependent.name)) { return true; } - conMgr.getDependents(dependent.name).forEach((superDependent) => queue.push(superDependent)); + foreach TestFunction superDependent in conMgr.getDependents(dependent.name) { + queue.push(superDependent); + } } return nestedEnabledDependentsAvailable(queue); } +isolated function isDataDrivenTest(TestFunction testFunction) returns boolean => + testFunction.params is map || testFunction.params is AnyOrError[][]; + isolated function enableShouldSkip() { lock { shouldSkip = true; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal index 8a7300adea7f..a686e0afff45 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal @@ -15,7 +15,7 @@ // under the License.` ` import ballerina/jballerina.java; -handle outStreamObj = outStream(); +isolated handle outStreamObj = outStream(); isolated function print(handle printStream, any|error obj) = @java:Method { name: "print", @@ -23,11 +23,13 @@ isolated function print(handle printStream, any|error obj) = @java:Method { paramTypes: ["java.lang.Object"] } external; -function println(any|error... objs) { - foreach var obj in objs { - print(outStreamObj, obj); +isolated function println(anydata|error... objs) { + lock { + foreach var obj in objs.clone() { + print(outStreamObj, obj); + } + print(outStreamObj, "\n"); } - print(outStreamObj, "\n"); } isolated function outStream() returns handle = @java:FieldGet { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index fb6df02c739a..8902f03bcaec 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -16,31 +16,23 @@ string[] filterGroups = []; string[] filterDisableGroups = []; -string[] filterTests = []; -map filterTestModules = {}; -map filterSubTests = {}; -string moduleName = ""; -string packageName = ""; -boolean hasFilteredTests = false; -string targetPath = ""; boolean terminate = false; boolean listGroups = false; -int testWorkers = 1; +isolated final TestOptions testOptions = new (); public function setTestOptions(string inTargetPath, string inPackageName, string inModuleName, string inReport, string inCoverage, string inGroups, string inDisableGroups, string inTests, string inRerunFailed, string inListGroups, string inTestWorkers) { - - targetPath = inTargetPath; - packageName = inPackageName; - moduleName = inModuleName; + testOptions.setModuleName(inModuleName); + testOptions.setPackageName(inPackageName); + testOptions.setTargetPath(inTargetPath); filterGroups = parseStringArrayInput(inGroups); filterDisableGroups = parseStringArrayInput(inDisableGroups); boolean rerunFailed = parseBooleanInput(inRerunFailed, "rerun-failed"); boolean testReport = parseBooleanInput(inReport, "test-report"); boolean codeCoverage = parseBooleanInput(inCoverage, "code-coverage"); listGroups = parseBooleanInput(inListGroups, "list-groups"); - testWorkers = parseIntegerInput(inTestWorkers, "testWorkers"); + int testWorkers = parseIntegerInput(inTestWorkers, "testWorkers"); conMgr.setIntialWorkers(testWorkers); if rerunFailed { @@ -50,11 +42,11 @@ public function setTestOptions(string inTargetPath, string inPackageName, string enableExit(); return; } - hasFilteredTests = true; + testOptions.setHasFilteredTests(true); } else { string[] singleExecTests = parseStringArrayInput(inTests); - filterKeyBasedTests(inPackageName, moduleName, singleExecTests); - hasFilteredTests = filterTests.length() > 0; + filterKeyBasedTests(inPackageName, inModuleName, singleExecTests); + testOptions.setHasFilteredTests(testOptions.getFilterTestSize() > 0); } if testReport || codeCoverage { @@ -77,16 +69,16 @@ function filterKeyBasedTests(string packageName, string moduleName, string[] tes int separatorIndex = updatedName.indexOf(DATA_KEY_SEPARATOR); string suffix = updatedName.substring(separatorIndex + 1); string testPart = updatedName.substring(0, separatorIndex); - if (filterSubTests.hasKey(updatedName) && filterSubTests[updatedName] is string[]) { - string[] subTestList = filterSubTests[testPart]; + if (testOptions.isFilterSubTestsContains(updatedName) && testOptions.getFilterSubTest(updatedName) is string[]) { + string[] subTestList = testOptions.getFilterSubTest(testPart); subTestList.push(suffix); } else { - filterSubTests[testPart] = [suffix]; + testOptions.addFilterSubTest(testPart, [suffix]); } updatedName = testPart; } - filterTests.push(updatedName); - filterTestModules[updatedName] = prefix; + testOptions.addFilterTest(updatedName); + testOptions.setFilterTestModule(updatedName, prefix); } } @@ -111,7 +103,7 @@ function parseIntegerInput(string input, string variableName) returns int { } function parseRerunJson() returns error? { - string rerunJsonFilePath = targetPath + "/" + RERUN_JSON_FILE; + string rerunJsonFilePath = testOptions.getTargetPath() + "/" + RERUN_JSON_FILE; // if there are no previous `bal test`` runs if !fileExists(rerunJsonFilePath) { @@ -126,17 +118,17 @@ function parseRerunJson() returns error? { // but they are abstracted from the user return error("error while running failed tests : Invalid failed test data. Please run `bal test` command."); } - ModuleRerunJson? moduleRerunJson = rerunJson[moduleName]; + ModuleRerunJson? moduleRerunJson = rerunJson[testOptions.getModuleName()]; if moduleRerunJson is () { return error("error while running failed tests : Invalid failed test data. Please run `bal test` command."); } - filterTests = moduleRerunJson.testNames; - filterTestModules = moduleRerunJson.testModuleNames; - filterSubTests = moduleRerunJson.subTestNames; + testOptions.setFilterTests(moduleRerunJson.testNames); + testOptions.setFilterTestModules(moduleRerunJson.testModuleNames); + testOptions.setFilterSubTests(moduleRerunJson.subTestNames); } -function readRerunJson() returns map|error { - string|error content = trap readContent(targetPath + "/" + RERUN_JSON_FILE); +isolated function readRerunJson() returns map|error { + string|error content = trap readContent(testOptions.getTargetPath() + "/" + RERUN_JSON_FILE); if content is error { return content; } @@ -169,6 +161,7 @@ function isPrefixInCorrectFormat(string packageName, string moduleName, string t return prefix.includes(packageName) || prefix.includes(packageName + DOT + moduleName); } -function getFullModuleName() returns string { - return packageName == moduleName ? packageName : packageName + DOT + moduleName; +isolated function getFullModuleName() returns string { + return testOptions.getPackageName() == testOptions.getModuleName() ? testOptions.getPackageName() + : testOptions.getPackageName() + DOT + testOptions.getModuleName(); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal new file mode 100644 index 000000000000..5259a4530258 --- /dev/null +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal @@ -0,0 +1,334 @@ +isolated function executeTestIso(TestFunction testFunction) returns error? { + if !conMgr.isEnabled(testFunction.name) { + conMgr.setExecutionDone(testFunction.name); + conMgr.releaseWorker(); + return; + } + + error? diagnoseError = testFunction.diagnostics; + if diagnoseError is error { + reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); + println("\n" + testFunction.name + " has failed.\n"); + enableExit(); + conMgr.setExecutionDone(testFunction.name); + conMgr.releaseWorker(); + return; + } + + executeBeforeGroupFunctionsIso(testFunction); + executeBeforeEachFunctionsIso(); + + boolean shouldSkipDependents = false; + if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { + if (isDataDrivenTest(testFunction)) { + check executeDataDrivenTestSetIso(testFunction); + } else { + shouldSkipDependents = executeNonDataDrivenTestIso(testFunction); + } + } else { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + shouldSkipDependents = true; + } + + testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); + executeAfterEachFunctionsIso(); + executeAfterGroupFunctionsIso(testFunction); + + if shouldSkipDependents { + conMgr.getDependents(testFunction.name).forEach(isolated function(TestFunction dependent) { + conMgr.setSkip(dependent.name); + }); + } + + conMgr.setExecutionDone(testFunction.name); + conMgr.releaseWorker(); +} + +isolated function executeBeforeGroupFunctionsIso(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); + if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { + ExecutionError? err = executeFunctionsIso(beforeGroupFunctions, getShouldSkip()); + if err is ExecutionError { + conMgr.setSkip(testFunction.name); + groupStatusRegistry.setSkipAfterGroup('group); + enableExit(); + printExecutionError(err, "before test group function for the test"); + } + } + } +} + +isolated function executeBeforeEachFunctionsIso() { + ExecutionError? err = executeFunctionsIso(beforeEachRegistry.getFunctions(), getShouldSkip()); + if err is ExecutionError { + enableShouldSkip(); + enableExit(); + printExecutionError(err, "before each test function for the test"); + } +} + +isolated function executeDataDrivenTestSetIso(TestFunction testFunction) returns error? { + DataProviderReturnType? params = testFunction.params; + string[] keys = []; + AnyOrError[][] values = []; + TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; + if params is map { + foreach [string, AnyOrError[]] entry in params.entries() { + keys.push(entry[0]); + values.push(entry[1]); + } + } else if params is AnyOrError[][] { + testType = DATA_DRIVEN_TUPLE_OF_TUPLE; + int i = 0; + foreach AnyOrError[] entry in params { + keys.push(i.toString()); + values.push(entry); + i += 1; + } + } + + boolean isIntialJob = true; + + while keys.length() != 0 { + + if isIntialJob || conMgr.getAvailableWorkers() > 0 { + string key = keys.remove(0); + readonly & AnyOrError[] value = values.remove(0); + + if !isIntialJob { + conMgr.allocateWorker(); + } + + future<()> serialWaiter = start prepareDataDrivenTestIso(testFunction, key, value, testType); + + if !testFunction.parallelizable { + any _ = check wait serialWaiter; + } + + } + + isIntialJob = false; + conMgr.waitForWorkers(); + } + + conMgr.waitForWorkers(); + + if !isIntialJob { + conMgr.allocateWorker(); + } +} + +isolated function executeNonDataDrivenTestIso(TestFunction testFunction) returns boolean { + boolean failed = false; + boolean beforeFailed = executeBeforeFunctionIso(testFunction); + if (beforeFailed) { + conMgr.setSkip(testFunction.name); + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + return true; + } + + ExecutionError|boolean output = executeTestFunctionIso(testFunction, "", GENERAL_TEST); + if output is ExecutionError { + failed = true; + reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); + println("\n" + testFunction.name + " has failed.\n"); + } + + else if output { + failed = true; + } + boolean afterFailed = executeAfterFunctionIso(testFunction); + if (afterFailed) { + return true; + } + return failed; + +} + +isolated function executeAfterEachFunctionsIso() { + ExecutionError? err = executeFunctionsIso(afterEachRegistry.getFunctions(), getShouldSkip()); + if err is ExecutionError { + enableShouldSkip(); + enableExit(); + printExecutionError(err, "after each test function for the test"); + } +} + +isolated function executeAfterGroupFunctionsIso(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); + if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { + ExecutionError? err = executeFunctionsIso(afterGroupFunctions, + getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); + if err is ExecutionError { + enableExit(); + printExecutionError(err, "after test group function for the test"); + } + } + } +} + +isolated function executeFunctionsIso(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { + foreach TestFunction testFunction in testFunctions { + if !skip || testFunction.alwaysRun { + check executeFunctionIso(testFunction); + } + } +} + +isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { + boolean beforeFailed = executeBeforeFunctionIso(testFunction); + if (beforeFailed) { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + } + + else { + executeDataDrivenTestIso(testFunction, key, testType, value); + var _ = executeAfterFunctionIso(testFunction); + } + conMgr.releaseWorker(); + +} + +isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { + if (skipDataDrivenTestIso(testFunction, suffix, testType)) { + return; + } + + ExecutionError|boolean err = executeTestFunctionIso(testFunction, suffix, testType, params); + if err is ExecutionError { + reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + + "]\n" + getErrorMessage(err), testType = testType); + println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); + enableExit(); + } + +} + +isolated function executeBeforeFunctionIso(TestFunction testFunction) returns boolean { + boolean failed = false; + if testFunction.before is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { + ExecutionError? err = executeFunctionIso(testFunction.before); + if err is ExecutionError { + enableExit(); + printExecutionError(err, "before test function for the test"); + failed = true; + } + } + return failed; +} + +isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { + any|error output = params == () ? trap function:call(testFunction.executableFunction) + : trap function:call(testFunction.executableFunction, ...params); + if output is TestError { + enableExit(); + reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); + println("\n" + testFunction.name + ":" + suffix + " has failed\n"); + return true; + } + + else if output is any { + reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); + return false; + } + + else { + enableExit(); + return error(getErrorMessage(output), functionName = testFunction.name); + + } +} + +isolated function executeAfterFunctionIso(TestFunction testFunction) returns boolean { + boolean failed = false; + if testFunction.after is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { + ExecutionError? err = executeFunctionIso(testFunction.after); + if err is ExecutionError { + enableExit(); + printExecutionError(err, "after test function for the test"); + failed = true; + } + } + return failed; +} + +isolated function skipDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType) returns boolean { + string functionName = testFunction.name; + if (!testOptions.getHasFilteredTests()) { + return false; + } + TestFunction[] dependents = conMgr.getDependents(functionName); + + // if a dependent in a below level is enabled, this test should run + if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { + return false; + } + string functionKey = functionName; + + // check if prefix matches directly + boolean prefixMatch = testOptions.isFilterSubTestsContains(functionName); + + // if prefix matches to a wildcard + if (!prefixMatch && hasTest(functionName)) { + + // get the matching wildcard + prefixMatch = true; + foreach string filter in testOptions.getFilterTests() { + if (filter.includes(WILDCARD)) { + boolean|error wildCardMatch = matchWildcard(functionKey, filter); + if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { + functionKey = filter; + break; + } + } + } + + } + + // check if no filterSubTests found for a given prefix + boolean suffixMatch = testOptions.isFilterSubTestsContains(functionKey); + + // if a subtest is found specified + if (!suffixMatch) { + string[] subTests = testOptions.getFilterSubTest(functionKey); + + foreach string subFilter in subTests { + + string updatedSubFilter = subFilter; + if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { + if (subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE)) { + updatedSubFilter = subFilter.substring(1, subFilter.length() - 1); + } else { + continue; + } + } + string|error decodedSubFilter = escapeSpecialCharacters(updatedSubFilter); + updatedSubFilter = decodedSubFilter is string ? decodedSubFilter : updatedSubFilter; + string|error decodedSuffix = escapeSpecialCharacters(suffix); + string updatedSuffix = decodedSuffix is string ? decodedSuffix : suffix; + + boolean wildCardMatchBoolean = false; + if (updatedSubFilter.includes(WILDCARD)) { + boolean|error wildCardMatch = matchWildcard(updatedSuffix, updatedSubFilter); + wildCardMatchBoolean = wildCardMatch is boolean && wildCardMatch; + } + if ((updatedSubFilter == updatedSuffix) || wildCardMatchBoolean) { + suffixMatch = true; + break; + } + } + } + + // do not skip iff both matches + return !(prefixMatch && suffixMatch); +} + +isolated function executeFunctionIso(TestFunction|function testFunction) returns ExecutionError? { + any|error output = trap function:call((testFunction is function ? testFunction : testFunction.executableFunction)); + if output is error { + enableExit(); + return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); + } +} diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal new file mode 100644 index 000000000000..dfca6a91e212 --- /dev/null +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal @@ -0,0 +1,321 @@ +function executeTest(TestFunction testFunction) returns error? { + if !conMgr.isEnabled(testFunction.name) { + conMgr.setExecutionDone(testFunction.name); + conMgr.releaseWorker(); + return; + } + + error? diagnoseError = testFunction.diagnostics; + if diagnoseError is error { + reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); + println("\n" + testFunction.name + " has failed.\n"); + enableExit(); + conMgr.setExecutionDone(testFunction.name); + conMgr.releaseWorker(); + return; + } + + executeBeforeGroupFunctions(testFunction); + executeBeforeEachFunctions(); + + boolean shouldSkipDependents = false; + if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { + if (isDataDrivenTest(testFunction)) { + check executeDataDrivenTestSet(testFunction); + } else { + shouldSkipDependents = executeNonDataDrivenTest(testFunction); + } + } else { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + shouldSkipDependents = true; + } + + testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); + executeAfterEachFunctions(); + executeAfterGroupFunctions(testFunction); + + if shouldSkipDependents { + conMgr.getDependents(testFunction.name).forEach(function(TestFunction dependent) { + conMgr.setSkip(dependent.name); + }); + } + conMgr.setExecutionDone(testFunction.name); + conMgr.releaseWorker(); +} + +function executeBeforeGroupFunctions(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); + if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { + ExecutionError? err = executeFunctions(beforeGroupFunctions, getShouldSkip()); + if err is ExecutionError { + conMgr.setSkip(testFunction.name); + groupStatusRegistry.setSkipAfterGroup('group); + enableExit(); + printExecutionError(err, "before test group function for the test"); + } + } + } +} + +function executeBeforeEachFunctions() { + ExecutionError? err = executeFunctions(beforeEachRegistry.getFunctions(), getShouldSkip()); + if err is ExecutionError { + enableShouldSkip(); + enableExit(); + printExecutionError(err, "before each test function for the test"); + } +} + +function executeDataDrivenTestSet(TestFunction testFunction) returns error? { + DataProviderReturnType? params = testFunction.params; + string[] keys = []; + AnyOrError[][] values = []; + TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; + if params is map { + foreach [string, AnyOrError[]] entry in params.entries() { + keys.push(entry[0]); + values.push(entry[1]); + } + } else if params is AnyOrError[][] { + testType = DATA_DRIVEN_TUPLE_OF_TUPLE; + int i = 0; + foreach AnyOrError[] entry in params { + keys.push(i.toString()); + values.push(entry); + i += 1; + } + } + + boolean isIntialJob = true; + + while keys.length() != 0 { + + if isIntialJob || conMgr.getAvailableWorkers() > 0 { + string key = keys.remove(0); + AnyOrError[] value = values.remove(0); + + if !isIntialJob { + conMgr.allocateWorker(); + } + + future<()> serialWaiter = start prepareDataDrivenTest(testFunction, key, value, testType); + + if !testFunction.parallelizable { + any _ = check wait serialWaiter; + } + + } + + isIntialJob = false; + conMgr.waitForWorkers(); + } + + conMgr.waitForWorkers(); + + if !isIntialJob { + conMgr.allocateWorker(); + } +} + +function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { + boolean failed = false; + boolean beforeFailed = executeBeforeFunction(testFunction); + if (beforeFailed) { + conMgr.setSkip(testFunction.name); + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + return true; + } + ExecutionError|boolean output = executeTestFunction(testFunction, "", GENERAL_TEST); + if output is ExecutionError { + failed = true; + reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); + println("\n" + testFunction.name + " has failed.\n"); + } + + else if output { + failed = true; + } + boolean afterFailed = executeAfterFunction(testFunction); + if (afterFailed) { + return true; + } + return failed; +} + +function executeAfterEachFunctions() { + ExecutionError? err = executeFunctions(afterEachRegistry.getFunctions(), getShouldSkip()); + if err is ExecutionError { + enableShouldSkip(); + enableExit(); + printExecutionError(err, "after each test function for the test"); + } +} + +function executeAfterGroupFunctions(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); + if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { + ExecutionError? err = executeFunctions(afterGroupFunctions, + getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); + if err is ExecutionError { + enableExit(); + printExecutionError(err, "after test group function for the test"); + } + } + } +} + +function executeFunctions(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { + foreach TestFunction testFunction in testFunctions { + if !skip || testFunction.alwaysRun { + check executeFunction(testFunction); + } + } +} + +function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { + boolean beforeFailed = executeBeforeFunction(testFunction); + if (beforeFailed) { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + } else { + executeDataDrivenTest(testFunction, key, testType, value); + var _ = executeAfterFunction(testFunction); + } + conMgr.releaseWorker(); +} + +function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { + if (skipDataDrivenTest(testFunction, suffix, testType)) { + return; + } + + ExecutionError|boolean err = executeTestFunction(testFunction, suffix, testType, params); + if err is ExecutionError { + reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + + "]\n" + getErrorMessage(err), testType = testType); + println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); + enableExit(); + + } +} + +function executeBeforeFunction(TestFunction testFunction) returns boolean { + boolean failed = false; + if testFunction.before is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { + ExecutionError? err = executeFunction(testFunction.before); + if err is ExecutionError { + enableExit(); + printExecutionError(err, "before test function for the test"); + failed = true; + } + } + return failed; +} + +function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { + any|error output = params == () ? trap function:call(testFunction.executableFunction) + : trap function:call(testFunction.executableFunction, ...params); + if output is TestError { + enableExit(); + reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); + println("\n" + testFunction.name + ":" + suffix + " has failed\n"); + return true; + } else if output is any { + reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); + return false; + } else { + enableExit(); + return error(getErrorMessage(output), functionName = testFunction.name); + } +} + +function executeAfterFunction(TestFunction testFunction) returns boolean { + boolean failed = false; + if testFunction.after is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { + ExecutionError? err = executeFunction(testFunction.after); + if err is ExecutionError { + enableExit(); + printExecutionError(err, "after test function for the test"); + failed = true; + } + } + return failed; +} + +function skipDataDrivenTest(TestFunction testFunction, string suffix, TestType testType) returns boolean { + string functionName = testFunction.name; + if (!testOptions.getHasFilteredTests()) { + return false; + } + TestFunction[] dependents = conMgr.getDependents(functionName); + + // if a dependent in a below level is enabled, this test should run + if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { + return false; + } + string functionKey = functionName; + + // check if prefix matches directly + boolean prefixMatch = testOptions.isFilterSubTestsContains(functionName); + + // if prefix matches to a wildcard + if (!prefixMatch && hasTest(functionName)) { + + // get the matching wildcard + prefixMatch = true; + foreach string filter in testOptions.getFilterTests() { + if (filter.includes(WILDCARD)) { + boolean|error wildCardMatch = matchWildcard(functionKey, filter); + if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { + functionKey = filter; + break; + } + } + } + } + + // check if no filterSubTests found for a given prefix + boolean suffixMatch = !testOptions.isFilterSubTestsContains(functionKey); + + // if a subtest is found specified + if (!suffixMatch) { + string[] subTests = testOptions.getFilterSubTest(functionKey); + foreach string subFilter in subTests { + + string updatedSubFilter = subFilter; + if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { + if (subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE)) { + updatedSubFilter = subFilter.substring(1, subFilter.length() - 1); + } else { + continue; + } + } + string|error decodedSubFilter = escapeSpecialCharacters(updatedSubFilter); + updatedSubFilter = decodedSubFilter is string ? decodedSubFilter : updatedSubFilter; + string|error decodedSuffix = escapeSpecialCharacters(suffix); + string updatedSuffix = decodedSuffix is string ? decodedSuffix : suffix; + + boolean wildCardMatchBoolean = false; + if (updatedSubFilter.includes(WILDCARD)) { + boolean|error wildCardMatch = matchWildcard(updatedSuffix, updatedSubFilter); + wildCardMatchBoolean = wildCardMatch is boolean && wildCardMatch; + } + if ((updatedSubFilter == updatedSuffix) || wildCardMatchBoolean) { + suffixMatch = true; + break; + } + } + } + + // do not skip iff both matches + return !(prefixMatch && suffixMatch); +} + +function executeFunction(TestFunction|function testFunction) returns ExecutionError? { + any|error output = trap function:call(testFunction is function ? testFunction : testFunction.executableFunction); + if output is error { + enableExit(); + return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); + } +} diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index bdae52b7df65..68153e2184ba 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -1,3 +1,4 @@ +import ballerina/lang.array; // Copyright (c) 2022 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. // // WSO2 Inc. licenses this file to you under the Apache License, @@ -15,15 +16,15 @@ // under the License. import ballerina/lang.runtime; -final TestRegistry testRegistry = new (); -final TestRegistry beforeSuiteRegistry = new (); -final TestRegistry afterSuiteRegistry = new (); -final TestRegistry beforeEachRegistry = new (); -final TestRegistry afterEachRegistry = new (); +isolated final TestRegistry testRegistry = new (); +isolated final TestRegistry beforeSuiteRegistry = new (); +isolated final TestRegistry afterSuiteRegistry = new (); +isolated final TestRegistry beforeEachRegistry = new (); +isolated final TestRegistry afterEachRegistry = new (); -final GroupRegistry beforeGroupsRegistry = new (); -final GroupRegistry afterGroupsRegistry = new (); -final GroupStatusRegistry groupStatusRegistry = new (); +isolated final GroupRegistry beforeGroupsRegistry = new (); +isolated final GroupRegistry afterGroupsRegistry = new (); +isolated final GroupStatusRegistry groupStatusRegistry = new (); type TestFunction record {| string name; @@ -303,23 +304,166 @@ isolated class ConcurrentExecutionManager { } -class TestRegistry { +isolated class TestOptions { + private string moduleName = ""; + private string packageName = ""; + private string targetPath = ""; + private map filterTestModules = {}; + private boolean hasFilteredTests = false; + private string[] filterTests = []; + private map filterSubTests = {}; + + isolated function isFilterSubTestsContains(string key) returns boolean { + lock { + return self.filterSubTests.hasKey(key); + } + } + + isolated function getFilterSubTest(string key) returns string[] { + lock { + string[]? nullOrSubTests = self.filterSubTests[key]; + if nullOrSubTests is string[] { + return nullOrSubTests.clone(); + } + return []; + } + } + + isolated function addFilterSubTest(string key, string[] subTests) { + lock { + self.filterSubTests[key] = subTests.clone(); + } + } + + isolated function setFilterSubTests(map filterSubTests) { + lock { + self.filterSubTests = filterSubTests.clone(); + } + } + + isolated function setFilterTests(string[] filterTests) { + lock { + self.filterTests = filterTests.clone(); + } + } + + isolated function addFilterTest(string filterTest) { + lock { + self.filterTests.push(filterTest); + } + } + + isolated function getFilterTestSize() returns int { + lock { + return self.filterTests.length(); + } + } + + isolated function getFilterTestIndex(string testName) returns int? { + lock { + return self.filterTests.indexOf(testName); + } + } + + isolated function getFilterTests() returns string[] { + lock { + return self.filterTests.clone(); + } + } + + isolated function setModuleName(string moduleName) { + lock { + self.moduleName = moduleName; + } + } + + isolated function setPackageName(string packageName) { + lock { + self.packageName = packageName; + } + } + + isolated function getModuleName() returns string { + lock { + return self.moduleName; + } + } + + isolated function getPackageName() returns string { + lock { + return self.packageName; + } + } + + isolated function setTargetPath(string targetPath) { + lock { + self.targetPath = targetPath; + } + } + + isolated function getTargetPath() returns string { + lock { + return self.targetPath; + } + } + + isolated function setFilterTestModules(map filterTestModulesMap) { + lock { + self.filterTestModules = filterTestModulesMap.clone(); + } + } + + isolated function getFilterTestModule(string name) returns string? { + lock { + return self.filterTestModules[name]; + } + } + + isolated function setFilterTestModule(string key, string? value) { + lock { + self.filterTestModules[key] = value; + } + } + + isolated function setHasFilteredTests(boolean hasFilteredTests) { + lock { + self.hasFilteredTests = hasFilteredTests; + } + } + + isolated function getHasFilteredTests() returns boolean { + lock { + return self.hasFilteredTests; + } + } + +} + +isolated class TestRegistry { private final TestFunction[] rootRegistry = []; private final TestFunction[] dependentRegistry = []; - function addFunction(*TestFunction functionDetails) { + isolated function addFunction(*TestFunction functionDetails) { if functionDetails.dependsOn == [] { - self.rootRegistry.push(functionDetails); + lock { + self.rootRegistry.push(functionDetails); + } } else { - self.dependentRegistry.push(functionDetails); + lock { + self.dependentRegistry.push(functionDetails); + } } } - function getTestFunction(function f) returns TestFunction|error { + isolated function getTestFunction(function f) returns TestFunction|error { TestFunction[] filter; - filter = self.rootRegistry.filter(testFunction => f === testFunction.executableFunction); + lock { + filter = self.rootRegistry.filter(testFunction => f === testFunction.executableFunction).clone(); + } if filter.length() == 0 { - filter = self.dependentRegistry.filter(testFunction => f === testFunction.executableFunction); + lock { + filter = self.dependentRegistry.filter(testFunction => f === testFunction.executableFunction).clone(); + } if filter.length() == 0 { //TODO: need to obtain the function name form the variable return error(string `The dependent test function is either disabled or not included.`); @@ -328,28 +472,42 @@ class TestRegistry { return filter.pop(); } - function getFunctions() returns TestFunction[] => self.rootRegistry.sort(key = testFunctionsSort); + isolated function getFunctions() returns TestFunction[] { + lock { + return self.rootRegistry.sort(array:ASCENDING, (item) => item.name).cloneReadOnly(); + } + } + + isolated function getDependentFunctions() returns TestFunction[] { + lock { + return self.dependentRegistry.clone(); + } - function getDependentFunctions() returns TestFunction[] => self.dependentRegistry; + } } -class GroupRegistry { +isolated class GroupRegistry { private final map registry = {}; function addFunction(string 'group, *TestFunction testFunction) { - if self.registry.hasKey('group) { - self.registry.get('group).push(testFunction); - } else { - self.registry['group] = [testFunction]; + lock { + if self.registry.hasKey('group) { + self.registry.get('group).push(testFunction); + } else { + self.registry['group] = [testFunction]; + } } } - function getFunctions(string 'group) returns TestFunction[]? { - if self.registry.hasKey('group) { - return self.registry.get('group); + isolated function getFunctions(string 'group) returns TestFunction[]? { + lock { + if self.registry.hasKey('group) { + return self.registry.get('group).cloneReadOnly(); + } + return; + } - return; } } @@ -421,6 +579,3 @@ isolated class GroupStatusRegistry { } isolated function testFunctionsSort(TestFunction testFunction) returns string => testFunction.name; - -function isDataDrivenTest(TestFunction testFunction) returns boolean => - testFunction.params is map || testFunction.params is AnyOrError[][]; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal index b3f35f595a84..93f277f45548 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal @@ -21,7 +21,7 @@ const string TESTS_CACHE_DIRECTORY = "tests_cache"; type ReportGenerate function (ReportData data); -ReportData reportData = new (); +isolated final ReportData reportData = new (); ReportGenerate[] reportGenerators = [consoleReport, failedTestsReport]; @@ -30,57 +30,125 @@ type ResultData record {| string suffix = ""; string message = ""; TestType testType; -|}; +|} & readonly; -class Result { +isolated class Result { private ResultData data; - function init(ResultData data) { - self.data = data; + isolated function init(ResultData data) { + lock { + self.data = data.clone(); + } } - function fullName() returns string => - self.data.suffix == "" ? self.data.name : self.data.name + DATA_KEY_SEPARATOR + self.data.suffix; + isolated function fullName() returns string { + lock { + return self.data.suffix == "" ? self.data.name : self.data.name + DATA_KEY_SEPARATOR + self.data.suffix; + } + } - function isDataProvider() returns boolean => self.data.suffix != ""; + isolated function isDataProvider() returns boolean { + lock { + return self.data.suffix != ""; + } + } - function testPrefix() returns string => self.data.name; + isolated function testPrefix() returns string { + lock { + return self.data.name; + } + } - function testSuffix() returns string => self.data.suffix; + isolated function testSuffix() returns string { + lock { + return self.data.suffix; + } + } - function message() returns string => self.data.message; + isolated function message() returns string { + lock { + return self.data.message; + } + } - function testType() returns TestType => self.data.testType; + isolated function testType() returns TestType { + lock { + return self.data.testType; + } + } } -class ReportData { - private Result[] passed = []; - private Result[] failed = []; - private Result[] skipped = []; +isolated class ReportData { + private ResultData[] passed = []; + private ResultData[] failed = []; + private ResultData[] skipped = []; - function onPassed(*ResultData result) => self.passed.push(new Result(result)); - function onFailed(*ResultData result) => self.failed.push(new Result(result)); - function onSkipped(*ResultData result) => self.skipped.push(new Result(result)); + isolated function onPassed(*ResultData result) { + lock { + self.passed.push(result); + } + } + isolated function onFailed(*ResultData result) { + lock { + self.passed.push(result); + } + } + isolated function onSkipped(*ResultData result) { + lock { + self.passed.push(result); + } + } - function passedCases() returns Result[] => self.passed; - function failedCases() returns Result[] => self.failed; - function skippedCases() returns Result[] => self.skipped; + isolated function passedCases() returns ResultData[] { + lock { + return self.passed.clone(); + } + } + isolated function failedCases() returns ResultData[] { + lock { + return self.failed.clone(); + } + } + function skippedCases() returns ResultData[] { + lock { + return self.skipped.clone(); + } + } + + isolated function passedCount() returns int { + lock { + return self.passed.length(); + } + } + isolated function failedCount() returns int { + lock { + return self.failed.length(); + } + } + isolated function skippedCount() returns int { + lock { + return self.skipped.length(); + } + } - function passedCount() returns int => self.passed.length(); - function failedCount() returns int => self.failed.length(); - function skippedCount() returns int => self.skipped.length(); } -function consoleReport(ReportData data) { +isolated function consoleReport(ReportData data) { if (!isSystemConsole()) { - data.passedCases().forEach(entry => println("\t\t[pass] " + entry.fullName())); + data.passedCases().forEach(isolated function(ResultData entrydata) { + Result entry = new (entrydata); + println("\t\t[pass] " + entry.fullName()); + }); } - data.failedCases().forEach(function(Result entry) { + + data.failedCases().forEach(isolated function(ResultData entrydata) { + Result entry = new (entrydata); println("\n\t\t[fail] " + entry.fullName() + ":"); println("\n\t\t " + formatFailedError(entry.message(), 3)); }); int totalTestCount = data.passedCount() + data.failedCount() + data.skippedCount(); + println("\n"); if (totalTestCount == 0) { println("\t\tNo tests found"); @@ -91,7 +159,7 @@ function consoleReport(ReportData data) { } } -function formatFailedError(string message, int tabCount) returns string { +isolated function formatFailedError(string message, int tabCount) returns string { string[] lines = split(message, "\n"); lines.push(""); string tabs = ""; @@ -103,16 +171,17 @@ function formatFailedError(string message, int tabCount) returns string { return string:'join("\n" + tabs, ...lines); } -function failedTestsReport(ReportData data) { +isolated function failedTestsReport(ReportData data) { string[] testNames = []; map testModuleNames = {}; map subTestNames = {}; - foreach Result result in data.failedCases() { + foreach ResultData resultdata in data.failedCases() { + Result result = new (resultdata); string testPrefix = result.testPrefix(); string testSuffix = result.testType() == DATA_DRIVEN_MAP_OF_TUPLE ? SINGLE_QUOTE + result.testSuffix() + SINGLE_QUOTE : result.testSuffix(); testNames.push(testPrefix); - testModuleNames[testPrefix] = moduleName; + testModuleNames[testPrefix] = testOptions.getModuleName(); if (result.isDataProvider()) { if (subTestNames.hasKey(testPrefix) && subTestNames[testPrefix] is string[]) { string[] subTestList = subTestNames[testPrefix]; @@ -123,7 +192,7 @@ function failedTestsReport(ReportData data) { } } ModuleRerunJson moduleReport = {testNames, testModuleNames, subTestNames}; - string filePath = targetPath + "/" + RERUN_JSON_FILE; + string filePath = testOptions.getTargetPath() + "/" + RERUN_JSON_FILE; map rerunJson; if fileExists(filePath) { @@ -136,27 +205,28 @@ function failedTestsReport(ReportData data) { } else { rerunJson = {}; } - rerunJson[moduleName] = moduleReport; + rerunJson[testOptions.getModuleName()] = moduleReport; error? err = writeContent(filePath, rerunJson.toString()); if err is error { println(err.message()); + } } function moduleStatusReport(ReportData data) { map[] tests = []; data.passedCases().forEach(result => tests.push({ - "name": escapeSpecialCharactersJson(result.fullName()), + "name": escapeSpecialCharactersJson(new Result(result).fullName()), "status": "PASSED" })); data.failedCases().forEach(result => tests.push({ - "name": escapeSpecialCharactersJson(result.fullName()), + "name": escapeSpecialCharactersJson(new Result(result).fullName()), "status": "FAILURE", - "failureMessage": replaceDoubleQuotes(result.message()) + "failureMessage": replaceDoubleQuotes(new Result(result).message()) })); data.skippedCases().forEach(result => tests.push({ - "name": escapeSpecialCharactersJson(result.fullName()), + "name": escapeSpecialCharactersJson((new Result(result).fullName())), "status": "SKIPPED" })); @@ -168,8 +238,8 @@ function moduleStatusReport(ReportData data) { "tests": tests }; - error? err = writeContent(targetPath + "/" + CACHE_DIRECTORY + "/" + TESTS_CACHE_DIRECTORY - + "/" + moduleName + "/" + MODULE_STATUS_JSON_FILE, output.toString()); + error? err = writeContent(testOptions.getTargetPath() + "/" + CACHE_DIRECTORY + "/" + TESTS_CACHE_DIRECTORY + + "/" + testOptions.getModuleName() + "/" + MODULE_STATUS_JSON_FILE, output.toString()); if err is error { println(err.message()); } From ae8532c710ab83f552425ad58f03135e41db5623 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Fri, 5 Jan 2024 14:35:26 +0530 Subject: [PATCH 24/37] Remove async function based execution for serial execution --- .../src/main/ballerina/execute.bal | 7 +-- .../src/main/ballerina/filter.bal | 1 + .../src/main/ballerina/isolateExecuter.bal | 23 +++++--- .../src/main/ballerina/nonIsolateExecuter.bal | 53 ++++++++----------- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 2350411bf1af..645cb37f29af 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -81,20 +81,17 @@ function executeTests() returns error? { TestFunction testFunction = conMgr.getSerialTest(); conMgr.addTestInExecution(testFunction); conMgr.allocateWorker(); - future serialWaiter = start executeTest(testFunction); - any _ = check wait serialWaiter; - + executeTest(testFunction); } else if conMgr.getParallelQueueLength() != 0 && conMgr.getSerialQueueLength() == 0 { TestFunction testFunction = conMgr.getParallelTest(); conMgr.addTestInExecution(testFunction); conMgr.allocateWorker(); - future<(error?)> parallelWaiter = start executeTest(testFunction); + future<(error?)> parallelWaiter = start executeTestIso(testFunction); if isDataDrivenTest(testFunction) { any _ = check wait parallelWaiter; } - } } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index 8902f03bcaec..8f68d29b8c7c 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -72,6 +72,7 @@ function filterKeyBasedTests(string packageName, string moduleName, string[] tes if (testOptions.isFilterSubTestsContains(updatedName) && testOptions.getFilterSubTest(updatedName) is string[]) { string[] subTestList = testOptions.getFilterSubTest(testPart); subTestList.push(suffix); + testOptions.addFilterSubTest(testPart, subTestList); } else { testOptions.addFilterSubTest(testPart, [suffix]); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal index 5259a4530258..ceaadbebf348 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal @@ -1,3 +1,19 @@ +// Copyright (c) 2023 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. +// +// WSO2 Inc. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + isolated function executeTestIso(TestFunction testFunction) returns error? { if !conMgr.isEnabled(testFunction.name) { conMgr.setExecutionDone(testFunction.name); @@ -100,12 +116,7 @@ isolated function executeDataDrivenTestSetIso(TestFunction testFunction) returns conMgr.allocateWorker(); } - future<()> serialWaiter = start prepareDataDrivenTestIso(testFunction, key, value, testType); - - if !testFunction.parallelizable { - any _ = check wait serialWaiter; - } - + future<()> _ = start prepareDataDrivenTestIso(testFunction, key, value, testType); } isIntialJob = false; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal index dfca6a91e212..cec706488493 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal @@ -1,4 +1,20 @@ -function executeTest(TestFunction testFunction) returns error? { +// Copyright (c) 2023 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. +// +// WSO2 Inc. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +function executeTest(TestFunction testFunction) { if !conMgr.isEnabled(testFunction.name) { conMgr.setExecutionDone(testFunction.name); conMgr.releaseWorker(); @@ -21,7 +37,7 @@ function executeTest(TestFunction testFunction) returns error? { boolean shouldSkipDependents = false; if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { if (isDataDrivenTest(testFunction)) { - check executeDataDrivenTestSet(testFunction); + executeDataDrivenTestSet(testFunction); } else { shouldSkipDependents = executeNonDataDrivenTest(testFunction); } @@ -67,7 +83,7 @@ function executeBeforeEachFunctions() { } } -function executeDataDrivenTestSet(TestFunction testFunction) returns error? { +function executeDataDrivenTestSet(TestFunction testFunction) { DataProviderReturnType? params = testFunction.params; string[] keys = []; AnyOrError[][] values = []; @@ -87,35 +103,12 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? { } } - boolean isIntialJob = true; - while keys.length() != 0 { - - if isIntialJob || conMgr.getAvailableWorkers() > 0 { - string key = keys.remove(0); - AnyOrError[] value = values.remove(0); - - if !isIntialJob { - conMgr.allocateWorker(); - } - - future<()> serialWaiter = start prepareDataDrivenTest(testFunction, key, value, testType); - - if !testFunction.parallelizable { - any _ = check wait serialWaiter; - } - - } - - isIntialJob = false; - conMgr.waitForWorkers(); + string key = keys.remove(0); + AnyOrError[] value = values.remove(0); + prepareDataDrivenTest(testFunction, key, value, testType); } - conMgr.waitForWorkers(); - - if !isIntialJob { - conMgr.allocateWorker(); - } } function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { @@ -182,7 +175,6 @@ function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError executeDataDrivenTest(testFunction, key, testType, value); var _ = executeAfterFunction(testFunction); } - conMgr.releaseWorker(); } function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { @@ -196,7 +188,6 @@ function executeDataDrivenTest(TestFunction testFunction, string suffix, TestTyp + "]\n" + getErrorMessage(err), testType = testType); println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); enableExit(); - } } From 0c7b92bdddf0b2f5f442ad76ec84b4fbb0d456b1 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Mon, 8 Jan 2024 11:28:39 +0530 Subject: [PATCH 25/37] Allow readonly data provider return types only --- .../main/ballerina/annotation_processor.bal | 8 ++-- .../src/main/ballerina/execute.bal | 19 ++++---- .../src/main/ballerina/isolateExecuter.bal | 43 ++++++++++--------- .../src/main/ballerina/nonIsolateExecuter.bal | 30 ++++++------- .../src/main/ballerina/register.bal | 5 ++- .../src/main/ballerina/report.bal | 4 +- .../src/main/ballerina/types.bal | 6 ++- .../testerina/natives/CommonUtils.java | 11 ++--- 8 files changed, 63 insertions(+), 63 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 734b34421b51..1b362aecb67a 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -53,7 +53,7 @@ function processConfigAnnotation(string name, function f) returns boolean { boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated; string[] reasonToSerialExecution = []; - DataProviderReturnType? & readonly params = (); + DataProviderReturnType? params = (); error? diagnostics = (); if config.dataProvider != () { var providerFn = config.dataProvider; @@ -63,7 +63,7 @@ function processConfigAnnotation(string name, function f) returns boolean { isTestFunctionParamSafe = isFunctionParamConcurrencySafe(f); isSatisfiedParallelizableConditions = isTestFunctionIsolated && isDataProviderIsolated && isTestFunctionParamSafe; DataProviderReturnType providerOutput = providerFn(); - params = providerOutput; + params = providerOutput; } else { diagnostics = error("Failed to execute the data provider"); } @@ -90,8 +90,8 @@ function processConfigAnnotation(string name, function f) returns boolean { boolean enabled = config.enable && (filterGroups.length() == 0 ? true : hasGroup(config.groups, filterGroups)) && (filterDisableGroups.length() == 0 ? true : !hasGroup(config.groups, filterDisableGroups)) && hasTest(name); config.groups.forEach('group => groupStatusRegistry.incrementTotalTest('group, enabled)); - - testRegistry.addFunction(name = name, executableFunction = f, params = params, before = config.before, + dataDrivenTestParams[name] = params; + testRegistry.addFunction(name = name, executableFunction = f, before = config.before, after = config.after, groups = config.groups.cloneReadOnly(), diagnostics = diagnostics, dependsOn = config.dependsOn.cloneReadOnly(), parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (conMgr.getConfiguredWorkers() > 1)), config = config.cloneReadOnly()); conMgr.createTestFunctionMetaData(functionName = name, dependsOnCount = config.dependsOn.length(), enabled = enabled); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 645cb37f29af..ca7cb4a19121 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -20,6 +20,7 @@ isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; isolated int exitCode = 0; isolated final ConcurrentExecutionManager conMgr = new (); +map dataDrivenTestParams = {}; public function startSuite() returns int { // exit if setTestOptions has failed @@ -71,12 +72,10 @@ function executeTests() returns error? { foreach TestFunction testFunction in testRegistry.getFunctions() { _ = testFunction.parallelizable ? conMgr.addInitialParallelTest(testFunction) : conMgr.addInitialSerialTest(testFunction); } - while !conMgr.isExecutionDone() { if conMgr.getAvailableWorkers() != 0 { conMgr.waitUntilEmptyQueueFilled(); - if conMgr.getSerialQueueLength() != 0 && conMgr.getAvailableWorkers() == conMgr.getConfiguredWorkers() { TestFunction testFunction = conMgr.getSerialTest(); conMgr.addTestInExecution(testFunction); @@ -88,8 +87,9 @@ function executeTests() returns error? { TestFunction testFunction = conMgr.getParallelTest(); conMgr.addTestInExecution(testFunction); conMgr.allocateWorker(); - future<(error?)> parallelWaiter = start executeTestIso(testFunction); - if isDataDrivenTest(testFunction) { + ParallelExecutionArgs parallelExecutionMetaData = {params: dataDrivenTestParams[testFunction.name]}; + future<(error?)> parallelWaiter = start executeTestIso(testFunction, parallelExecutionMetaData); + if isDataDrivenTest(dataDrivenTestParams[testFunction.name]) { any _ = check wait parallelWaiter; } } @@ -176,11 +176,10 @@ isolated function getErrorMessage(error err) returns string { return message + "\n" + accumulatedTrace; } -isolated function getTestType(TestFunction testFunction) returns TestType { - DataProviderReturnType? params = testFunction.params; - if (params is map) { +isolated function getTestType(DataProviderReturnType? params) returns TestType { + if (params is map) { return DATA_DRIVEN_MAP_OF_TUPLE; - } else if (params is AnyOrError[][]) { + } else if (params is AnyOrErrorOrReadOnlyType[][]) { return DATA_DRIVEN_TUPLE_OF_TUPLE; } return GENERAL_TEST; @@ -202,8 +201,8 @@ isolated function nestedEnabledDependentsAvailable(TestFunction[] dependents) re return nestedEnabledDependentsAvailable(queue); } -isolated function isDataDrivenTest(TestFunction testFunction) returns boolean => - testFunction.params is map || testFunction.params is AnyOrError[][]; +isolated function isDataDrivenTest(DataProviderReturnType? params) returns boolean => + params is map || params is AnyOrErrorOrReadOnlyType[][]; isolated function enableShouldSkip() { lock { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal index ceaadbebf348..a6a558036d84 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal @@ -14,7 +14,7 @@ // specific language governing permissions and limitations // under the License. -isolated function executeTestIso(TestFunction testFunction) returns error? { +isolated function executeTestIso(TestFunction testFunction, ParallelExecutionArgs parallelExecutionMetaData) returns error? { if !conMgr.isEnabled(testFunction.name) { conMgr.setExecutionDone(testFunction.name); conMgr.releaseWorker(); @@ -22,8 +22,9 @@ isolated function executeTestIso(TestFunction testFunction) returns error? { } error? diagnoseError = testFunction.diagnostics; + DataProviderReturnType? dataProviderParams = parallelExecutionMetaData.params; if diagnoseError is error { - reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); + reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(dataProviderParams)); println("\n" + testFunction.name + " has failed.\n"); enableExit(); conMgr.setExecutionDone(testFunction.name); @@ -36,13 +37,13 @@ isolated function executeTestIso(TestFunction testFunction) returns error? { boolean shouldSkipDependents = false; if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { - if (isDataDrivenTest(testFunction)) { - check executeDataDrivenTestSetIso(testFunction); + if (isDataDrivenTest(dataProviderParams)) { + check executeDataDrivenTestSetIso(testFunction, parallelExecutionMetaData); } else { - shouldSkipDependents = executeNonDataDrivenTestIso(testFunction); + shouldSkipDependents = executeNonDataDrivenTestIso(testFunction, parallelExecutionMetaData); } } else { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + reportData.onSkipped(name = testFunction.name, testType = getTestType(dataProviderParams)); shouldSkipDependents = true; } @@ -84,20 +85,20 @@ isolated function executeBeforeEachFunctionsIso() { } } -isolated function executeDataDrivenTestSetIso(TestFunction testFunction) returns error? { - DataProviderReturnType? params = testFunction.params; +isolated function executeDataDrivenTestSetIso(TestFunction testFunction, ParallelExecutionArgs parallelExecutionMetaData) returns error? { + DataProviderReturnType? params = parallelExecutionMetaData.params; string[] keys = []; - AnyOrError[][] values = []; + AnyOrErrorOrReadOnlyType[][] values = []; TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; - if params is map { - foreach [string, AnyOrError[]] entry in params.entries() { + if params is map { + foreach [string, AnyOrErrorOrReadOnlyType[]] entry in params.entries() { keys.push(entry[0]); values.push(entry[1]); } - } else if params is AnyOrError[][] { + } else if params is AnyOrErrorOrReadOnlyType[][] { testType = DATA_DRIVEN_TUPLE_OF_TUPLE; int i = 0; - foreach AnyOrError[] entry in params { + foreach AnyOrErrorOrReadOnlyType[] entry in params { keys.push(i.toString()); values.push(entry); i += 1; @@ -110,13 +111,13 @@ isolated function executeDataDrivenTestSetIso(TestFunction testFunction) returns if isIntialJob || conMgr.getAvailableWorkers() > 0 { string key = keys.remove(0); - readonly & AnyOrError[] value = values.remove(0); + ReadOnlyType[] value = values.remove(0); if !isIntialJob { conMgr.allocateWorker(); } - future<()> _ = start prepareDataDrivenTestIso(testFunction, key, value, testType); + future<()> _ = start prepareDataDrivenTestIso(testFunction, key, value.clone(), testType); } isIntialJob = false; @@ -130,12 +131,12 @@ isolated function executeDataDrivenTestSetIso(TestFunction testFunction) returns } } -isolated function executeNonDataDrivenTestIso(TestFunction testFunction) returns boolean { +isolated function executeNonDataDrivenTestIso(TestFunction testFunction, ParallelExecutionArgs parallelExecutionMetaData) returns boolean { boolean failed = false; boolean beforeFailed = executeBeforeFunctionIso(testFunction); if (beforeFailed) { conMgr.setSkip(testFunction.name); - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + reportData.onSkipped(name = testFunction.name, testType = getTestType(parallelExecutionMetaData.params)); return true; } @@ -188,10 +189,10 @@ isolated function executeFunctionsIso(TestFunction[] testFunctions, boolean skip } } -isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { +isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, ReadOnlyType[] value, TestType testType) { boolean beforeFailed = executeBeforeFunctionIso(testFunction); if (beforeFailed) { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + reportData.onSkipped(name = testFunction.name, testType = testType); } else { @@ -202,7 +203,7 @@ isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key } -isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { +isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { if (skipDataDrivenTestIso(testFunction, suffix, testType)) { return; } @@ -230,7 +231,7 @@ isolated function executeBeforeFunctionIso(TestFunction testFunction) returns bo return failed; } -isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { +isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { any|error output = params == () ? trap function:call(testFunction.executableFunction) : trap function:call(testFunction.executableFunction, ...params); if output is TestError { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal index cec706488493..e0e3213dee04 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal @@ -23,7 +23,7 @@ function executeTest(TestFunction testFunction) { error? diagnoseError = testFunction.diagnostics; if diagnoseError is error { - reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunction)); + reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(dataDrivenTestParams[testFunction.name])); println("\n" + testFunction.name + " has failed.\n"); enableExit(); conMgr.setExecutionDone(testFunction.name); @@ -36,13 +36,13 @@ function executeTest(TestFunction testFunction) { boolean shouldSkipDependents = false; if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { - if (isDataDrivenTest(testFunction)) { + if (isDataDrivenTest(dataDrivenTestParams[testFunction.name])) { executeDataDrivenTestSet(testFunction); } else { shouldSkipDependents = executeNonDataDrivenTest(testFunction); } } else { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); shouldSkipDependents = true; } @@ -84,19 +84,19 @@ function executeBeforeEachFunctions() { } function executeDataDrivenTestSet(TestFunction testFunction) { - DataProviderReturnType? params = testFunction.params; + DataProviderReturnType? params = dataDrivenTestParams[testFunction.name]; string[] keys = []; - AnyOrError[][] values = []; + AnyOrErrorOrReadOnlyType[][] values = []; TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; - if params is map { - foreach [string, AnyOrError[]] entry in params.entries() { + if params is map { + foreach [string, AnyOrErrorOrReadOnlyType[]] entry in params.entries() { keys.push(entry[0]); values.push(entry[1]); } - } else if params is AnyOrError[][] { + } else if params is AnyOrErrorOrReadOnlyType[][] { testType = DATA_DRIVEN_TUPLE_OF_TUPLE; int i = 0; - foreach AnyOrError[] entry in params { + foreach AnyOrErrorOrReadOnlyType[] entry in params { keys.push(i.toString()); values.push(entry); i += 1; @@ -105,7 +105,7 @@ function executeDataDrivenTestSet(TestFunction testFunction) { while keys.length() != 0 { string key = keys.remove(0); - AnyOrError[] value = values.remove(0); + AnyOrErrorOrReadOnlyType[] value = values.remove(0); prepareDataDrivenTest(testFunction, key, value, testType); } @@ -116,7 +116,7 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { boolean beforeFailed = executeBeforeFunction(testFunction); if (beforeFailed) { conMgr.setSkip(testFunction.name); - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); return true; } ExecutionError|boolean output = executeTestFunction(testFunction, "", GENERAL_TEST); @@ -167,17 +167,17 @@ function executeFunctions(TestFunction[] testFunctions, boolean skip = false) re } } -function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { +function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrErrorOrReadOnlyType[] value, TestType testType) { boolean beforeFailed = executeBeforeFunction(testFunction); if (beforeFailed) { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunction)); + reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); } else { executeDataDrivenTest(testFunction, key, testType, value); var _ = executeAfterFunction(testFunction); } } -function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { +function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { if (skipDataDrivenTest(testFunction, suffix, testType)) { return; } @@ -204,7 +204,7 @@ function executeBeforeFunction(TestFunction testFunction) returns boolean { return failed; } -function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { +function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { any|error output = params == () ? trap function:call(testFunction.executableFunction) : trap function:call(testFunction.executableFunction, ...params); if output is TestError { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 68153e2184ba..a78c725b30fa 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -26,10 +26,13 @@ isolated final GroupRegistry beforeGroupsRegistry = new (); isolated final GroupRegistry afterGroupsRegistry = new (); isolated final GroupStatusRegistry groupStatusRegistry = new (); +type ParallelExecutionArgs record {| + DataProviderReturnType? params; +|}; + type TestFunction record {| string name; function executableFunction; - DataProviderReturnType? params = (); function? before = (); function? after = (); boolean alwaysRun = false; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal index 93f277f45548..e96d5edc4227 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal @@ -90,12 +90,12 @@ isolated class ReportData { } isolated function onFailed(*ResultData result) { lock { - self.passed.push(result); + self.failed.push(result); } } isolated function onSkipped(*ResultData result) { lock { - self.passed.push(result); + self.skipped.push(result); } } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal index 3e1258f8182b..3101d18d8ba4 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal @@ -15,9 +15,11 @@ // under the License. # Possible return types of the data provider function. -public type DataProviderReturnType error|map|AnyOrError[][]; +public type DataProviderReturnType error|map|AnyOrErrorOrReadOnlyType[][]; -public type AnyOrError any|error; +public type AnyOrErrorOrReadOnlyType any|error|ReadOnlyType; + +public type ReadOnlyType readonly; # Represents errors generated by the Testerina module. public type TestError distinct error; diff --git a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java index 85f7565eb22b..b2ed671959e6 100644 --- a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java +++ b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java @@ -18,7 +18,6 @@ package org.ballerinalang.testerina.natives; import io.ballerina.runtime.api.types.FunctionType; -import io.ballerina.runtime.api.types.ObjectType; import io.ballerina.runtime.api.types.Parameter; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.types.UnionType; @@ -42,7 +41,7 @@ public static Object isFunctionParamConcurrencySafe(BFunctionPointer func) { Parameter[] functionParameters = functionType.getParameters(); for (Parameter functionParameter : functionParameters) { Type parameterType = functionParameter.type; - if (isSubTypeOfReadOnlyOrIsolatedObjectUnion(parameterType)) { + if (isSubTypeOfReadOnly(parameterType)) { continue; } return false; @@ -50,21 +49,17 @@ public static Object isFunctionParamConcurrencySafe(BFunctionPointer func) { return true; } - private static boolean isSubTypeOfReadOnlyOrIsolatedObjectUnion(Type type) { + private static boolean isSubTypeOfReadOnly(Type type) { if (type.isReadOnly()) { return true; } - if (type instanceof ObjectType) { - return ((ObjectType) type).isIsolated(); - } - if (!(type instanceof UnionType)) { return false; } for (Type memberType : ((UnionType) type).getMemberTypes()) { - if (!isSubTypeOfReadOnlyOrIsolatedObjectUnion(memberType)) { + if (!isSubTypeOfReadOnly(memberType)) { return false; } } From d66c5fc24871cab9b0ebe07a554c3716684356ae Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 11 Jan 2024 10:47:14 +0530 Subject: [PATCH 26/37] Add a new test completion status --- .../io/ballerina/cli/cmd/TestCommand.java | 4 +- .../TesterinaCompilerPluginConstants.java | 2 +- .../TesterinaCompilerPluginUtils.java | 4 +- .../main/ballerina/annotation_processor.bal | 21 ++- .../src/main/ballerina/execute.bal | 27 ++-- .../src/main/ballerina/isolateExecuter.bal | 71 +++------- .../src/main/ballerina/nonIsolateExecuter.bal | 11 +- .../src/main/ballerina/register.bal | 101 +++++--------- .../src/main/ballerina/report.bal | 1 - .../src/main/ballerina/types.bal | 6 + .../test/TestparallelizationTest.java | 6 +- ...iderTest-testDataProviderSingleFailure.txt | 11 +- ...oviderTestCase-testInvalidDataProvider.txt | 22 ++- ...viderTestCase-testInvalidDataProvider2.txt | 22 ++- ...rTestCase-testInvalidTupleDataProvider.txt | 22 ++- ...iderTest-testDataProviderSingleFailure.txt | 13 +- ...oviderTestCase-testInvalidDataProvider.txt | 24 +++- ...viderTestCase-testInvalidDataProvider2.txt | 24 +++- ...rTestCase-testInvalidTupleDataProvider.txt | 24 +++- .../tests/normalTest.bal | 127 +++++++++--------- 20 files changed, 270 insertions(+), 273 deletions(-) diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java index b0db733f1f09..86d692714150 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java @@ -132,7 +132,7 @@ public TestCommand() { @CommandLine.Option(names = "--debug", description = "start in remote debugging mode") private String debugPort; - @CommandLine.Option(names = "--workers", description = "maximum number of parallel test jobs") + @CommandLine.Option(names = "--workers", description = "maximum number of parallel test jobs", defaultValue = "1") private int workers; @CommandLine.Option(names = "--list-groups", description = "list the groups available in the tests") @@ -282,7 +282,7 @@ public void execute() { return; } - if (workers < 0) { + if (workers < 1) { this.outStream.println("\nWarning: Workers can not be negative or zero. Test execution is proceeded " + "with default worker count.\n"); } diff --git a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java index 9cc8d9ba4cd6..c0f8bb2990b1 100644 --- a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java +++ b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java @@ -46,7 +46,7 @@ public class TesterinaCompilerPluginConstants { public static final String TESTS_PARAMETER = "tests"; public static final String RERUN_FAILED_PARAMETER = "rerunFailed"; public static final String LIST_GROUPS_PARAMETER = "listGroups"; - public static final String TEST_PARALLEL_JOBS_PARAMETER = "testWorkers"; + public static final String PARALLEL_TEST_JOB_COUNT_PARAMETER = "testWorkers"; private TesterinaCompilerPluginConstants() {} } diff --git a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java index c1b6a6241f2b..5596c3f6ee3f 100644 --- a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java +++ b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java @@ -79,7 +79,7 @@ public static void addSetTestOptionsCall(List statements) { getPositionalArg(TesterinaCompilerPluginConstants.TESTS_PARAMETER), getPositionalArg(TesterinaCompilerPluginConstants.RERUN_FAILED_PARAMETER), getPositionalArg(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER), - getPositionalArg(TesterinaCompilerPluginConstants.TEST_PARALLEL_JOBS_PARAMETER))))); + getPositionalArg(TesterinaCompilerPluginConstants.PARALLEL_TEST_JOB_COUNT_PARAMETER))))); } public static void addStartSuiteCall(List statements) { @@ -287,7 +287,7 @@ public static FunctionSignatureNode getFunctionSignature() { NodeFactory.createToken(SyntaxKind.COMMA_TOKEN), getStringParameter(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER), NodeFactory.createToken(SyntaxKind.COMMA_TOKEN), - getStringParameter(TesterinaCompilerPluginConstants.TEST_PARALLEL_JOBS_PARAMETER)), + getStringParameter(TesterinaCompilerPluginConstants.PARALLEL_TEST_JOB_COUNT_PARAMETER)), NodeFactory.createToken(SyntaxKind.CLOSE_PAREN_TOKEN), returnTypeDescriptorNode); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 1b362aecb67a..75f97ef1f18a 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -51,13 +51,11 @@ function processConfigAnnotation(string name, function f) returns boolean { boolean isDataProviderIsolated = true; boolean isTestFunctionParamSafe = true; boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated; - string[] reasonToSerialExecution = []; - + string[] reasonForSerialExecution = []; DataProviderReturnType? params = (); error? diagnostics = (); if config.dataProvider != () { var providerFn = config.dataProvider; - if providerFn is function () returns (DataProviderReturnType?) { isDataProviderIsolated = (providerFn is isolated function); isTestFunctionParamSafe = isFunctionParamConcurrencySafe(f); @@ -71,20 +69,18 @@ function processConfigAnnotation(string name, function f) returns boolean { // Register the reason for serial execution. if !isTestFunctionIsolated { - reasonToSerialExecution.push("non-isolated test function"); + reasonForSerialExecution.push("non-isolated test function"); } - if !isDataProviderIsolated { - reasonToSerialExecution.push("non-isolated data-provider function"); + reasonForSerialExecution.push("non-isolated data-provider function"); } - if !isTestFunctionParamSafe { - reasonToSerialExecution.push("unsafe test parameters"); + reasonForSerialExecution.push("unsafe test parameters"); } // If the test function is not parallelizable, then print the reason for serial execution. if !isSatisfiedParallelizableConditions && !config.serialExecution && (conMgr.getConfiguredWorkers() > 1) { - println("WARNING: Test function '" + name + "' cannot be parallelized due to " + string:'join(",", ...reasonToSerialExecution)); + println("WARNING: Test function '" + name + "' cannot be parallelized, reason: " + string:'join(",", ...reasonForSerialExecution)); } boolean enabled = config.enable && (filterGroups.length() == 0 ? true : hasGroup(config.groups, filterGroups)) @@ -93,7 +89,8 @@ function processConfigAnnotation(string name, function f) returns boolean { dataDrivenTestParams[name] = params; testRegistry.addFunction(name = name, executableFunction = f, before = config.before, after = config.after, groups = config.groups.cloneReadOnly(), diagnostics = diagnostics, dependsOn = config.dependsOn.cloneReadOnly(), - parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (conMgr.getConfiguredWorkers() > 1)), config = config.cloneReadOnly()); + parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (conMgr.getConfiguredWorkers() > 1)), + config = config.cloneReadOnly()); conMgr.createTestFunctionMetaData(functionName = name, dependsOnCount = config.dependsOn.length(), enabled = enabled); } return false; @@ -172,9 +169,7 @@ isolated function hasTest(string name) returns boolean { foreach string filter in testOptions.getFilterTests() { if (filter.includes(WILDCARD)) { boolean|error wildCardMatch = matchWildcard(testName, filter); - if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { - return true; - } + return (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)); } } return false; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index ca7cb4a19121..f246078971fd 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -14,7 +14,6 @@ // specific language governing permissions and limitations // under the License. import ballerina/lang.'error as langError; -import ballerina/lang.runtime; isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; @@ -44,7 +43,6 @@ public function startSuite() returns int { lock { return exitCode; } - } error? err = orderTests(); @@ -73,31 +71,31 @@ function executeTests() returns error? { _ = testFunction.parallelizable ? conMgr.addInitialParallelTest(testFunction) : conMgr.addInitialSerialTest(testFunction); } while !conMgr.isExecutionDone() { - if conMgr.getAvailableWorkers() != 0 { - conMgr.waitUntilEmptyQueueFilled(); + conMgr.populateExecutionQueues(); if conMgr.getSerialQueueLength() != 0 && conMgr.getAvailableWorkers() == conMgr.getConfiguredWorkers() { TestFunction testFunction = conMgr.getSerialTest(); conMgr.addTestInExecution(testFunction); conMgr.allocateWorker(); executeTest(testFunction); - } - - else if conMgr.getParallelQueueLength() != 0 && conMgr.getSerialQueueLength() == 0 { + } else if conMgr.getParallelQueueLength() != 0 && conMgr.getSerialQueueLength() == 0 { TestFunction testFunction = conMgr.getParallelTest(); conMgr.addTestInExecution(testFunction); conMgr.allocateWorker(); - ParallelExecutionArgs parallelExecutionMetaData = {params: dataDrivenTestParams[testFunction.name]}; - future<(error?)> parallelWaiter = start executeTestIso(testFunction, parallelExecutionMetaData); + DataProviderReturnType? testFunctionArgs = dataDrivenTestParams[testFunction.name]; + if testFunctionArgs is map|readonly[][] { + testFunctionArgs = testFunctionArgs.cloneReadOnly(); + } + future<()> parallelWaiter = start executeTestIso(testFunction, testFunctionArgs); + + // For data driven tests, wait for the worker allocation to complete + // before proceeding to the next test if isDataDrivenTest(dataDrivenTestParams[testFunction.name]) { any _ = check wait parallelWaiter; } } - } - runtime:sleep(0.0001); // sleep is added to yield the strand } - println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n"); } @@ -121,7 +119,6 @@ function executeAfterSuiteFunctions() { function orderTests() returns error? { string[] descendants = []; - foreach TestFunction testFunction in testRegistry.getDependentFunctions() { if !conMgr.isVisited(testFunction.name) && conMgr.isEnabled(testFunction.name) { check restructureTest(testFunction, descendants); @@ -131,7 +128,6 @@ function orderTests() returns error? { function restructureTest(TestFunction testFunction, string[] descendants) returns error? { descendants.push(testFunction.name); - foreach function dependsOnFunction in testFunction.dependsOn { TestFunction dependsOnTestFunction = check testRegistry.getTestFunction(dependsOnFunction); @@ -144,7 +140,6 @@ function restructureTest(TestFunction testFunction, string[] descendants) return + string `but it is either disabled or not included.`; return error(errMsg); } - conMgr.addDependent(dependsOnTestFunction.name, testFunction); // Contains cyclic dependencies @@ -157,7 +152,6 @@ function restructureTest(TestFunction testFunction, string[] descendants) return check restructureTest(dependsOnTestFunction, descendants); } } - conMgr.setEnabled(testFunction.name); conMgr.setVisited(testFunction.name); _ = descendants.pop(); @@ -168,7 +162,6 @@ isolated function printExecutionError(ExecutionError err, string functionSuffix) isolated function getErrorMessage(error err) returns string { string message = err.toBalString(); - string accumulatedTrace = ""; foreach langError:StackFrame stackFrame in err.stackTrace() { accumulatedTrace = accumulatedTrace + "\t" + stackFrame.toString() + "\n"; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal index a6a558036d84..2efcbfefaf2e 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal @@ -14,39 +14,36 @@ // specific language governing permissions and limitations // under the License. -isolated function executeTestIso(TestFunction testFunction, ParallelExecutionArgs parallelExecutionMetaData) returns error? { +isolated function executeTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { if !conMgr.isEnabled(testFunction.name) { - conMgr.setExecutionDone(testFunction.name); + conMgr.setExecutionSuspended(testFunction.name); conMgr.releaseWorker(); return; } error? diagnoseError = testFunction.diagnostics; - DataProviderReturnType? dataProviderParams = parallelExecutionMetaData.params; if diagnoseError is error { - reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(dataProviderParams)); + reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunctionArgs)); println("\n" + testFunction.name + " has failed.\n"); enableExit(); - conMgr.setExecutionDone(testFunction.name); + conMgr.setExecutionSuspended(testFunction.name); conMgr.releaseWorker(); return; } - executeBeforeGroupFunctionsIso(testFunction); executeBeforeEachFunctionsIso(); boolean shouldSkipDependents = false; if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { - if (isDataDrivenTest(dataProviderParams)) { - check executeDataDrivenTestSetIso(testFunction, parallelExecutionMetaData); + if (isDataDrivenTest(testFunctionArgs)) { + executeDataDrivenTestSetIso(testFunction, testFunctionArgs); } else { - shouldSkipDependents = executeNonDataDrivenTestIso(testFunction, parallelExecutionMetaData); + shouldSkipDependents = executeNonDataDrivenTestIso(testFunction, testFunctionArgs); } } else { - reportData.onSkipped(name = testFunction.name, testType = getTestType(dataProviderParams)); + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); shouldSkipDependents = true; } - testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); executeAfterEachFunctionsIso(); executeAfterGroupFunctionsIso(testFunction); @@ -56,7 +53,6 @@ isolated function executeTestIso(TestFunction testFunction, ParallelExecutionArg conMgr.setSkip(dependent.name); }); } - conMgr.setExecutionDone(testFunction.name); conMgr.releaseWorker(); } @@ -85,20 +81,19 @@ isolated function executeBeforeEachFunctionsIso() { } } -isolated function executeDataDrivenTestSetIso(TestFunction testFunction, ParallelExecutionArgs parallelExecutionMetaData) returns error? { - DataProviderReturnType? params = parallelExecutionMetaData.params; +isolated function executeDataDrivenTestSetIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { string[] keys = []; AnyOrErrorOrReadOnlyType[][] values = []; TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; - if params is map { - foreach [string, AnyOrErrorOrReadOnlyType[]] entry in params.entries() { + if testFunctionArgs is map { + foreach [string, AnyOrErrorOrReadOnlyType[]] entry in testFunctionArgs.entries() { keys.push(entry[0]); values.push(entry[1]); } - } else if params is AnyOrErrorOrReadOnlyType[][] { + } else if testFunctionArgs is AnyOrErrorOrReadOnlyType[][] { testType = DATA_DRIVEN_TUPLE_OF_TUPLE; int i = 0; - foreach AnyOrErrorOrReadOnlyType[] entry in params { + foreach AnyOrErrorOrReadOnlyType[] entry in testFunctionArgs { keys.push(i.toString()); values.push(entry); i += 1; @@ -106,37 +101,29 @@ isolated function executeDataDrivenTestSetIso(TestFunction testFunction, Paralle } boolean isIntialJob = true; - while keys.length() != 0 { - if isIntialJob || conMgr.getAvailableWorkers() > 0 { string key = keys.remove(0); ReadOnlyType[] value = values.remove(0); - if !isIntialJob { conMgr.allocateWorker(); } - future<()> _ = start prepareDataDrivenTestIso(testFunction, key, value.clone(), testType); } - isIntialJob = false; - conMgr.waitForWorkers(); } - conMgr.waitForWorkers(); - if !isIntialJob { conMgr.allocateWorker(); } } -isolated function executeNonDataDrivenTestIso(TestFunction testFunction, ParallelExecutionArgs parallelExecutionMetaData) returns boolean { +isolated function executeNonDataDrivenTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { boolean failed = false; boolean beforeFailed = executeBeforeFunctionIso(testFunction); - if (beforeFailed) { + if beforeFailed { conMgr.setSkip(testFunction.name); - reportData.onSkipped(name = testFunction.name, testType = getTestType(parallelExecutionMetaData.params)); + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); return true; } @@ -145,17 +132,15 @@ isolated function executeNonDataDrivenTestIso(TestFunction testFunction, Paralle failed = true; reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); println("\n" + testFunction.name + " has failed.\n"); - } - - else if output { + } else if output { failed = true; } + boolean afterFailed = executeAfterFunctionIso(testFunction); - if (afterFailed) { + if afterFailed { return true; } return failed; - } isolated function executeAfterEachFunctionsIso() { @@ -193,14 +178,11 @@ isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key boolean beforeFailed = executeBeforeFunctionIso(testFunction); if (beforeFailed) { reportData.onSkipped(name = testFunction.name, testType = testType); - } - - else { + } else { executeDataDrivenTestIso(testFunction, key, testType, value); var _ = executeAfterFunctionIso(testFunction); } conMgr.releaseWorker(); - } isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { @@ -215,7 +197,6 @@ isolated function executeDataDrivenTestIso(TestFunction testFunction, string suf println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); enableExit(); } - } isolated function executeBeforeFunctionIso(TestFunction testFunction) returns boolean { @@ -239,17 +220,12 @@ isolated function executeTestFunctionIso(TestFunction testFunction, string suffi reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); println("\n" + testFunction.name + ":" + suffix + " has failed\n"); return true; - } - - else if output is any { + } else if output is any { reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); return false; - } - - else { + } else { enableExit(); return error(getErrorMessage(output), functionName = testFunction.name); - } } @@ -296,7 +272,6 @@ isolated function skipDataDrivenTestIso(TestFunction testFunction, string suffix } } } - } // check if no filterSubTests found for a given prefix @@ -305,9 +280,7 @@ isolated function skipDataDrivenTestIso(TestFunction testFunction, string suffix // if a subtest is found specified if (!suffixMatch) { string[] subTests = testOptions.getFilterSubTest(functionKey); - foreach string subFilter in subTests { - string updatedSubFilter = subFilter; if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { if (subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE)) { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal index e0e3213dee04..1eb1c7aebf56 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal @@ -16,7 +16,7 @@ function executeTest(TestFunction testFunction) { if !conMgr.isEnabled(testFunction.name) { - conMgr.setExecutionDone(testFunction.name); + conMgr.setExecutionSuspended(testFunction.name); conMgr.releaseWorker(); return; } @@ -26,11 +26,10 @@ function executeTest(TestFunction testFunction) { reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(dataDrivenTestParams[testFunction.name])); println("\n" + testFunction.name + " has failed.\n"); enableExit(); - conMgr.setExecutionDone(testFunction.name); + conMgr.setExecutionSuspended(testFunction.name); conMgr.releaseWorker(); return; } - executeBeforeGroupFunctions(testFunction); executeBeforeEachFunctions(); @@ -45,7 +44,6 @@ function executeTest(TestFunction testFunction) { reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); shouldSkipDependents = true; } - testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); executeAfterEachFunctions(); executeAfterGroupFunctions(testFunction); @@ -108,7 +106,6 @@ function executeDataDrivenTestSet(TestFunction testFunction) { AnyOrErrorOrReadOnlyType[] value = values.remove(0); prepareDataDrivenTest(testFunction, key, value, testType); } - } function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { @@ -124,9 +121,7 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { failed = true; reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); println("\n" + testFunction.name + " has failed.\n"); - } - - else if output { + } else if output { failed = true; } boolean afterFailed = executeAfterFunction(testFunction); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index a78c725b30fa..60dc7b96be9b 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -1,4 +1,3 @@ -import ballerina/lang.array; // Copyright (c) 2022 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. // // WSO2 Inc. licenses this file to you under the Apache License, @@ -14,7 +13,7 @@ import ballerina/lang.array; // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -import ballerina/lang.runtime; +import ballerina/lang.array; isolated final TestRegistry testRegistry = new (); isolated final TestRegistry beforeSuiteRegistry = new (); @@ -26,10 +25,6 @@ isolated final GroupRegistry beforeGroupsRegistry = new (); isolated final GroupRegistry afterGroupsRegistry = new (); isolated final GroupStatusRegistry groupStatusRegistry = new (); -type ParallelExecutionArgs record {| - DataProviderReturnType? params; -|}; - type TestFunction record {| string name; function executableFunction; @@ -50,7 +45,7 @@ type TestFunctionMetaData record {| TestFunction[] dependents = []; boolean visited = false; boolean isInExecutionQueue = false; - boolean isExecutionDone = false; + TestCompletionStatus executionCompletionStatus = YET_TO_COMPLETE; |}; isolated class ConcurrentExecutionManager { @@ -69,7 +64,13 @@ isolated class ConcurrentExecutionManager { isolated function setExecutionDone(string functionName) { lock { - self.testMetaData[functionName].isExecutionDone = true; + self.testMetaData[functionName].executionCompletionStatus = COMPLETED; + } + } + + isolated function setExecutionSuspended(string functionName) { + lock { + self.testMetaData[functionName].executionCompletionStatus = SUSPENDED; } } @@ -87,13 +88,12 @@ isolated class ConcurrentExecutionManager { isolated function isEnabled(string functionName) returns boolean { lock { - TestFunctionMetaData? unionResult = self.testMetaData[functionName]; - if unionResult is TestFunctionMetaData { - return unionResult.enabled; + TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; + if testFunctionMetaData is TestFunctionMetaData { + return testFunctionMetaData.enabled; } else { return false; } - } } @@ -105,25 +105,23 @@ isolated class ConcurrentExecutionManager { isolated function isVisited(string functionName) returns boolean { lock { - TestFunctionMetaData? unionResult = self.testMetaData[functionName]; - if unionResult is TestFunctionMetaData { - return unionResult.visited; + TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; + if testFunctionMetaData is TestFunctionMetaData { + return testFunctionMetaData.visited; } else { return false; } - } } isolated function isSkip(string functionName) returns boolean { lock { - TestFunctionMetaData? unionResult = self.testMetaData[functionName]; - if unionResult is TestFunctionMetaData { - return unionResult.skip; + TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; + if testFunctionMetaData is TestFunctionMetaData { + return testFunctionMetaData.skip; } else { return false; } - } } @@ -135,18 +133,18 @@ isolated class ConcurrentExecutionManager { isolated function addDependent(string functionName, TestFunction dependent) { lock { - TestFunctionMetaData? unionResult = self.testMetaData[functionName]; - if unionResult is TestFunctionMetaData { - unionResult.dependents.push(dependent); + TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; + if testFunctionMetaData is TestFunctionMetaData { + testFunctionMetaData.dependents.push(dependent); } } } isolated function getDependents(string functionName) returns TestFunction[] { lock { - TestFunctionMetaData? unionResult = self.testMetaData[functionName]; - if unionResult is TestFunctionMetaData { - return unionResult.dependents.clone(); + TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; + if testFunctionMetaData is TestFunctionMetaData { + return testFunctionMetaData.dependents.clone(); } else { return []; } @@ -229,12 +227,6 @@ isolated class ConcurrentExecutionManager { } } - isolated function waitForWorkers() { - while self.getAvailableWorkers() < 1 { - runtime:sleep(0.0001); // sleep is added to yield the strand - } - } - isolated function getAvailableWorkers() returns int { lock { return self.unAllocatedTestWorkers; @@ -253,37 +245,22 @@ isolated class ConcurrentExecutionManager { } } - isolated function waitUntilEmptyQueueFilled() { - int parallelQueueLength = 0; - int serialQueueLength = 0; - self.populateExecutionQueues(); - lock { - parallelQueueLength = self.parallelTestExecutionList.length(); - serialQueueLength = self.serialTestExecutionList.length(); - } - while parallelQueueLength == 0 && serialQueueLength == 0 { - self.populateExecutionQueues(); - runtime:sleep(0.0001); // sleep is added to yield the strand - if self.isExecutionDone() { - break; - } - } - } - isolated function populateExecutionQueues() { lock { int i = 0; - boolean isExecutionDone = false; + TestCompletionStatus executionCompletionStatus = YET_TO_COMPLETE; while i < self.testsInExecution.length() { TestFunction testInProgress = self.testsInExecution[i]; TestFunctionMetaData? inProgressTestMetaData = self.testMetaData[testInProgress.name]; if inProgressTestMetaData == () { return; } - isExecutionDone = inProgressTestMetaData.isExecutionDone; - if isExecutionDone { + executionCompletionStatus = inProgressTestMetaData.executionCompletionStatus; + if executionCompletionStatus == COMPLETED { inProgressTestMetaData.dependents.reverse().forEach(dependent => self.checkExecutionReadiness(dependent)); _ = self.testsInExecution.remove(i); + } else if executionCompletionStatus == SUSPENDED { + _ = self.testsInExecution.remove(i); } else { i = i + 1; } @@ -293,18 +270,16 @@ isolated class ConcurrentExecutionManager { private isolated function checkExecutionReadiness(TestFunction testFunction) { lock { - TestFunctionMetaData? unionResult = self.testMetaData[testFunction.name]; - if unionResult is TestFunctionMetaData { - unionResult.dependsOnCount -= 1; - if unionResult.dependsOnCount == 0 && unionResult.isInExecutionQueue != true { - unionResult.isInExecutionQueue = true; + TestFunctionMetaData? testFunctionMetaData = self.testMetaData[testFunction.name]; + if testFunctionMetaData is TestFunctionMetaData { + testFunctionMetaData.dependsOnCount -= 1; + if testFunctionMetaData.dependsOnCount == 0 && testFunctionMetaData.isInExecutionQueue != true { + testFunctionMetaData.isInExecutionQueue = true; _ = testFunction.parallelizable ? self.parallelTestExecutionList.push(testFunction) : self.serialTestExecutionList.push(testFunction); } - } } } - } isolated class TestOptions { @@ -439,7 +414,6 @@ isolated class TestOptions { return self.hasFilteredTests; } } - } isolated class TestRegistry { @@ -485,9 +459,7 @@ isolated class TestRegistry { lock { return self.dependentRegistry.clone(); } - } - } isolated class GroupRegistry { @@ -509,7 +481,6 @@ isolated class GroupRegistry { return self.registry.get('group).cloneReadOnly(); } return; - } } } @@ -527,7 +498,6 @@ isolated class GroupStatusRegistry { } isolated function lastExecuted(string 'group) returns boolean { lock { - return self.executedTests.get('group) == self.enabledTests.get('group); } } @@ -578,7 +548,4 @@ isolated class GroupStatusRegistry { return self.totalTests.keys().clone(); } } - } - -isolated function testFunctionsSort(TestFunction testFunction) returns string => testFunction.name; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal index e96d5edc4227..2c757325fbce 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal @@ -210,7 +210,6 @@ isolated function failedTestsReport(ReportData data) { error? err = writeContent(filePath, rerunJson.toString()); if err is error { println(err.message()); - } } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal index 3101d18d8ba4..690c55656c9e 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal @@ -37,3 +37,9 @@ enum TestType { DATA_DRIVEN_TUPLE_OF_TUPLE, DATA_DRIVEN_MAP_OF_TUPLE } + +enum TestCompletionStatus { + YET_TO_COMPLETE, + SUSPENDED, + COMPLETED +} diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index ecf1c24b2514..f0fdb26f9882 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -123,7 +123,7 @@ public void testNonParalallelizableTupleDataProvider() throws BallerinaTestExcep @Test public void testNonIsolatedTestFunction() throws BallerinaTestException, IOException { - String warningDiagnostics = "WARNING: Test function 'testAssertEquals*' cannot be parallelized due to " + + String warningDiagnostics = "WARNING: Test function 'testAssertEquals*' cannot be parallelized, reason: " + "non-isolated test function"; String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-tests"}); String output = balClient.runMainAndReadStdOut("test", args, @@ -146,7 +146,7 @@ public void testNonIsolatedTestFunction() throws BallerinaTestException, IOExcep @Test public void testNonIsolatedTestParameter() throws BallerinaTestException, IOException { - String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized due to " + + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "unsafe test parameters"; String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-test-params"}); String output = balClient.runMainAndReadStdOut("test", args, @@ -166,7 +166,7 @@ public void testNonIsolatedTestParameter() throws BallerinaTestException, IOExce @Test public void testNonIsolatedDataProvider() throws BallerinaTestException, IOException { - String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized due to " + + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated data-provider function"; String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt index 9a4e2a518840..37c810001ad8 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt @@ -25,9 +25,14 @@ testDividingValuesNegative:2 has failed. callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 18 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 27 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt index 525766123d68..3f8c987698b2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -19,13 +19,23 @@ testInvalidDataProvider:0 has failed. [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 209 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index 092f0ecf5fad..5392207c090a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -21,13 +21,23 @@ testInvalidDataProvider2:0 has failed. [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 209 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 03f75dcf1607..a4fade76ebed 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -18,13 +18,23 @@ testInvalidTupleDataProvider:0 has failed. [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 209 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt index f70be3d2fcb0..e1eeacf81eac 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt @@ -25,9 +25,14 @@ testDividingValuesNegative:2 has failed. callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 18 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 27 @@ -46,4 +51,4 @@ testDividingValuesNegative:2 has failed. Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt index 8bf3862f950f..71d7872ccbff 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -19,17 +19,27 @@ testInvalidDataProvider:0 has failed. [fail data provider for the function testInvalidDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 209 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 0 passing 1 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index cbad83abbcd3..1679c2993052 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -21,17 +21,27 @@ testInvalidDataProvider2:0 has failed. [fail data provider for the function testInvalidDataProvider2] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 209 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 0 passing 1 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 6b452602fee8..1cb3cfc72952 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -18,17 +18,27 @@ testInvalidTupleDataProvider:0 has failed. [fail data provider for the function testInvalidTupleDataProvider] error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 446 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 209 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 461 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 233 - callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 222 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 220 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 185 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 175 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 109 + callableName: executeTest moduleName: ballerina.test.0 fileName: nonIsolateExecuter.bal lineNumber: 40 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 81 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 55 + callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 0 passing 1 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal index 3e04d23518d9..e2cc5973b7e2 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal @@ -1,10 +1,9 @@ - -import ballerina/test; import ballerina/lang.runtime; +import ballerina/test; @test:BeforeSuite function beforeSuiteFunc() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @@ -15,361 +14,361 @@ function testFunction() { @test:Config {} function testAssertEquals1() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1]} function testAssertEquals2() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} function testAssertEquals3() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} function testAssertEquals4() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals4]} function testAssertEquals5() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals4, testAssertEquals5], serialExecution: true} function testAssertEquals6() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {} function testAssertEquals7() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals6]} function testAssertEquals8() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals4, testAssertEquals8]} function testAssertEquals9() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals8, testAssertEquals7, testAssertEquals11]} function testAssertEquals10() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {serialExecution: true} function testAssertEquals11() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals7]} function testAssertEquals12() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals6]} function testAssertEquals13() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {serialExecution: true} function testAssertEquals14() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {} function testAssertEquals15() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals13]} function testAssertEquals16() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {} function testAssertEquals17() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals17]} function testAssertEquals18() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals16]} function testAssertEquals19() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals16]} function testAssertEquals20() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals14]} function testAssertEquals21() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals14, testAssertEquals15]} function testAssertEquals22() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals15, testAssertEquals21]} function testAssertEquals23() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals19, testAssertEquals20], serialExecution: true} function testAssertEquals24() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals20, testAssertEquals24, testAssertEquals12, testAssertEquals11]} function testAssertEquals25() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals25, testAssertEquals24]} function testAssertEquals26() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals24, testAssertEquals26]} function testAssertEquals27() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {serialExecution: true} function testAssertEquals28() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals29() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals30() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals31() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals32() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals33() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals34() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals35() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals36() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals37() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals38() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals39() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals40() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals41() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals42() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals43() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals44() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals45() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals46() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals47() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals48() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals49() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals50() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals51() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals52() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals53() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals54() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals55() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals56() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals57() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals58() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals59() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals60() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } @@ -377,7 +376,7 @@ function testAssertEquals60() { @test:AfterSuite function afterSuiteFunc() { - runtime:sleep(0.5); + runtime:sleep(30); test:assertEquals(100, 100); } From c4c178fb819e677f293b90eccb7bd7c9afa90116 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 11 Jan 2024 16:37:30 +0530 Subject: [PATCH 27/37] Separate the common code from isolation and non-isolation --- .../src/main/ballerina/concurrentExecuter.bal | 161 +++++++++ .../src/main/ballerina/execute.bal | 211 ++++++++++++ .../src/main/ballerina/isolateExecuter.bal | 319 ------------------ .../src/main/ballerina/nonIsolateExecuter.bal | 307 ----------------- .../src/main/ballerina/register.bal | 18 +- .../src/main/ballerina/serialExecuter.bal | 148 ++++++++ .../testerina/natives/CommonUtils.java | 3 +- 7 files changed, 528 insertions(+), 639 deletions(-) create mode 100644 misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal delete mode 100644 misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal delete mode 100644 misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal create mode 100644 misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal new file mode 100644 index 000000000000..be716f49a44a --- /dev/null +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal @@ -0,0 +1,161 @@ +// Copyright (c) 2024 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. +// +// WSO2 Inc. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +isolated function executeTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { + if !isTestReadyToExecute(testFunction, testFunctionArgs) { + return; + } + + executeBeforeGroupFunctionsIso(testFunction); + executeBeforeEachFunctionsIso(); + + boolean shouldSkipDependents = false; + if !isSkipFunction(testFunction) { + if (isDataDrivenTest(testFunctionArgs)) { + executeDataDrivenTestSetIso(testFunction, testFunctionArgs); + } else { + shouldSkipDependents = executeNonDataDrivenTestIso(testFunction, testFunctionArgs); + } + } else { + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); + shouldSkipDependents = true; + } + testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); + executeAfterEachFunctionsIso(); + executeAfterGroupFunctionsIso(testFunction); + finishTestExecution(testFunction, shouldSkipDependents); +} + +isolated function executeBeforeGroupFunctionsIso(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); + if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { + handleBeforeGroupOutput(testFunction, 'group, executeFunctionsIso(beforeGroupFunctions, getShouldSkip())); + } + } +} + +isolated function executeBeforeEachFunctionsIso() => + handleBeforeEachOutput(executeFunctionsIso(beforeEachRegistry.getFunctions(), getShouldSkip())); + +isolated function executeDataDrivenTestSetIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { + string[] keys = []; + AnyOrErrorOrReadOnlyType[][] values = []; + TestType testType = prepareDataSet(testFunctionArgs, keys, values); + + boolean isIntialJob = true; + while keys.length() != 0 { + if isIntialJob || conMgr.getAvailableWorkers() > 0 { + string key = keys.remove(0); + ReadOnlyType[] value = values.remove(0); + if !isIntialJob { + conMgr.allocateWorker(); + } + future<()> _ = start prepareDataDrivenTestIso(testFunction, key, value.clone(), testType); + } + isIntialJob = false; + } + + if !isIntialJob { + conMgr.allocateWorker(); + } +} + +isolated function executeNonDataDrivenTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { + if executeBeforeFunctionIso(testFunction) { + conMgr.setSkip(testFunction.name); + reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); + return true; + } + + boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunctionIso(testFunction, "", GENERAL_TEST)); + + if executeAfterFunctionIso(testFunction) { + return true; + } + return failed; +} + +isolated function executeAfterEachFunctionsIso() => + handleAfterEachOutput(executeFunctionsIso(afterEachRegistry.getFunctions(), getShouldSkip())); + +isolated function executeAfterGroupFunctionsIso(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); + if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { + ExecutionError? err = executeFunctionsIso(afterGroupFunctions, + getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); + handleAfterGroupOutput(err); + } + } +} + +isolated function executeFunctionsIso(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { + foreach TestFunction testFunction in testFunctions { + if !skip || testFunction.alwaysRun { + check executeFunctionIso(testFunction); + } + } +} + +isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, ReadOnlyType[] value, TestType testType) { + if executeBeforeFunctionIso(testFunction) { + reportData.onSkipped(name = testFunction.name, testType = testType); + } else { + executeDataDrivenTestIso(testFunction, key, testType, value); + var _ = executeAfterFunctionIso(testFunction); + } + conMgr.releaseWorker(); +} + +isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { + if (skipDataDrivenTest(testFunction, suffix, testType)) { + return; + } + + ExecutionError|boolean err = executeTestFunctionIso(testFunction, suffix, testType, params); + handleDataDrivenTestOutput(err, testFunction, suffix, testType); +} + +isolated function executeBeforeFunctionIso(TestFunction testFunction) returns boolean { + boolean failed = false; + if isBeforeFuncConditionMet(testFunction) { + failed = handleBeforeFunctionOutput(executeFunctionIso(testFunction.before)); + } + return failed; +} + +isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { + any|error output = params == () ? trap function:call(testFunction.executableFunction) + : trap function:call(testFunction.executableFunction, ...params); + return handleTestFuncOutput(output, testFunction, suffix, testType); +} + +isolated function executeAfterFunctionIso(TestFunction testFunction) returns boolean { + boolean failed = false; + if isAfterFuncConditionMet(testFunction) { + failed = handleAfterFunctionOutput(executeFunctionIso(testFunction.after)); + } + return failed; +} + +isolated function executeFunctionIso(TestFunction|function testFunction) returns ExecutionError? { + any|error output = trap function:call((testFunction is function ? testFunction : testFunction.executableFunction)); + if output is error { + enableExit(); + return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); + } +} diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index f246078971fd..ba981ecc52a1 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -215,3 +215,214 @@ isolated function enableExit() { } } +isolated function isTestReadyToExecute(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { + if !conMgr.isEnabled(testFunction.name) { + conMgr.setExecutionSuspended(testFunction.name); + conMgr.releaseWorker(); + return false; + } + + error? diagnoseError = testFunction.diagnostics; + if diagnoseError is error { + reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunctionArgs)); + println("\n" + testFunction.name + " has failed.\n"); + enableExit(); + conMgr.setExecutionSuspended(testFunction.name); + conMgr.releaseWorker(); + return false; + } + return true; +} + +isolated function finishTestExecution(TestFunction testFunction, boolean shouldSkipDependents) { + if shouldSkipDependents { + conMgr.getDependents(testFunction.name).forEach(isolated function(TestFunction dependent) { + conMgr.setSkip(dependent.name); + }); + } + conMgr.setExecutionDone(testFunction.name); + conMgr.releaseWorker(); +} + +isolated function handleBeforeGroupOutput(TestFunction testFunction, string 'group, ExecutionError? err) { + if err is ExecutionError { + conMgr.setSkip(testFunction.name); + groupStatusRegistry.setSkipAfterGroup('group); + enableExit(); + printExecutionError(err, "before test group function for the test"); + } +} + +isolated function handleBeforeEachOutput(ExecutionError? err) { + if err is ExecutionError { + enableShouldSkip(); + enableExit(); + printExecutionError(err, "before each test function for the test"); + } +} + +isolated function handleNonDataDrivenTestOutput(TestFunction testFunction, ExecutionError|boolean output) returns boolean { + boolean failed = false; + if output is ExecutionError { + failed = true; + reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); + println("\n" + testFunction.name + " has failed.\n"); + } else if output { + failed = true; + } + return failed; +} + +isolated function handleAfterEachOutput(ExecutionError? err) { + if err is ExecutionError { + enableShouldSkip(); + enableExit(); + printExecutionError(err, "after each test function for the test"); + } +} + +isolated function handleAfterGroupOutput(ExecutionError? err) { + if err is ExecutionError { + enableExit(); + printExecutionError(err, "after test group function for the test"); + } +} + +isolated function handleDataDrivenTestOutput(ExecutionError|boolean err, TestFunction testFunction, string suffix, + TestType testType) { + if err is ExecutionError { + reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + + "]\n" + getErrorMessage(err), testType = testType); + println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); + enableExit(); + } +} + +isolated function handleBeforeFunctionOutput(ExecutionError? err) returns boolean { + if err is ExecutionError { + enableExit(); + printExecutionError(err, "before test function for the test"); + return true; + } + return false; +} + +isolated function handleAfterFunctionOutput(ExecutionError? err) returns boolean { + if err is ExecutionError { + enableExit(); + printExecutionError(err, "after test function for the test"); + return true; + } + return false; +} + +isolated function handleTestFuncOutput(any|error output, TestFunction testFunction, string suffix, TestType testType) returns ExecutionError|boolean { + if output is TestError { + enableExit(); + reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); + println("\n" + testFunction.name + ":" + suffix + " has failed\n"); + return true; + } else if output is any { + reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); + return false; + } else { + enableExit(); + return error(getErrorMessage(output), functionName = testFunction.name); + } +} + +isolated function prepareDataSet(DataProviderReturnType? testFunctionArgs, string[] keys, + AnyOrErrorOrReadOnlyType[][] values) returns TestType { + TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; + if testFunctionArgs is map { + foreach [string, AnyOrErrorOrReadOnlyType[]] entry in testFunctionArgs.entries() { + keys.push(entry[0]); + values.push(entry[1]); + } + } else if testFunctionArgs is AnyOrErrorOrReadOnlyType[][] { + testType = DATA_DRIVEN_TUPLE_OF_TUPLE; + int i = 0; + foreach AnyOrErrorOrReadOnlyType[] entry in testFunctionArgs { + keys.push(i.toString()); + values.push(entry); + i += 1; + } + } + return testType; +} + +isolated function skipDataDrivenTest(TestFunction testFunction, string suffix, TestType testType) returns boolean { + string functionName = testFunction.name; + if (!testOptions.getHasFilteredTests()) { + return false; + } + TestFunction[] dependents = conMgr.getDependents(functionName); + + // if a dependent in a below level is enabled, this test should run + if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { + return false; + } + string functionKey = functionName; + + // check if prefix matches directly + boolean prefixMatch = testOptions.isFilterSubTestsContains(functionName); + + // if prefix matches to a wildcard + if (!prefixMatch && hasTest(functionName)) { + + // get the matching wildcard + prefixMatch = true; + foreach string filter in testOptions.getFilterTests() { + if (filter.includes(WILDCARD)) { + boolean|error wildCardMatch = matchWildcard(functionKey, filter); + if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { + functionKey = filter; + break; + } + } + } + } + + // check if no filterSubTests found for a given prefix + boolean suffixMatch = testOptions.isFilterSubTestsContains(functionKey); + + // if a subtest is found specified + if (!suffixMatch) { + string[] subTests = testOptions.getFilterSubTest(functionKey); + foreach string subFilter in subTests { + string updatedSubFilter = subFilter; + if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { + if !subFilter.startsWith(SINGLE_QUOTE) || !subFilter.endsWith(SINGLE_QUOTE) { + continue; + } + updatedSubFilter = subFilter.substring(1, subFilter.length() - 1); + } + string|error decodedSubFilter = escapeSpecialCharacters(updatedSubFilter); + updatedSubFilter = decodedSubFilter is string ? decodedSubFilter : updatedSubFilter; + string|error decodedSuffix = escapeSpecialCharacters(suffix); + string updatedSuffix = decodedSuffix is string ? decodedSuffix : suffix; + + boolean wildCardMatchBoolean = false; + if (updatedSubFilter.includes(WILDCARD)) { + boolean|error wildCardMatch = matchWildcard(updatedSuffix, updatedSubFilter); + wildCardMatchBoolean = wildCardMatch is boolean && wildCardMatch; + } + if ((updatedSubFilter == updatedSuffix) || wildCardMatchBoolean) { + suffixMatch = true; + break; + } + } + } + + // do not skip iff both matches + return !(prefixMatch && suffixMatch); +} + +isolated function isBeforeFuncConditionMet(TestFunction testFunction) returns boolean => + testFunction.before is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name); + +isolated function isAfterFuncConditionMet(TestFunction testFunction) returns boolean => + testFunction.after is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name); + +isolated function isSkipFunction(TestFunction testFunction) returns boolean => + conMgr.isSkip(testFunction.name) || getShouldSkip(); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal deleted file mode 100644 index 2efcbfefaf2e..000000000000 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/isolateExecuter.bal +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright (c) 2023 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. -// -// WSO2 Inc. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -isolated function executeTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { - if !conMgr.isEnabled(testFunction.name) { - conMgr.setExecutionSuspended(testFunction.name); - conMgr.releaseWorker(); - return; - } - - error? diagnoseError = testFunction.diagnostics; - if diagnoseError is error { - reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunctionArgs)); - println("\n" + testFunction.name + " has failed.\n"); - enableExit(); - conMgr.setExecutionSuspended(testFunction.name); - conMgr.releaseWorker(); - return; - } - executeBeforeGroupFunctionsIso(testFunction); - executeBeforeEachFunctionsIso(); - - boolean shouldSkipDependents = false; - if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { - if (isDataDrivenTest(testFunctionArgs)) { - executeDataDrivenTestSetIso(testFunction, testFunctionArgs); - } else { - shouldSkipDependents = executeNonDataDrivenTestIso(testFunction, testFunctionArgs); - } - } else { - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); - shouldSkipDependents = true; - } - testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); - executeAfterEachFunctionsIso(); - executeAfterGroupFunctionsIso(testFunction); - - if shouldSkipDependents { - conMgr.getDependents(testFunction.name).forEach(isolated function(TestFunction dependent) { - conMgr.setSkip(dependent.name); - }); - } - conMgr.setExecutionDone(testFunction.name); - conMgr.releaseWorker(); -} - -isolated function executeBeforeGroupFunctionsIso(TestFunction testFunction) { - foreach string 'group in testFunction.groups { - TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); - if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { - ExecutionError? err = executeFunctionsIso(beforeGroupFunctions, getShouldSkip()); - if err is ExecutionError { - conMgr.setSkip(testFunction.name); - groupStatusRegistry.setSkipAfterGroup('group); - enableExit(); - printExecutionError(err, "before test group function for the test"); - } - } - } -} - -isolated function executeBeforeEachFunctionsIso() { - ExecutionError? err = executeFunctionsIso(beforeEachRegistry.getFunctions(), getShouldSkip()); - if err is ExecutionError { - enableShouldSkip(); - enableExit(); - printExecutionError(err, "before each test function for the test"); - } -} - -isolated function executeDataDrivenTestSetIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { - string[] keys = []; - AnyOrErrorOrReadOnlyType[][] values = []; - TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; - if testFunctionArgs is map { - foreach [string, AnyOrErrorOrReadOnlyType[]] entry in testFunctionArgs.entries() { - keys.push(entry[0]); - values.push(entry[1]); - } - } else if testFunctionArgs is AnyOrErrorOrReadOnlyType[][] { - testType = DATA_DRIVEN_TUPLE_OF_TUPLE; - int i = 0; - foreach AnyOrErrorOrReadOnlyType[] entry in testFunctionArgs { - keys.push(i.toString()); - values.push(entry); - i += 1; - } - } - - boolean isIntialJob = true; - while keys.length() != 0 { - if isIntialJob || conMgr.getAvailableWorkers() > 0 { - string key = keys.remove(0); - ReadOnlyType[] value = values.remove(0); - if !isIntialJob { - conMgr.allocateWorker(); - } - future<()> _ = start prepareDataDrivenTestIso(testFunction, key, value.clone(), testType); - } - isIntialJob = false; - } - - if !isIntialJob { - conMgr.allocateWorker(); - } -} - -isolated function executeNonDataDrivenTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { - boolean failed = false; - boolean beforeFailed = executeBeforeFunctionIso(testFunction); - if beforeFailed { - conMgr.setSkip(testFunction.name); - reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); - return true; - } - - ExecutionError|boolean output = executeTestFunctionIso(testFunction, "", GENERAL_TEST); - if output is ExecutionError { - failed = true; - reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); - println("\n" + testFunction.name + " has failed.\n"); - } else if output { - failed = true; - } - - boolean afterFailed = executeAfterFunctionIso(testFunction); - if afterFailed { - return true; - } - return failed; -} - -isolated function executeAfterEachFunctionsIso() { - ExecutionError? err = executeFunctionsIso(afterEachRegistry.getFunctions(), getShouldSkip()); - if err is ExecutionError { - enableShouldSkip(); - enableExit(); - printExecutionError(err, "after each test function for the test"); - } -} - -isolated function executeAfterGroupFunctionsIso(TestFunction testFunction) { - foreach string 'group in testFunction.groups { - TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); - if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { - ExecutionError? err = executeFunctionsIso(afterGroupFunctions, - getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "after test group function for the test"); - } - } - } -} - -isolated function executeFunctionsIso(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { - foreach TestFunction testFunction in testFunctions { - if !skip || testFunction.alwaysRun { - check executeFunctionIso(testFunction); - } - } -} - -isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, ReadOnlyType[] value, TestType testType) { - boolean beforeFailed = executeBeforeFunctionIso(testFunction); - if (beforeFailed) { - reportData.onSkipped(name = testFunction.name, testType = testType); - } else { - executeDataDrivenTestIso(testFunction, key, testType, value); - var _ = executeAfterFunctionIso(testFunction); - } - conMgr.releaseWorker(); -} - -isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { - if (skipDataDrivenTestIso(testFunction, suffix, testType)) { - return; - } - - ExecutionError|boolean err = executeTestFunctionIso(testFunction, suffix, testType, params); - if err is ExecutionError { - reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name - + "]\n" + getErrorMessage(err), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); - enableExit(); - } -} - -isolated function executeBeforeFunctionIso(TestFunction testFunction) returns boolean { - boolean failed = false; - if testFunction.before is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { - ExecutionError? err = executeFunctionIso(testFunction.before); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "before test function for the test"); - failed = true; - } - } - return failed; -} - -isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { - any|error output = params == () ? trap function:call(testFunction.executableFunction) - : trap function:call(testFunction.executableFunction, ...params); - if output is TestError { - enableExit(); - reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed\n"); - return true; - } else if output is any { - reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); - return false; - } else { - enableExit(); - return error(getErrorMessage(output), functionName = testFunction.name); - } -} - -isolated function executeAfterFunctionIso(TestFunction testFunction) returns boolean { - boolean failed = false; - if testFunction.after is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { - ExecutionError? err = executeFunctionIso(testFunction.after); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "after test function for the test"); - failed = true; - } - } - return failed; -} - -isolated function skipDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType) returns boolean { - string functionName = testFunction.name; - if (!testOptions.getHasFilteredTests()) { - return false; - } - TestFunction[] dependents = conMgr.getDependents(functionName); - - // if a dependent in a below level is enabled, this test should run - if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { - return false; - } - string functionKey = functionName; - - // check if prefix matches directly - boolean prefixMatch = testOptions.isFilterSubTestsContains(functionName); - - // if prefix matches to a wildcard - if (!prefixMatch && hasTest(functionName)) { - - // get the matching wildcard - prefixMatch = true; - foreach string filter in testOptions.getFilterTests() { - if (filter.includes(WILDCARD)) { - boolean|error wildCardMatch = matchWildcard(functionKey, filter); - if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { - functionKey = filter; - break; - } - } - } - } - - // check if no filterSubTests found for a given prefix - boolean suffixMatch = testOptions.isFilterSubTestsContains(functionKey); - - // if a subtest is found specified - if (!suffixMatch) { - string[] subTests = testOptions.getFilterSubTest(functionKey); - foreach string subFilter in subTests { - string updatedSubFilter = subFilter; - if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { - if (subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE)) { - updatedSubFilter = subFilter.substring(1, subFilter.length() - 1); - } else { - continue; - } - } - string|error decodedSubFilter = escapeSpecialCharacters(updatedSubFilter); - updatedSubFilter = decodedSubFilter is string ? decodedSubFilter : updatedSubFilter; - string|error decodedSuffix = escapeSpecialCharacters(suffix); - string updatedSuffix = decodedSuffix is string ? decodedSuffix : suffix; - - boolean wildCardMatchBoolean = false; - if (updatedSubFilter.includes(WILDCARD)) { - boolean|error wildCardMatch = matchWildcard(updatedSuffix, updatedSubFilter); - wildCardMatchBoolean = wildCardMatch is boolean && wildCardMatch; - } - if ((updatedSubFilter == updatedSuffix) || wildCardMatchBoolean) { - suffixMatch = true; - break; - } - } - } - - // do not skip iff both matches - return !(prefixMatch && suffixMatch); -} - -isolated function executeFunctionIso(TestFunction|function testFunction) returns ExecutionError? { - any|error output = trap function:call((testFunction is function ? testFunction : testFunction.executableFunction)); - if output is error { - enableExit(); - return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); - } -} diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal deleted file mode 100644 index 1eb1c7aebf56..000000000000 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/nonIsolateExecuter.bal +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright (c) 2023 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. -// -// WSO2 Inc. licenses this file to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -function executeTest(TestFunction testFunction) { - if !conMgr.isEnabled(testFunction.name) { - conMgr.setExecutionSuspended(testFunction.name); - conMgr.releaseWorker(); - return; - } - - error? diagnoseError = testFunction.diagnostics; - if diagnoseError is error { - reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(dataDrivenTestParams[testFunction.name])); - println("\n" + testFunction.name + " has failed.\n"); - enableExit(); - conMgr.setExecutionSuspended(testFunction.name); - conMgr.releaseWorker(); - return; - } - executeBeforeGroupFunctions(testFunction); - executeBeforeEachFunctions(); - - boolean shouldSkipDependents = false; - if !conMgr.isSkip(testFunction.name) && !getShouldSkip() { - if (isDataDrivenTest(dataDrivenTestParams[testFunction.name])) { - executeDataDrivenTestSet(testFunction); - } else { - shouldSkipDependents = executeNonDataDrivenTest(testFunction); - } - } else { - reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); - shouldSkipDependents = true; - } - testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); - executeAfterEachFunctions(); - executeAfterGroupFunctions(testFunction); - - if shouldSkipDependents { - conMgr.getDependents(testFunction.name).forEach(function(TestFunction dependent) { - conMgr.setSkip(dependent.name); - }); - } - conMgr.setExecutionDone(testFunction.name); - conMgr.releaseWorker(); -} - -function executeBeforeGroupFunctions(TestFunction testFunction) { - foreach string 'group in testFunction.groups { - TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); - if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { - ExecutionError? err = executeFunctions(beforeGroupFunctions, getShouldSkip()); - if err is ExecutionError { - conMgr.setSkip(testFunction.name); - groupStatusRegistry.setSkipAfterGroup('group); - enableExit(); - printExecutionError(err, "before test group function for the test"); - } - } - } -} - -function executeBeforeEachFunctions() { - ExecutionError? err = executeFunctions(beforeEachRegistry.getFunctions(), getShouldSkip()); - if err is ExecutionError { - enableShouldSkip(); - enableExit(); - printExecutionError(err, "before each test function for the test"); - } -} - -function executeDataDrivenTestSet(TestFunction testFunction) { - DataProviderReturnType? params = dataDrivenTestParams[testFunction.name]; - string[] keys = []; - AnyOrErrorOrReadOnlyType[][] values = []; - TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; - if params is map { - foreach [string, AnyOrErrorOrReadOnlyType[]] entry in params.entries() { - keys.push(entry[0]); - values.push(entry[1]); - } - } else if params is AnyOrErrorOrReadOnlyType[][] { - testType = DATA_DRIVEN_TUPLE_OF_TUPLE; - int i = 0; - foreach AnyOrErrorOrReadOnlyType[] entry in params { - keys.push(i.toString()); - values.push(entry); - i += 1; - } - } - - while keys.length() != 0 { - string key = keys.remove(0); - AnyOrErrorOrReadOnlyType[] value = values.remove(0); - prepareDataDrivenTest(testFunction, key, value, testType); - } -} - -function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { - boolean failed = false; - boolean beforeFailed = executeBeforeFunction(testFunction); - if (beforeFailed) { - conMgr.setSkip(testFunction.name); - reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); - return true; - } - ExecutionError|boolean output = executeTestFunction(testFunction, "", GENERAL_TEST); - if output is ExecutionError { - failed = true; - reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); - println("\n" + testFunction.name + " has failed.\n"); - } else if output { - failed = true; - } - boolean afterFailed = executeAfterFunction(testFunction); - if (afterFailed) { - return true; - } - return failed; -} - -function executeAfterEachFunctions() { - ExecutionError? err = executeFunctions(afterEachRegistry.getFunctions(), getShouldSkip()); - if err is ExecutionError { - enableShouldSkip(); - enableExit(); - printExecutionError(err, "after each test function for the test"); - } -} - -function executeAfterGroupFunctions(TestFunction testFunction) { - foreach string 'group in testFunction.groups { - TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); - if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { - ExecutionError? err = executeFunctions(afterGroupFunctions, - getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "after test group function for the test"); - } - } - } -} - -function executeFunctions(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { - foreach TestFunction testFunction in testFunctions { - if !skip || testFunction.alwaysRun { - check executeFunction(testFunction); - } - } -} - -function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrErrorOrReadOnlyType[] value, TestType testType) { - boolean beforeFailed = executeBeforeFunction(testFunction); - if (beforeFailed) { - reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); - } else { - executeDataDrivenTest(testFunction, key, testType, value); - var _ = executeAfterFunction(testFunction); - } -} - -function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { - if (skipDataDrivenTest(testFunction, suffix, testType)) { - return; - } - - ExecutionError|boolean err = executeTestFunction(testFunction, suffix, testType, params); - if err is ExecutionError { - reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name - + "]\n" + getErrorMessage(err), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); - enableExit(); - } -} - -function executeBeforeFunction(TestFunction testFunction) returns boolean { - boolean failed = false; - if testFunction.before is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { - ExecutionError? err = executeFunction(testFunction.before); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "before test function for the test"); - failed = true; - } - } - return failed; -} - -function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { - any|error output = params == () ? trap function:call(testFunction.executableFunction) - : trap function:call(testFunction.executableFunction, ...params); - if output is TestError { - enableExit(); - reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed\n"); - return true; - } else if output is any { - reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); - return false; - } else { - enableExit(); - return error(getErrorMessage(output), functionName = testFunction.name); - } -} - -function executeAfterFunction(TestFunction testFunction) returns boolean { - boolean failed = false; - if testFunction.after is function && !getShouldSkip() && !conMgr.isSkip(testFunction.name) { - ExecutionError? err = executeFunction(testFunction.after); - if err is ExecutionError { - enableExit(); - printExecutionError(err, "after test function for the test"); - failed = true; - } - } - return failed; -} - -function skipDataDrivenTest(TestFunction testFunction, string suffix, TestType testType) returns boolean { - string functionName = testFunction.name; - if (!testOptions.getHasFilteredTests()) { - return false; - } - TestFunction[] dependents = conMgr.getDependents(functionName); - - // if a dependent in a below level is enabled, this test should run - if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { - return false; - } - string functionKey = functionName; - - // check if prefix matches directly - boolean prefixMatch = testOptions.isFilterSubTestsContains(functionName); - - // if prefix matches to a wildcard - if (!prefixMatch && hasTest(functionName)) { - - // get the matching wildcard - prefixMatch = true; - foreach string filter in testOptions.getFilterTests() { - if (filter.includes(WILDCARD)) { - boolean|error wildCardMatch = matchWildcard(functionKey, filter); - if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { - functionKey = filter; - break; - } - } - } - } - - // check if no filterSubTests found for a given prefix - boolean suffixMatch = !testOptions.isFilterSubTestsContains(functionKey); - - // if a subtest is found specified - if (!suffixMatch) { - string[] subTests = testOptions.getFilterSubTest(functionKey); - foreach string subFilter in subTests { - - string updatedSubFilter = subFilter; - if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { - if (subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE)) { - updatedSubFilter = subFilter.substring(1, subFilter.length() - 1); - } else { - continue; - } - } - string|error decodedSubFilter = escapeSpecialCharacters(updatedSubFilter); - updatedSubFilter = decodedSubFilter is string ? decodedSubFilter : updatedSubFilter; - string|error decodedSuffix = escapeSpecialCharacters(suffix); - string updatedSuffix = decodedSuffix is string ? decodedSuffix : suffix; - - boolean wildCardMatchBoolean = false; - if (updatedSubFilter.includes(WILDCARD)) { - boolean|error wildCardMatch = matchWildcard(updatedSuffix, updatedSubFilter); - wildCardMatchBoolean = wildCardMatch is boolean && wildCardMatch; - } - if ((updatedSubFilter == updatedSuffix) || wildCardMatchBoolean) { - suffixMatch = true; - break; - } - } - } - - // do not skip iff both matches - return !(prefixMatch && suffixMatch); -} - -function executeFunction(TestFunction|function testFunction) returns ExecutionError? { - any|error output = trap function:call(testFunction is function ? testFunction : testFunction.executableFunction); - if output is error { - enableExit(); - return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); - } -} diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 60dc7b96be9b..0155019f353d 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -49,9 +49,9 @@ type TestFunctionMetaData record {| |}; isolated class ConcurrentExecutionManager { - private TestFunction[] parallelTestExecutionList = []; - private TestFunction[] serialTestExecutionList = []; - private TestFunction[] testsInExecution = []; + private final TestFunction[] parallelTestExecutionList = []; + private final TestFunction[] serialTestExecutionList = []; + private final TestFunction[] testsInExecution = []; private int unAllocatedTestWorkers = 1; private int intialWorkers = 1; private final map testMetaData = {}; @@ -91,9 +91,8 @@ isolated class ConcurrentExecutionManager { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; if testFunctionMetaData is TestFunctionMetaData { return testFunctionMetaData.enabled; - } else { - return false; } + return false; } } @@ -108,9 +107,8 @@ isolated class ConcurrentExecutionManager { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; if testFunctionMetaData is TestFunctionMetaData { return testFunctionMetaData.visited; - } else { - return false; } + return false; } } @@ -119,9 +117,8 @@ isolated class ConcurrentExecutionManager { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; if testFunctionMetaData is TestFunctionMetaData { return testFunctionMetaData.skip; - } else { - return false; } + return false; } } @@ -145,9 +142,8 @@ isolated class ConcurrentExecutionManager { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; if testFunctionMetaData is TestFunctionMetaData { return testFunctionMetaData.dependents.clone(); - } else { - return []; } + return []; } } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal new file mode 100644 index 000000000000..69e56137f3e4 --- /dev/null +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal @@ -0,0 +1,148 @@ +// Copyright (c) 2024 WSO2 Inc. (http://www.wso2.org) All Rights Reserved. +// +// WSO2 Inc. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +function executeTest(TestFunction testFunction) { + if !isTestReadyToExecute(testFunction, dataDrivenTestParams[testFunction.name]) { + return; + } + + executeBeforeGroupFunctions(testFunction); + executeBeforeEachFunctions(); + + boolean shouldSkipDependents = false; + if !isSkipFunction(testFunction) { + if (isDataDrivenTest(dataDrivenTestParams[testFunction.name])) { + executeDataDrivenTestSet(testFunction); + } else { + shouldSkipDependents = executeNonDataDrivenTest(testFunction); + } + } else { + reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); + shouldSkipDependents = true; + } + testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); + executeAfterEachFunctions(); + executeAfterGroupFunctions(testFunction); + finishTestExecution(testFunction, shouldSkipDependents); +} + +function executeBeforeGroupFunctions(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); + if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { + handleBeforeGroupOutput(testFunction, 'group, executeFunctions(beforeGroupFunctions, getShouldSkip())); + } + } +} + +function executeBeforeEachFunctions() => + handleBeforeEachOutput(executeFunctions(beforeEachRegistry.getFunctions(), getShouldSkip())); + +function executeDataDrivenTestSet(TestFunction testFunction) { + DataProviderReturnType? params = dataDrivenTestParams[testFunction.name]; + string[] keys = []; + AnyOrErrorOrReadOnlyType[][] values = []; + TestType testType = prepareDataSet(params, keys, values); + + while keys.length() != 0 { + string key = keys.remove(0); + AnyOrErrorOrReadOnlyType[] value = values.remove(0); + prepareDataDrivenTest(testFunction, key, value, testType); + } +} + +function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { + if executeBeforeFunction(testFunction) { + conMgr.setSkip(testFunction.name); + reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); + return true; + } + boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunctionIso(testFunction, "", GENERAL_TEST)); + + if executeAfterFunction(testFunction) { + return true; + } + return failed; +} + +function executeAfterEachFunctions() => + handleAfterEachOutput(executeFunctions(afterEachRegistry.getFunctions(), getShouldSkip())); + +function executeAfterGroupFunctions(TestFunction testFunction) { + foreach string 'group in testFunction.groups { + TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); + if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { + ExecutionError? err = executeFunctions(afterGroupFunctions, + getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); + handleAfterGroupOutput(err); + } + } +} + +function executeFunctions(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { + foreach TestFunction testFunction in testFunctions { + if !skip || testFunction.alwaysRun { + check executeFunction(testFunction); + } + } +} + +function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrErrorOrReadOnlyType[] value, TestType testType) { + if executeBeforeFunction(testFunction) { + reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); + } else { + executeDataDrivenTest(testFunction, key, testType, value); + var _ = executeAfterFunction(testFunction); + } +} + +function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { + if skipDataDrivenTest(testFunction, suffix, testType) { + return; + } + ExecutionError|boolean err = executeTestFunction(testFunction, suffix, testType, params); + handleDataDrivenTestOutput(err, testFunction, suffix, testType); +} + +function executeBeforeFunction(TestFunction testFunction) returns boolean { + boolean failed = false; + if isBeforeFuncConditionMet(testFunction) { + failed = handleBeforeFunctionOutput(executeFunction(testFunction.before)); + } + return failed; +} + +function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { + any|error output = params == () ? trap function:call(testFunction.executableFunction) + : trap function:call(testFunction.executableFunction, ...params); + return handleTestFuncOutput(output, testFunction, suffix, testType); +} + +function executeAfterFunction(TestFunction testFunction) returns boolean { + boolean failed = false; + if isAfterFuncConditionMet(testFunction) { + failed = handleAfterFunctionOutput(executeFunction(testFunction.after)); + } + return failed; +} + +function executeFunction(TestFunction|function testFunction) returns ExecutionError? { + any|error output = trap function:call(testFunction is function ? testFunction : testFunction.executableFunction); + if output is error { + enableExit(); + return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); + } +} diff --git a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java index b2ed671959e6..509a6ec7011b 100644 --- a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java +++ b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java @@ -32,8 +32,7 @@ */ public class CommonUtils { public static BDecimal currentTimeInMillis() { - long currentTime = System.currentTimeMillis(); - return BDecimal.valueOf(currentTime); + return BDecimal.valueOf(System.currentTimeMillis()); } public static Object isFunctionParamConcurrencySafe(BFunctionPointer func) { From 88e784eb16fdc6a0cb71ccf3a98e74639ccc8b6b Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 11 Jan 2024 17:10:49 +0530 Subject: [PATCH 28/37] Fix build failure --- .../testerina-core/src/main/ballerina/serialExecuter.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal index 69e56137f3e4..5f0d656c3841 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal @@ -70,7 +70,7 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); return true; } - boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunctionIso(testFunction, "", GENERAL_TEST)); + boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunction(testFunction, "", GENERAL_TEST)); if executeAfterFunction(testFunction) { return true; From 988ab03ca4abbfac5d2f0e296fa30710989f117f Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 17 Jan 2024 17:14:47 +0530 Subject: [PATCH 29/37] Add conditions for setup, teardown functions --- .../compiler/TestFunctionVisitor.java | 17 ++++-- .../main/ballerina/annotation_processor.bal | 61 +++++++++++++++++-- .../src/main/ballerina/register.bal | 8 +-- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TestFunctionVisitor.java b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TestFunctionVisitor.java index c76d1188805f..66173548c58b 100644 --- a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TestFunctionVisitor.java +++ b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TestFunctionVisitor.java @@ -39,15 +39,18 @@ public class TestFunctionVisitor extends NodeVisitor { static final List TEST_STATIC_ANNOTATION_NAMES = List.of( "Config", "BeforeSuite", "AfterSuite", "BeforeGroups", "AfterGroups", "BeforeEach", "AfterEach"); + static final String CONFIG_ANNOTATION = "Config"; private static final String TEST_DYNAMIC_ANNOTATION_NAME = "Factory"; private static final String TEST_MODULE_NAME = "test"; - private final List testStaticFunctions; + private final List testSetUpTearDownFunctions; private final List testDynamicFunctions; + private final List testFunctions; public TestFunctionVisitor() { - this.testStaticFunctions = new ArrayList<>(); + this.testSetUpTearDownFunctions = new ArrayList<>(); this.testDynamicFunctions = new ArrayList<>(); + this.testFunctions = new ArrayList<>(); } @Override @@ -73,7 +76,11 @@ public void visit(FunctionDefinitionNode functionDefinitionNode) { String identifier = qualifiedNameReferenceNode.identifier().text(); if (TEST_MODULE_NAME.equals(modulePrefix)) { if (TEST_STATIC_ANNOTATION_NAMES.contains(identifier)) { - testStaticFunctions.add(functionDefinitionNode); + if (CONFIG_ANNOTATION.equals(identifier)) { + testFunctions.add(functionDefinitionNode); + } else { + testSetUpTearDownFunctions.add(functionDefinitionNode); + } } else if (TEST_DYNAMIC_ANNOTATION_NAME.equals(identifier)) { testDynamicFunctions.add(functionDefinitionNode); } @@ -83,7 +90,9 @@ public void visit(FunctionDefinitionNode functionDefinitionNode) { } public List getTestStaticFunctions() { - return this.testStaticFunctions; + List testStaticFunctions = new ArrayList<>(testSetUpTearDownFunctions); + testStaticFunctions.addAll(testFunctions); + return testStaticFunctions; } public List getTestDynamicFunctions() { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 75f97ef1f18a..4a709c87f327 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -17,13 +17,13 @@ type AnnotationProcessor function (string name, function f) returns boolean; AnnotationProcessor[] annotationProcessors = [ - processConfigAnnotation, processBeforeSuiteAnnotation, processAfterSuiteAnnotation, processBeforeEachAnnotation, processAfterEachAnnotation, processBeforeGroupsAnnotation, - processAfterGroupsAnnotation + processAfterGroupsAnnotation, + processConfigAnnotation ]; public function registerTest(string name, function f) { @@ -50,8 +50,8 @@ function processConfigAnnotation(string name, function f) returns boolean { boolean isTestFunctionIsolated = f is isolated function; boolean isDataProviderIsolated = true; boolean isTestFunctionParamSafe = true; - boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated; string[] reasonForSerialExecution = []; + boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated && isBeforeAfterFuncSetIsolated(config, reasonForSerialExecution); DataProviderReturnType? params = (); error? diagnostics = (); if config.dataProvider != () { @@ -59,7 +59,7 @@ function processConfigAnnotation(string name, function f) returns boolean { if providerFn is function () returns (DataProviderReturnType?) { isDataProviderIsolated = (providerFn is isolated function); isTestFunctionParamSafe = isFunctionParamConcurrencySafe(f); - isSatisfiedParallelizableConditions = isTestFunctionIsolated && isDataProviderIsolated && isTestFunctionParamSafe; + isSatisfiedParallelizableConditions = isSatisfiedParallelizableConditions && isDataProviderIsolated && isTestFunctionParamSafe; DataProviderReturnType providerOutput = providerFn(); params = providerOutput; } else { @@ -80,7 +80,7 @@ function processConfigAnnotation(string name, function f) returns boolean { // If the test function is not parallelizable, then print the reason for serial execution. if !isSatisfiedParallelizableConditions && !config.serialExecution && (conMgr.getConfiguredWorkers() > 1) { - println("WARNING: Test function '" + name + "' cannot be parallelized, reason: " + string:'join(",", ...reasonForSerialExecution)); + println("WARNING: Test function '" + name + "' cannot be parallelized, reason: " + string:'join(", ", ...reasonForSerialExecution)); } boolean enabled = config.enable && (filterGroups.length() == 0 ? true : hasGroup(config.groups, filterGroups)) @@ -96,6 +96,57 @@ function processConfigAnnotation(string name, function f) returns boolean { return false; } +function isBeforeAfterFuncSetIsolated(TestConfig config, string[] reasonForSerialExecution) returns boolean { + boolean isBeforeAfterFunctionSetIsolated = true; + if config.before != () { + if !(config.before is isolated function) { + isBeforeAfterFunctionSetIsolated = false; + reasonForSerialExecution.push("non-isolated before function"); + } + } + if config.after != () { + if !(config.after is isolated function) { + isBeforeAfterFunctionSetIsolated = false; + reasonForSerialExecution.push("non-isolated after function"); + } + } + foreach string 'group in config.groups { + TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); + if beforeGroupFunctions != () { + foreach TestFunction beforeGroupFunction in beforeGroupFunctions { + if !(beforeGroupFunction.executableFunction is isolated function) { + isBeforeAfterFunctionSetIsolated = false; + reasonForSerialExecution.push("non-isolated before group function"); + } + } + } + TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); + if afterGroupFunctions != () { + foreach TestFunction afterGroupFunction in afterGroupFunctions { + if !(afterGroupFunction.executableFunction is isolated function) { + isBeforeAfterFunctionSetIsolated = false; + reasonForSerialExecution.push("non-isolated after group function"); + } + } + } + } + TestFunction[] beforeEachFunctions = beforeEachRegistry.getFunctions(); + foreach TestFunction beforeEachFunction in beforeEachFunctions { + if !(beforeEachFunction.executableFunction is isolated function) { + isBeforeAfterFunctionSetIsolated = false; + reasonForSerialExecution.push("non-isolated before each function"); + } + } + TestFunction[] afterEachFunctions = afterEachRegistry.getFunctions(); + foreach TestFunction afterEachFunction in afterEachFunctions { + if !(afterEachFunction.executableFunction is isolated function) { + isBeforeAfterFunctionSetIsolated = false; + reasonForSerialExecution.push("non-isolated after each function"); + } + } + return isBeforeAfterFunctionSetIsolated; +} + function processBeforeSuiteAnnotation(string name, function f) returns boolean { boolean? isTrue = (typeof f).@BeforeSuite; if isTrue == true { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 0155019f353d..19f23b8b7766 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -31,9 +31,9 @@ type TestFunction record {| function? before = (); function? after = (); boolean alwaysRun = false; - string[] groups = []; + readonly & string[] groups = []; error? diagnostics = (); - function[] dependsOn = []; + readonly & function[] dependsOn = []; boolean parallelizable = true; TestConfig? config = (); |} & readonly; @@ -447,7 +447,7 @@ isolated class TestRegistry { isolated function getFunctions() returns TestFunction[] { lock { - return self.rootRegistry.sort(array:ASCENDING, (item) => item.name).cloneReadOnly(); + return self.rootRegistry.sort(array:ASCENDING, (item) => item.name).clone(); } } @@ -474,7 +474,7 @@ isolated class GroupRegistry { isolated function getFunctions(string 'group) returns TestFunction[]? { lock { if self.registry.hasKey('group) { - return self.registry.get('group).cloneReadOnly(); + return self.registry.get('group).clone(); } return; } From f8dc82dab6543f1a7c1b2479e520ef37a8972be6 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Fri, 19 Jan 2024 09:48:44 +0530 Subject: [PATCH 30/37] Fix data provider related failure --- .../src/main/ballerina/execute.bal | 2 +- .../tests/normalTest.bal | 124 +++++++++--------- 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index ba981ecc52a1..45ef819ce8df 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -384,7 +384,7 @@ isolated function skipDataDrivenTest(TestFunction testFunction, string suffix, T } // check if no filterSubTests found for a given prefix - boolean suffixMatch = testOptions.isFilterSubTestsContains(functionKey); + boolean suffixMatch = !testOptions.isFilterSubTestsContains(functionKey); // if a subtest is found specified if (!suffixMatch) { diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal index e2cc5973b7e2..7aeb57fd649a 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/tests/normalTest.bal @@ -3,7 +3,7 @@ import ballerina/test; @test:BeforeSuite function beforeSuiteFunc() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @@ -14,361 +14,361 @@ function testFunction() { @test:Config {} function testAssertEquals1() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1]} function testAssertEquals2() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} function testAssertEquals3() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2]} function testAssertEquals4() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals4]} function testAssertEquals5() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals4, testAssertEquals5], serialExecution: true} function testAssertEquals6() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {} function testAssertEquals7() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals6]} function testAssertEquals8() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals4, testAssertEquals8]} function testAssertEquals9() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals8, testAssertEquals7, testAssertEquals11]} function testAssertEquals10() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {serialExecution: true} function testAssertEquals11() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals7]} function testAssertEquals12() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals6]} function testAssertEquals13() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {serialExecution: true} function testAssertEquals14() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {} function testAssertEquals15() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals13]} function testAssertEquals16() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {} function testAssertEquals17() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals17]} function testAssertEquals18() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals16]} function testAssertEquals19() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals16]} function testAssertEquals20() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals14]} function testAssertEquals21() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals14, testAssertEquals15]} function testAssertEquals22() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals15, testAssertEquals21]} function testAssertEquals23() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals19, testAssertEquals20], serialExecution: true} function testAssertEquals24() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals20, testAssertEquals24, testAssertEquals12, testAssertEquals11]} function testAssertEquals25() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals25, testAssertEquals24]} function testAssertEquals26() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals24, testAssertEquals26]} function testAssertEquals27() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {serialExecution: true} function testAssertEquals28() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals29() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals30() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals31() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals32() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals28]} function testAssertEquals33() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals34() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals35() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals36() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals37() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals38() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals39() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals40() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals41() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals42() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals43() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals44() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals45() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals46() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals47() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals48() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals49() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals50() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals51() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals52() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals53() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals54() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals55() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals56() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals57() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals58() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals59() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @test:Config {dependsOn: [testAssertEquals1, testAssertEquals2, testAssertEquals4, testAssertEquals5, testAssertEquals6, testAssertEquals7, testAssertEquals8, testAssertEquals9, testAssertEquals10, testFunction]} function testAssertEquals60() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } @@ -376,7 +376,7 @@ function testAssertEquals60() { @test:AfterSuite function afterSuiteFunc() { - runtime:sleep(30); + runtime:sleep(0.5); test:assertEquals(100, 100); } From f18010bbcea6204785d637aa712e3b12fae50538 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Fri, 19 Jan 2024 11:50:55 +0530 Subject: [PATCH 31/37] Add more parallelization test cases --- .../main/ballerina/annotation_processor.bal | 8 +- .../test/TestparallelizationTest.java | 155 ++++++++++++++++++ .../isolated-set-up-tear-down/Ballerina.toml | 8 + .../isolated-set-up-tear-down/main.bal | 55 +++++++ .../tests/data_driven_test.bal | 96 +++++++++++ .../non-isolated-after-each/Ballerina.toml | 8 + .../non-isolated-after-each/main.bal | 55 +++++++ .../tests/data_driven_test.bal | 96 +++++++++++ .../non-isolated-after-func/Ballerina.toml | 8 + .../non-isolated-after-func/main.bal | 55 +++++++ .../tests/data_driven_test.bal | 96 +++++++++++ .../non-isolated-after-grp/Ballerina.toml | 8 + .../non-isolated-after-grp/main.bal | 55 +++++++ .../tests/data_driven_test.bal | 96 +++++++++++ .../non-isolated-before-each/Ballerina.toml | 8 + .../non-isolated-before-each/main.bal | 55 +++++++ .../tests/data_driven_test.bal | 96 +++++++++++ .../non-isolated-before-func/Ballerina.toml | 8 + .../non-isolated-before-func/main.bal | 55 +++++++ .../tests/data_driven_test.bal | 96 +++++++++++ .../non-isolated-before-grp/Ballerina.toml | 8 + .../non-isolated-before-grp/main.bal | 55 +++++++ .../tests/data_driven_test.bal | 96 +++++++++++ 23 files changed, 1272 insertions(+), 4 deletions(-) create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/Ballerina.toml create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal create mode 100644 tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index 4a709c87f327..f3e797c30c04 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -116,7 +116,7 @@ function isBeforeAfterFuncSetIsolated(TestConfig config, string[] reasonForSeria foreach TestFunction beforeGroupFunction in beforeGroupFunctions { if !(beforeGroupFunction.executableFunction is isolated function) { isBeforeAfterFunctionSetIsolated = false; - reasonForSerialExecution.push("non-isolated before group function"); + reasonForSerialExecution.push("non-isolated before-group function"); } } } @@ -125,7 +125,7 @@ function isBeforeAfterFuncSetIsolated(TestConfig config, string[] reasonForSeria foreach TestFunction afterGroupFunction in afterGroupFunctions { if !(afterGroupFunction.executableFunction is isolated function) { isBeforeAfterFunctionSetIsolated = false; - reasonForSerialExecution.push("non-isolated after group function"); + reasonForSerialExecution.push("non-isolated after-group function"); } } } @@ -134,14 +134,14 @@ function isBeforeAfterFuncSetIsolated(TestConfig config, string[] reasonForSeria foreach TestFunction beforeEachFunction in beforeEachFunctions { if !(beforeEachFunction.executableFunction is isolated function) { isBeforeAfterFunctionSetIsolated = false; - reasonForSerialExecution.push("non-isolated before each function"); + reasonForSerialExecution.push("non-isolated before-each function"); } } TestFunction[] afterEachFunctions = afterEachRegistry.getFunctions(); foreach TestFunction afterEachFunction in afterEachFunctions { if !(afterEachFunction.executableFunction is isolated function) { isBeforeAfterFunctionSetIsolated = false; - reasonForSerialExecution.push("non-isolated after each function"); + reasonForSerialExecution.push("non-isolated after-each function"); } } return isBeforeAfterFunctionSetIsolated; diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index f0fdb26f9882..c8a403a87ec5 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -184,6 +184,161 @@ public void testNonIsolatedDataProvider() throws BallerinaTestException, IOExcep Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } + @Test + public void testNonIsolatedAfterEach() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + + "non-isolated after-each function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-after-each"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-after-each"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testNonIsolatedBeforeEach() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + + "non-isolated before-each function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-before-each"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-before-each"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testNonIsolatedAfterFunc() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + + "non-isolated after function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-after-func"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-after-func"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testNonIsolatedBeforeFunc() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + + "non-isolated before function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-before-func"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-before-func"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testNonIsolatedAfterGroup() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + + "non-isolated after-group function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-after-grp"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-after-grp"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testNonIsolatedBeforeGroup() throws BallerinaTestException, IOException { + String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + + "non-isolated before-group function"; + String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-before-grp"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + + args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-before-grp"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + + @Test + public void testIsolatedSetUpTearDown() throws BallerinaTestException, IOException { + String[] args = mergeCoverageArgs(new String[]{"--workers=40", "isolated-set-up-tear-down"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + + args = mergeCoverageArgs(new String[]{"--workers=1", "isolated-set-up-tear-down"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(executionTimeW1 > 2 * executionTimeW30 ); + } + + @Test + public void testNegativeWorkerCount() throws BallerinaTestException, IOException { + String warningDiagnostics = "Warning: Workers can not be negative or zero. Test execution " + + "is proceeded with default worker count."; + String[] args = mergeCoverageArgs(new String[]{"--workers=1", "isolated-set-up-tear-down"}); + String output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW30 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + + args = mergeCoverageArgs(new String[]{"--workers=-12", "isolated-set-up-tear-down"}); + output = balClient.runMainAndReadStdOut("test", args, + new HashMap<>(), projectPath, false); + Float executionTimeW1 = getTimeForTestExecution(output); + Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); + Assert.assertTrue(output.contains(warningDiagnostics)); + Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); + } + @Test public void testNonParalallelizableMapDataProvider() throws BallerinaTestException, IOException { String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-parallelisation-map-data-provider"}); diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal new file mode 100644 index 000000000000..3f180e290a4d --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal @@ -0,0 +1,96 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:BeforeSuite +function setup() { + +} + +@test:BeforeEach +function beforeEachFunc() { +} + +@test:BeforeGroups {value: ["g1"]} +function beforeGroups1() { +} + +@test:Config { + dataProvider: mapDataProvider, + before: beforeFunc, + after: afterFunc, + groups: ["g1"] +} + +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.1); +} + +function beforeFunc() { + +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} + +function afterFunc() { + +} + +@test:AfterGroups {value: ["g1"]} +function afterGroups1() { + +} + +@test:AfterEach +function afterEachFunc() { +} + +@test:AfterSuite +function cleanup() { + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal new file mode 100644 index 000000000000..691f5cb68c4a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal @@ -0,0 +1,96 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:BeforeSuite +function setup() { + +} + +@test:BeforeEach +function beforeEachFunc() { +} + +@test:BeforeGroups {value: ["g1"]} +function beforeGroups1() { +} + +@test:Config { + dataProvider: mapDataProvider, + before: beforeFunc, + after: afterFunc, + groups: ["g1"] +} + +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.1); +} + +function beforeFunc() { + +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} + +function afterFunc() { + +} + +@test:AfterGroups {value: ["g1"]} +function afterGroups1() { + +} + +@test:AfterEach +public function afterEachFunc() { +} + +@test:AfterSuite +function cleanup() { + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal new file mode 100644 index 000000000000..a42988819c18 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal @@ -0,0 +1,96 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:BeforeSuite +function setup() { + +} + +@test:BeforeEach +function beforeEachFunc() { +} + +@test:BeforeGroups {value: ["g1"]} +function beforeGroups1() { +} + +@test:Config { + dataProvider: mapDataProvider, + before: beforeFunc, + after: afterFunc, + groups: ["g1"] +} + +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.1); +} + +function beforeFunc() { + +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} + +public function afterFunc() { + +} + +@test:AfterGroups {value: ["g1"]} +function afterGroups1() { + +} + +@test:AfterEach +function afterEachFunc() { +} + +@test:AfterSuite +function cleanup() { + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal new file mode 100644 index 000000000000..08b83b6293f9 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal @@ -0,0 +1,96 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:BeforeSuite +function setup() { + +} + +@test:BeforeEach +function beforeEachFunc() { +} + +@test:BeforeGroups {value: ["g1"]} +function beforeGroups1() { +} + +@test:Config { + dataProvider: mapDataProvider, + before: beforeFunc, + after: afterFunc, + groups: ["g1"] +} + +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.1); +} + +function beforeFunc() { + +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} + +function afterFunc() { + +} + +@test:AfterGroups {value: ["g1"]} +public function afterGroups1() { + +} + +@test:AfterEach +function afterEachFunc() { +} + +@test:AfterSuite +function cleanup() { + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal new file mode 100644 index 000000000000..67e4700c283e --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal @@ -0,0 +1,96 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:BeforeSuite +function setup() { + +} + +@test:BeforeEach +public function beforeEachFunc() { +} + +@test:BeforeGroups {value: ["g1"]} +function beforeGroups1() { +} + +@test:Config { + dataProvider: mapDataProvider, + before: beforeFunc, + after: afterFunc, + groups: ["g1"] +} + +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.1); +} + +function beforeFunc() { + +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} + +function afterFunc() { + +} + +@test:AfterGroups {value: ["g1"]} +function afterGroups1() { + +} + +@test:AfterEach +function afterEachFunc() { +} + +@test:AfterSuite +function cleanup() { + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal new file mode 100644 index 000000000000..9b8ecbca2223 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal @@ -0,0 +1,96 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:BeforeSuite +function setup() { + +} + +@test:BeforeEach +function beforeEachFunc() { +} + +@test:BeforeGroups {value: ["g1"]} +function beforeGroups1() { +} + +@test:Config { + dataProvider: mapDataProvider, + before: beforeFunc, + after: afterFunc, + groups: ["g1"] +} + +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.1); +} + +public function beforeFunc() { + +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} + +function afterFunc() { + +} + +@test:AfterGroups {value: ["g1"]} +function afterGroups1() { + +} + +@test:AfterEach +function afterEachFunc() { +} + +@test:AfterSuite +function cleanup() { + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/Ballerina.toml b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/Ballerina.toml new file mode 100644 index 000000000000..a0611873503a --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/Ballerina.toml @@ -0,0 +1,8 @@ +[package] +org = "luheerathan" +name = "test_parallelisation_test" +version = "0.1.0" +distribution = "2201.3.0-rc3" + +[build-options] +observabilityIncluded = true diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal new file mode 100644 index 000000000000..a37ada63a432 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal @@ -0,0 +1,55 @@ +import ballerina/io; +import ballerina/lang.runtime; +import ballerina/time; + +int employedCount = 20000; +int[] quantities_ = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + + 23, + 24 +]; + +public function main() { + decimal now = time:monotonicNow(); + int count = 10; + + while true { + if count > 0 { + future<()> f = start get_average(); + count -= 1; + } + // int avg = get_average(); + // io:println("Average: ", avg); + + } + +} + +function get_average() { + runtime:sleep(10); + io:println("Helllwo"); + +} diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal new file mode 100644 index 000000000000..1902106b50e3 --- /dev/null +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal @@ -0,0 +1,96 @@ +import ballerina/lang.runtime; +import ballerina/test; + +@test:BeforeSuite +function setup() { + +} + +@test:BeforeEach +function beforeEachFunc() { +} + +@test:BeforeGroups {value: ["g1"]} +public function beforeGroups1() { +} + +@test:Config { + dataProvider: mapDataProvider, + before: beforeFunc, + after: afterFunc, + groups: ["g1"] +} + +function mapDataProviderTest(int value1, int value2, string fruit) returns error? { + test:assertEquals(value1, value2, msg = "The provided values are not equal"); + runtime:sleep(0.1); +} + +function beforeFunc() { + +} + +function mapDataProvider() returns map<[int, int, string]>|error { + map<[int, int, string]> dataSet = { + "banana": [10, 10, "banana"], + "cherry": [5, 5, "cherry"], + "apple": [5, 5, "apple"], + "orange": [5, 5, "orange"], + "carrot": [5, 5, "carrot"], + "lemon": [5, 5, "lemon"], + "tomatto": [5, 5, "tomatto"], + "papaya": [5, 5, "papaya"], + "grapes": [5, 5, "grapes"], + "mango": [5, 5, "mango"], + "pineapple": [5, 5, "pineapple"], + "watermelon": [5, 5, "watermelon"], + "strawberry": [5, 5, "strawberry"], + "melon": [5, 5, "melon"], + "guava": [5, 5, "guava"], + "pomegranate": [5, 5, "pomegranate"], + "jackfruit": [5, 5, "jackfruit"], + "coconut": [5, 5, "coconut"], + "peach": [5, 5, "peach"], + "pear": [5, 5, "pear"], + "plum": [5, 5, "plum"], + "blueberry": [5, 5, "blueberry"], + "raspberry": [5, 5, "raspberry"], + "kiwi": [5, 5, "kiwi"], + "avocado": [5, 5, "avocado"], + "cucumber": [5, 5, "cucumber"], + "pepper": [5, 5, "pepper"], + "onion": [5, 5, "onion"], + "potato": [5, 5, "potato"], + "tomato": [5, 5, "tomato"], + "garlic": [5, 5, "garlic"], + "ginger": [5, 5, "ginger"], + "spinach": [5, 5, "spinach"], + "broccoli": [5, 5, "broccoli"], + "cauliflower": [5, 5, "cauliflower"], + "cabbage": [5, 5, "cabbage"], + "beetroot": [5, 5, "beetroot"], + "celery": [5, 5, "celery"], + "corn": [5, 5, "corn"], + "mushroom": [5, 5, "mushroom"] + + }; + return dataSet; +} + +function afterFunc() { + +} + +@test:AfterGroups {value: ["g1"]} +function afterGroups1() { + +} + +@test:AfterEach +function afterEachFunc() { +} + +@test:AfterSuite +function cleanup() { + +} From a98a48d8cc25e3414bfea6f368038e364b5b1dc4 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Mon, 22 Jan 2024 09:45:20 +0530 Subject: [PATCH 32/37] Modify serial execution starter logic --- .../modules/testerina-core/src/main/ballerina/execute.bal | 8 ++++---- .../testerina-core/src/main/ballerina/register.bal | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 45ef819ce8df..95c0c803001d 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -73,7 +73,7 @@ function executeTests() returns error? { while !conMgr.isExecutionDone() { if conMgr.getAvailableWorkers() != 0 { conMgr.populateExecutionQueues(); - if conMgr.getSerialQueueLength() != 0 && conMgr.getAvailableWorkers() == conMgr.getConfiguredWorkers() { + if conMgr.getSerialQueueLength() != 0 && conMgr.countTestInExecution() == 0 { TestFunction testFunction = conMgr.getSerialTest(); conMgr.addTestInExecution(testFunction); conMgr.allocateWorker(); @@ -217,8 +217,8 @@ isolated function enableExit() { isolated function isTestReadyToExecute(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { if !conMgr.isEnabled(testFunction.name) { - conMgr.setExecutionSuspended(testFunction.name); conMgr.releaseWorker(); + conMgr.setExecutionSuspended(testFunction.name); return false; } @@ -227,8 +227,8 @@ isolated function isTestReadyToExecute(TestFunction testFunction, DataProviderRe reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunctionArgs)); println("\n" + testFunction.name + " has failed.\n"); enableExit(); - conMgr.setExecutionSuspended(testFunction.name); conMgr.releaseWorker(); + conMgr.setExecutionSuspended(testFunction.name); return false; } return true; @@ -240,8 +240,8 @@ isolated function finishTestExecution(TestFunction testFunction, boolean shouldS conMgr.setSkip(dependent.name); }); } - conMgr.setExecutionDone(testFunction.name); conMgr.releaseWorker(); + conMgr.setExecutionDone(testFunction.name); } isolated function handleBeforeGroupOutput(TestFunction testFunction, string 'group, ExecutionError? err) { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 19f23b8b7766..3e6d62c40c6e 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -178,6 +178,12 @@ isolated class ConcurrentExecutionManager { } } + isolated function countTestInExecution() returns int { + lock { + return self.testsInExecution.length(); + } + } + isolated function addTestInExecution(TestFunction testFunction) { lock { self.testsInExecution.push(testFunction); @@ -205,7 +211,6 @@ isolated class ConcurrentExecutionManager { isolated function isExecutionDone() returns boolean { lock { return self.parallelTestExecutionList.length() == 0 && - self.getAvailableWorkers() == self.intialWorkers && self.serialTestExecutionList.length() == 0 && self.testsInExecution.length() == 0; } From 2857e6e42c3c3dc9ec6b1cadfeb67896e338b8ea Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Mon, 29 Jan 2024 17:26:28 +0530 Subject: [PATCH 33/37] Support --parallel instead --worker --- .../io/ballerina/cli/cmd/TestCommand.java | 13 +-- .../cli/task/RunNativeImageTestTask.java | 13 +-- .../io/ballerina/cli/task/RunTestsTask.java | 13 +-- .../resources/cli-help/ballerina-test.help | 3 + .../main/ballerina/annotation_processor.bal | 4 +- .../src/main/ballerina/annotations.bal | 2 +- .../src/main/ballerina/concurrentExecuter.bal | 28 +++--- .../src/main/ballerina/execute.bal | 37 +++----- .../src/main/ballerina/filter.bal | 6 +- .../src/main/ballerina/mock.bal | 8 +- .../src/main/ballerina/register.bal | 38 ++------ .../src/main/ballerina/types.bal | 2 + .../test/TestparallelizationTest.java | 95 +++++++------------ 13 files changed, 94 insertions(+), 168 deletions(-) diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java index 86d692714150..17cee10e564c 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java @@ -132,8 +132,8 @@ public TestCommand() { @CommandLine.Option(names = "--debug", description = "start in remote debugging mode") private String debugPort; - @CommandLine.Option(names = "--workers", description = "maximum number of parallel test jobs", defaultValue = "1") - private int workers; + @CommandLine.Option(names = "--parallel", description = "enable parallel execution", defaultValue = "false") + private boolean isParallelExecution; @CommandLine.Option(names = "--list-groups", description = "list the groups available in the tests") private boolean listGroups; @@ -282,11 +282,6 @@ public void execute() { return; } - if (workers < 1) { - this.outStream.println("\nWarning: Workers can not be negative or zero. Test execution is proceeded " + - "with default worker count.\n"); - } - // Sets the debug port as a system property, which will be used when setting up debug args before running tests. if (this.debugPort != null) { System.setProperty(SYSTEM_PROP_BAL_DEBUG, this.debugPort); @@ -351,10 +346,10 @@ public void execute() { .addTask(new CompileTask(outStream, errStream, false, isPackageModified, buildOptions.enableCache())) // .addTask(new CopyResourcesTask(), listGroups) // merged with CreateJarTask .addTask(new RunTestsTask(outStream, errStream, rerunTests, groupList, disableGroupList, testList, - includes, coverageFormat, moduleMap, listGroups, excludes, cliArgs, workers), + includes, coverageFormat, moduleMap, listGroups, excludes, cliArgs, isParallelExecution), project.buildOptions().nativeImage()) .addTask(new RunNativeImageTestTask(outStream, rerunTests, groupList, disableGroupList, - testList, includes, coverageFormat, moduleMap, listGroups, workers), + testList, includes, coverageFormat, moduleMap, listGroups, isParallelExecution), !project.buildOptions().nativeImage()) .addTask(new DumpBuildTimeTask(outStream), !project.buildOptions().dumpBuildTime()) .build(); diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java index 1346da2492a6..5757a0099203 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java @@ -136,13 +136,13 @@ public void run() { private boolean isRerunTestExecution; private String singleExecTests; private boolean listGroups; - private int testWorkers; + private final boolean isParallelExecution; TestReport testReport; public RunNativeImageTestTask(PrintStream out, boolean rerunTests, String groupList, String disableGroupList, String testList, String includes, String coverageFormat, - Map modules, boolean listGroups, int workers) { + Map modules, boolean listGroups, boolean isParallelExecution) { this.out = out; this.isRerunTestExecution = rerunTests; @@ -156,12 +156,7 @@ public RunNativeImageTestTask(PrintStream out, boolean rerunTests, String groupL singleExecTests = testList; } this.listGroups = listGroups; - - if (workers <= 0) { - testWorkers = DEFAULT_TEST_WORKERS; - } else { - testWorkers = workers; - } + this.isParallelExecution = isParallelExecution; } @@ -564,7 +559,7 @@ private int runTestSuiteWithNativeImage(Package currentPackage, Target target, cmdArgs.add(this.singleExecTests != null ? this.singleExecTests : ""); cmdArgs.add(Boolean.toString(isRerunTestExecution)); cmdArgs.add(Boolean.toString(listGroups)); // 8 - cmdArgs.add(Integer.toString(testWorkers)); + cmdArgs.add(Boolean.toString(isParallelExecution)); builder.command(cmdArgs.toArray(new String[0])); process = builder.start(); diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java index b3c2cad31cfe..47b961488728 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java @@ -74,7 +74,6 @@ import static io.ballerina.cli.utils.TestUtils.writeToTestSuiteJson; import static io.ballerina.projects.util.ProjectConstants.GENERATED_MODULES_ROOT; import static io.ballerina.projects.util.ProjectConstants.MODULES_ROOT; -import static org.ballerinalang.test.runtime.util.TesterinaConstants.DEFAULT_TEST_WORKERS; import static org.ballerinalang.test.runtime.util.TesterinaConstants.FULLY_QULAIFIED_MODULENAME_SEPRATOR; import static org.ballerinalang.test.runtime.util.TesterinaConstants.IGNORE_PATTERN; import static org.ballerinalang.test.runtime.util.TesterinaConstants.MOCK_FN_DELIMITER; @@ -109,7 +108,7 @@ public class RunTestsTask implements Task { private boolean listGroups; private final List cliArgs; - private int testWorkers; + private final boolean isParallelExecution; TestReport testReport; private static final Boolean isWindows = System.getProperty("os.name").toLowerCase(Locale.getDefault()) @@ -126,17 +125,13 @@ public class RunTestsTask implements Task { public RunTestsTask(PrintStream out, PrintStream err, boolean rerunTests, String groupList, String disableGroupList, String testList, String includes, String coverageFormat, Map modules, boolean listGroups, String excludes, String[] cliArgs, - int workers) { + boolean isParallelExecution) { this.out = out; this.err = err; this.isRerunTestExecution = rerunTests; this.cliArgs = List.of(cliArgs); + this.isParallelExecution = isParallelExecution; - if (workers <= 0) { - testWorkers = DEFAULT_TEST_WORKERS; - } else { - testWorkers = workers; - } if (disableGroupList != null) { this.disableGroupList = disableGroupList; @@ -359,7 +354,7 @@ private int runTestSuite(Target target, Package currentPackage, JBallerinaBacken cmdArgs.add(this.singleExecTests != null ? this.singleExecTests : ""); cmdArgs.add(Boolean.toString(isRerunTestExecution)); cmdArgs.add(Boolean.toString(listGroups)); - cmdArgs.add(Integer.toString(testWorkers)); + cmdArgs.add(Boolean.toString(isParallelExecution)); cliArgs.forEach((arg) -> { cmdArgs.add(arg); }); diff --git a/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help b/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help index 494e582d6c20..7cf5f6ef447a 100755 --- a/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help +++ b/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help @@ -90,6 +90,9 @@ OPTIONS --graalvm-build-options Additional build options to be passed to the GraalVM native image. + --parallel + Enable parallel execution of tests. + ARGUMENTS (-Ckey=value)... diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index f3e797c30c04..cb845567ee56 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -79,7 +79,7 @@ function processConfigAnnotation(string name, function f) returns boolean { } // If the test function is not parallelizable, then print the reason for serial execution. - if !isSatisfiedParallelizableConditions && !config.serialExecution && (conMgr.getConfiguredWorkers() > 1) { + if !isSatisfiedParallelizableConditions && !(config.serialExecution == () ? false : true) && conMgr.isParallelExecutionEnabled() { println("WARNING: Test function '" + name + "' cannot be parallelized, reason: " + string:'join(", ", ...reasonForSerialExecution)); } @@ -89,7 +89,7 @@ function processConfigAnnotation(string name, function f) returns boolean { dataDrivenTestParams[name] = params; testRegistry.addFunction(name = name, executableFunction = f, before = config.before, after = config.after, groups = config.groups.cloneReadOnly(), diagnostics = diagnostics, dependsOn = config.dependsOn.cloneReadOnly(), - parallelizable = (!config.serialExecution && isSatisfiedParallelizableConditions && (conMgr.getConfiguredWorkers() > 1)), + parallelizable = (!(config.serialExecution == () ? false : true) && isSatisfiedParallelizableConditions && conMgr.isParallelExecutionEnabled()), config = config.cloneReadOnly()); conMgr.createTestFunctionMetaData(functionName = name, dependsOnCount = config.dependsOn.length(), enabled = enabled); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal index 3fb8e8722261..3f7be3a726ad 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal @@ -30,7 +30,7 @@ public type TestConfig record { function () returns (any|error) before?; function () returns (any|error) after?; function[] dependsOn = []; - boolean serialExecution = false; + serialExecutionType serialExecution = (); }; # Configuration of the function to be mocked. diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal index be716f49a44a..221c8a64f9d0 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal @@ -55,22 +55,23 @@ isolated function executeDataDrivenTestSetIso(TestFunction testFunction, DataPro string[] keys = []; AnyOrErrorOrReadOnlyType[][] values = []; TestType testType = prepareDataSet(testFunctionArgs, keys, values); + map futures = {}; - boolean isIntialJob = true; while keys.length() != 0 { - if isIntialJob || conMgr.getAvailableWorkers() > 0 { - string key = keys.remove(0); - ReadOnlyType[] value = values.remove(0); - if !isIntialJob { - conMgr.allocateWorker(); - } - future<()> _ = start prepareDataDrivenTestIso(testFunction, key, value.clone(), testType); - } - isIntialJob = false; + string key = keys.remove(0); + ReadOnlyType[] value = values.remove(0); + future<()> futureResult = start prepareDataDrivenTestIso(testFunction, key, value.clone(), testType); + futures[key] = futureResult; } - - if !isIntialJob { - conMgr.allocateWorker(); + foreach [string, future] futureResult in futures.entries() { + string suffix = futureResult[0]; + any|error parallelDataProviderResult = wait futureResult[1]; + if parallelDataProviderResult is error { + reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + + "]\n" + getErrorMessage(parallelDataProviderResult), testType = testType); + println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); + enableExit(); + } } } @@ -118,7 +119,6 @@ isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key executeDataDrivenTestIso(testFunction, key, testType, value); var _ = executeAfterFunctionIso(testFunction); } - conMgr.releaseWorker(); } isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index 95c0c803001d..df95231a46a4 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -71,29 +71,19 @@ function executeTests() returns error? { _ = testFunction.parallelizable ? conMgr.addInitialParallelTest(testFunction) : conMgr.addInitialSerialTest(testFunction); } while !conMgr.isExecutionDone() { - if conMgr.getAvailableWorkers() != 0 { - conMgr.populateExecutionQueues(); - if conMgr.getSerialQueueLength() != 0 && conMgr.countTestInExecution() == 0 { - TestFunction testFunction = conMgr.getSerialTest(); - conMgr.addTestInExecution(testFunction); - conMgr.allocateWorker(); - executeTest(testFunction); - } else if conMgr.getParallelQueueLength() != 0 && conMgr.getSerialQueueLength() == 0 { - TestFunction testFunction = conMgr.getParallelTest(); - conMgr.addTestInExecution(testFunction); - conMgr.allocateWorker(); - DataProviderReturnType? testFunctionArgs = dataDrivenTestParams[testFunction.name]; - if testFunctionArgs is map|readonly[][] { - testFunctionArgs = testFunctionArgs.cloneReadOnly(); - } - future<()> parallelWaiter = start executeTestIso(testFunction, testFunctionArgs); - - // For data driven tests, wait for the worker allocation to complete - // before proceeding to the next test - if isDataDrivenTest(dataDrivenTestParams[testFunction.name]) { - any _ = check wait parallelWaiter; - } + conMgr.populateExecutionQueues(); + if conMgr.getSerialQueueLength() != 0 && conMgr.countTestInExecution() == 0 { + TestFunction testFunction = conMgr.getSerialTest(); + conMgr.addTestInExecution(testFunction); + executeTest(testFunction); + } else if conMgr.getParallelQueueLength() != 0 && conMgr.getSerialQueueLength() == 0 { + TestFunction testFunction = conMgr.getParallelTest(); + conMgr.addTestInExecution(testFunction); + DataProviderReturnType? testFunctionArgs = dataDrivenTestParams[testFunction.name]; + if testFunctionArgs is map|readonly[][] { + testFunctionArgs = testFunctionArgs.cloneReadOnly(); } + _ = start executeTestIso(testFunction, testFunctionArgs); } } println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n"); @@ -217,7 +207,6 @@ isolated function enableExit() { isolated function isTestReadyToExecute(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { if !conMgr.isEnabled(testFunction.name) { - conMgr.releaseWorker(); conMgr.setExecutionSuspended(testFunction.name); return false; } @@ -227,7 +216,6 @@ isolated function isTestReadyToExecute(TestFunction testFunction, DataProviderRe reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunctionArgs)); println("\n" + testFunction.name + " has failed.\n"); enableExit(); - conMgr.releaseWorker(); conMgr.setExecutionSuspended(testFunction.name); return false; } @@ -240,7 +228,6 @@ isolated function finishTestExecution(TestFunction testFunction, boolean shouldS conMgr.setSkip(dependent.name); }); } - conMgr.releaseWorker(); conMgr.setExecutionDone(testFunction.name); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index 8f68d29b8c7c..3a1921b40627 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -22,7 +22,7 @@ isolated final TestOptions testOptions = new (); public function setTestOptions(string inTargetPath, string inPackageName, string inModuleName, string inReport, string inCoverage, string inGroups, string inDisableGroups, string inTests, string inRerunFailed, - string inListGroups, string inTestWorkers) { + string inListGroups, string inIsParallelExecution) { testOptions.setModuleName(inModuleName); testOptions.setPackageName(inPackageName); testOptions.setTargetPath(inTargetPath); @@ -32,8 +32,8 @@ public function setTestOptions(string inTargetPath, string inPackageName, string boolean testReport = parseBooleanInput(inReport, "test-report"); boolean codeCoverage = parseBooleanInput(inCoverage, "code-coverage"); listGroups = parseBooleanInput(inListGroups, "list-groups"); - int testWorkers = parseIntegerInput(inTestWorkers, "testWorkers"); - conMgr.setIntialWorkers(testWorkers); + boolean isParallelExecution = parseBooleanInput(inIsParallelExecution, "isParallelExecution"); + conMgr.setParallelExecutionStatus(isParallelExecution); if rerunFailed { error? err = parseRerunJson(); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/mock.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/mock.bal index b9df5b0117e6..729939cc76e5 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/mock.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/mock.bal @@ -228,10 +228,10 @@ public isolated function when(MockFunction mockFunction) returns FunctionStub { } # Represents a MockFunction object. -public class MockFunction { - string functionToMock = ""; - string functionToMockPackage = ""; - string[] mockFunctionClasses = []; +public isolated class MockFunction { + private string functionToMock = ""; + private string functionToMockPackage = ""; + private string[] mockFunctionClasses = []; } # Represents an object that allows stubbing function invocations. diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 3e6d62c40c6e..9eded92c718b 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -52,9 +52,8 @@ isolated class ConcurrentExecutionManager { private final TestFunction[] parallelTestExecutionList = []; private final TestFunction[] serialTestExecutionList = []; private final TestFunction[] testsInExecution = []; - private int unAllocatedTestWorkers = 1; - private int intialWorkers = 1; private final map testMetaData = {}; + private boolean isParalleleExecutionEnabled = false; isolated function createTestFunctionMetaData(string functionName, *TestFunctionMetaData intialMetaData) { lock { @@ -147,10 +146,15 @@ isolated class ConcurrentExecutionManager { } } - isolated function setIntialWorkers(int workers) { + isolated function setParallelExecutionStatus(boolean isParalleleExecutionEnabled) { lock { - self.intialWorkers = workers; - self.unAllocatedTestWorkers = workers; + self.isParalleleExecutionEnabled = isParalleleExecutionEnabled; + } + } + + isolated function isParallelExecutionEnabled() returns boolean { + lock { + return self.isParalleleExecutionEnabled; } } @@ -190,12 +194,6 @@ isolated class ConcurrentExecutionManager { } } - isolated function getConfiguredWorkers() returns int { - lock { - return self.intialWorkers; - } - } - isolated function getSerialQueueLength() returns int { lock { return self.serialTestExecutionList.length(); @@ -216,24 +214,6 @@ isolated class ConcurrentExecutionManager { } } - isolated function allocateWorker() { - lock { - self.unAllocatedTestWorkers -= 1; - } - } - - isolated function releaseWorker() { - lock { - self.unAllocatedTestWorkers += 1; - } - } - - isolated function getAvailableWorkers() returns int { - lock { - return self.unAllocatedTestWorkers; - } - } - isolated function getParallelTest() returns TestFunction { lock { return self.parallelTestExecutionList.remove(0); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal index 690c55656c9e..aca163b03031 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal @@ -26,6 +26,8 @@ public type TestError distinct error; type ExecutionError distinct error; +public type serialExecutionType true|(); + type ModuleRerunJson record {| string[] testNames; map testModuleNames; diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index c8a403a87ec5..3029b6125a11 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -43,29 +43,28 @@ public void setup() throws BallerinaTestException { @Test public void testParallelization() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-simple-test"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-simple-test"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("61 passing") && output.contains("0 failing")); - args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-simple-test", }); + args = mergeCoverageArgs(new String[]{"parallelisation-simple-test"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("61 passing") && output.contains("0 failing")); Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30); - } @Test public void testNonParallelizable() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-serialExecution"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-serialExecution"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("61 passing") && output.contains("0 failing")); - args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-serialExecution", }); + args = mergeCoverageArgs(new String[]{"parallelisation-serialExecution"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); @@ -76,12 +75,12 @@ public void testNonParallelizable() throws BallerinaTestException, IOException { @Test public void testParalallelizableTupleDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-tuple-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-tuple-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("30 passing") && output.contains("0 failing")); - args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-tuple-data-provider", }); + args = mergeCoverageArgs(new String[]{"parallelisation-tuple-data-provider"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); @@ -91,13 +90,13 @@ public void testParalallelizableTupleDataProvider() throws BallerinaTestExceptio @Test public void testParalallelizableMapDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-map-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-map-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); - args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", }); + args = mergeCoverageArgs(new String[]{"parallelisation-map-data-provider"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); @@ -107,13 +106,13 @@ public void testParalallelizableMapDataProvider() throws BallerinaTestException, @Test public void testNonParalallelizableTupleDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-parallelisation-tuple-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-parallelisation-tuple-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("30 passing") && output.contains("0 failing")); - args = mergeCoverageArgs(new String[]{"--workers=1", "non-parallelisation-tuple-data-provider"}); + args = mergeCoverageArgs(new String[]{"non-parallelisation-tuple-data-provider"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); @@ -125,7 +124,7 @@ public void testNonParalallelizableTupleDataProvider() throws BallerinaTestExcep public void testNonIsolatedTestFunction() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'testAssertEquals*' cannot be parallelized, reason: " + "non-isolated test function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-tests"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-tests"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -133,14 +132,11 @@ public void testNonIsolatedTestFunction() throws BallerinaTestException, IOExcep for (int testNo = 1; testNo < 61; testNo++) { Assert.assertTrue(output.contains(warningDiagnostics.replace("*", Integer.toString(testNo)))); } - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-tests"}); + args = mergeCoverageArgs(new String[]{"non-isolated-tests"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("60 passing") && output.contains("0 failing")); - for (int testNo = 1; testNo < 61; testNo++) { - Assert.assertTrue(output.contains(warningDiagnostics.replace("*", Integer.toString(testNo)))); - } Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -148,19 +144,18 @@ public void testNonIsolatedTestFunction() throws BallerinaTestException, IOExcep public void testNonIsolatedTestParameter() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "unsafe test parameters"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-test-params"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-test-params"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-test-params"}); + args = mergeCoverageArgs(new String[]{"non-isolated-test-params"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -168,19 +163,18 @@ public void testNonIsolatedTestParameter() throws BallerinaTestException, IOExce public void testNonIsolatedDataProvider() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated data-provider function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-data-provider"}); + args = mergeCoverageArgs(new String[]{"non-isolated-data-provider"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -188,19 +182,18 @@ public void testNonIsolatedDataProvider() throws BallerinaTestException, IOExcep public void testNonIsolatedAfterEach() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated after-each function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-after-each"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-after-each"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-after-each"}); + args = mergeCoverageArgs(new String[]{"non-isolated-after-each"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -208,19 +201,18 @@ public void testNonIsolatedAfterEach() throws BallerinaTestException, IOExceptio public void testNonIsolatedBeforeEach() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated before-each function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-before-each"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-before-each"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-before-each"}); + args = mergeCoverageArgs(new String[]{"non-isolated-before-each"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -228,19 +220,18 @@ public void testNonIsolatedBeforeEach() throws BallerinaTestException, IOExcepti public void testNonIsolatedAfterFunc() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated after function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-after-func"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-after-func"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-after-func"}); + args = mergeCoverageArgs(new String[]{"non-isolated-after-func"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -248,19 +239,18 @@ public void testNonIsolatedAfterFunc() throws BallerinaTestException, IOExceptio public void testNonIsolatedBeforeFunc() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated before function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-before-func"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-before-func"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-before-func"}); + args = mergeCoverageArgs(new String[]{"non-isolated-before-func"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -268,19 +258,18 @@ public void testNonIsolatedBeforeFunc() throws BallerinaTestException, IOExcepti public void testNonIsolatedAfterGroup() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated after-group function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-after-grp"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-after-grp"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-after-grp"}); + args = mergeCoverageArgs(new String[]{"non-isolated-after-grp"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @@ -288,31 +277,30 @@ public void testNonIsolatedAfterGroup() throws BallerinaTestException, IOExcepti public void testNonIsolatedBeforeGroup() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated before-group function"; - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-isolated-before-grp"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-before-grp"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); Assert.assertTrue(output.contains(warningDiagnostics)); - args = mergeCoverageArgs(new String[]{"--workers=2", "non-isolated-before-grp"}); + args = mergeCoverageArgs(new String[]{"non-isolated-before-grp"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); } @Test public void testIsolatedSetUpTearDown() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=40", "isolated-set-up-tear-down"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "isolated-set-up-tear-down"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - args = mergeCoverageArgs(new String[]{"--workers=1", "isolated-set-up-tear-down"}); + args = mergeCoverageArgs(new String[]{"isolated-set-up-tear-down"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); @@ -320,34 +308,15 @@ public void testIsolatedSetUpTearDown() throws BallerinaTestException, IOExcepti Assert.assertTrue(executionTimeW1 > 2 * executionTimeW30 ); } - @Test - public void testNegativeWorkerCount() throws BallerinaTestException, IOException { - String warningDiagnostics = "Warning: Workers can not be negative or zero. Test execution " + - "is proceeded with default worker count."; - String[] args = mergeCoverageArgs(new String[]{"--workers=1", "isolated-set-up-tear-down"}); - String output = balClient.runMainAndReadStdOut("test", args, - new HashMap<>(), projectPath, false); - Float executionTimeW30 = getTimeForTestExecution(output); - Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - - args = mergeCoverageArgs(new String[]{"--workers=-12", "isolated-set-up-tear-down"}); - output = balClient.runMainAndReadStdOut("test", args, - new HashMap<>(), projectPath, false); - Float executionTimeW1 = getTimeForTestExecution(output); - Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(output.contains(warningDiagnostics)); - Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000); - } - @Test public void testNonParalallelizableMapDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-parallelisation-map-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-parallelisation-map-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("12 passing") && output.contains("28 failing")); - args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", }); + args = mergeCoverageArgs(new String[]{"parallelisation-map-data-provider"}); output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); From bf2a24d7c04f408ddb7056eb74b6871de5d2720d Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Fri, 16 Feb 2024 11:16:10 +0530 Subject: [PATCH 34/37] Address the review --- .../cli/task/RunNativeImageTestTask.java | 1 - .../io/ballerina/cli/task/RunTestsTask.java | 1 - .../io/ballerina/cli/utils/NativeUtils.java | 22 ++--- .../resources/cli-help/ballerina-test.help | 1 - .../TesterinaCompilerPluginConstants.java | 2 +- .../TesterinaCompilerPluginUtils.java | 4 +- .../main/ballerina/annotation_processor.bal | 45 ++++++---- .../src/main/ballerina/annotations.bal | 2 +- .../src/main/ballerina/concurrentExecuter.bal | 28 +++--- .../src/main/ballerina/execute.bal | 46 +++++----- .../src/main/ballerina/filter.bal | 1 + .../src/main/ballerina/register.bal | 85 ++++++++----------- .../src/main/ballerina/report.bal | 13 ++- .../src/main/ballerina/serialExecuter.bal | 11 ++- .../src/main/ballerina/types.bal | 8 +- .../testerina/natives/CommonUtils.java | 7 +- .../test/TestparallelizationTest.java | 33 +++---- .../isolated-set-up-tear-down/main.bal | 53 ------------ .../tests/data_driven_test.bal | 5 -- .../non-isolated-after-each/main.bal | 53 ------------ .../tests/data_driven_test.bal | 4 - .../non-isolated-after-func/main.bal | 53 ------------ .../tests/data_driven_test.bal | 3 - .../non-isolated-after-grp/main.bal | 53 ------------ .../tests/data_driven_test.bal | 5 -- .../non-isolated-before-each/main.bal | 53 ------------ .../tests/data_driven_test.bal | 5 -- .../non-isolated-before-func/main.bal | 53 ------------ .../tests/data_driven_test.bal | 5 -- .../non-isolated-before-grp/main.bal | 53 ------------ .../tests/data_driven_test.bal | 5 -- .../non-isolated-data-provider/main.bal | 53 ------------ .../non-isolated-test-params/main.bal | 53 ------------ .../non-isolated-tests/main.bal | 53 ------------ .../main.bal | 53 ------------ .../main.bal | 53 ------------ .../main.bal | 53 ------------ .../parallelisation-serialExecution/main.bal | 53 ------------ .../tests/normalTest.bal | 3 +- .../parallelisation-simple-test/main.bal | 53 ------------ .../main.bal | 53 ------------ 41 files changed, 150 insertions(+), 1043 deletions(-) diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java index 5757a0099203..d952262c96f9 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunNativeImageTestTask.java @@ -87,7 +87,6 @@ import static java.util.Objects.requireNonNull; import static org.ballerinalang.test.runtime.util.TesterinaConstants.CACHE_DIR; import static org.ballerinalang.test.runtime.util.TesterinaConstants.CLASS_EXTENSION; -import static org.ballerinalang.test.runtime.util.TesterinaConstants.DEFAULT_TEST_WORKERS; import static org.ballerinalang.test.runtime.util.TesterinaConstants.DOT; import static org.ballerinalang.test.runtime.util.TesterinaConstants.DOT_REPLACER; import static org.ballerinalang.test.runtime.util.TesterinaConstants.HYPHEN; diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java index 47b961488728..eb673698d543 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/task/RunTestsTask.java @@ -132,7 +132,6 @@ public RunTestsTask(PrintStream out, PrintStream err, boolean rerunTests, String this.cliArgs = List.of(cliArgs); this.isParallelExecution = isParallelExecution; - if (disableGroupList != null) { this.disableGroupList = disableGroupList; } else if (groupList != null) { diff --git a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java index 60fd0d971828..cf931cbc3e5d 100644 --- a/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java +++ b/cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/NativeUtils.java @@ -65,17 +65,17 @@ public class NativeUtils { private static final ReflectConfigClassMethod REFLECTION_CONFIG_EXECUTE_METHOD = new ReflectConfigClassMethod( TEST_EXEC_FUNCTION, new String[]{"io.ballerina.runtime.internal.scheduling.Strand", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString", - "io.ballerina.runtime.api.values.BString" + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString", + "io.ballerina.runtime.api.values.BString" }); //Add dynamically loading classes and methods to reflection config diff --git a/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help b/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help index 7cf5f6ef447a..bc93df3da266 100755 --- a/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help +++ b/cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help @@ -93,7 +93,6 @@ OPTIONS --parallel Enable parallel execution of tests. - ARGUMENTS (-Ckey=value)... The list of configurable variables for the Ballerina program. diff --git a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java index c0f8bb2990b1..c8cab84384ee 100644 --- a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java +++ b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginConstants.java @@ -46,7 +46,7 @@ public class TesterinaCompilerPluginConstants { public static final String TESTS_PARAMETER = "tests"; public static final String RERUN_FAILED_PARAMETER = "rerunFailed"; public static final String LIST_GROUPS_PARAMETER = "listGroups"; - public static final String PARALLEL_TEST_JOB_COUNT_PARAMETER = "testWorkers"; + public static final String PARALLEL_EXECUTION_PARAMETER = "parallelExecution"; private TesterinaCompilerPluginConstants() {} } diff --git a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java index 5596c3f6ee3f..f6e099bd7c69 100644 --- a/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java +++ b/misc/testerina/modules/testerina-compiler-plugin/src/main/java/org/ballerinalang/testerina/compiler/TesterinaCompilerPluginUtils.java @@ -79,7 +79,7 @@ public static void addSetTestOptionsCall(List statements) { getPositionalArg(TesterinaCompilerPluginConstants.TESTS_PARAMETER), getPositionalArg(TesterinaCompilerPluginConstants.RERUN_FAILED_PARAMETER), getPositionalArg(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER), - getPositionalArg(TesterinaCompilerPluginConstants.PARALLEL_TEST_JOB_COUNT_PARAMETER))))); + getPositionalArg(TesterinaCompilerPluginConstants.PARALLEL_EXECUTION_PARAMETER))))); } public static void addStartSuiteCall(List statements) { @@ -287,7 +287,7 @@ public static FunctionSignatureNode getFunctionSignature() { NodeFactory.createToken(SyntaxKind.COMMA_TOKEN), getStringParameter(TesterinaCompilerPluginConstants.LIST_GROUPS_PARAMETER), NodeFactory.createToken(SyntaxKind.COMMA_TOKEN), - getStringParameter(TesterinaCompilerPluginConstants.PARALLEL_TEST_JOB_COUNT_PARAMETER)), + getStringParameter(TesterinaCompilerPluginConstants.PARALLEL_EXECUTION_PARAMETER)), NodeFactory.createToken(SyntaxKind.CLOSE_PAREN_TOKEN), returnTypeDescriptorNode); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index cb845567ee56..d0d51b01f192 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -51,7 +51,8 @@ function processConfigAnnotation(string name, function f) returns boolean { boolean isDataProviderIsolated = true; boolean isTestFunctionParamSafe = true; string[] reasonForSerialExecution = []; - boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated && isBeforeAfterFuncSetIsolated(config, reasonForSerialExecution); + boolean isSatisfiedParallelizableConditions = isTestFunctionIsolated && + isBeforeAfterFuncSetIsolated(config, reasonForSerialExecution); DataProviderReturnType? params = (); error? diagnostics = (); if config.dataProvider != () { @@ -59,7 +60,9 @@ function processConfigAnnotation(string name, function f) returns boolean { if providerFn is function () returns (DataProviderReturnType?) { isDataProviderIsolated = (providerFn is isolated function); isTestFunctionParamSafe = isFunctionParamConcurrencySafe(f); - isSatisfiedParallelizableConditions = isSatisfiedParallelizableConditions && isDataProviderIsolated && isTestFunctionParamSafe; + isSatisfiedParallelizableConditions = isSatisfiedParallelizableConditions + && isDataProviderIsolated && isTestFunctionParamSafe; + DataProviderReturnType providerOutput = providerFn(); params = providerOutput; } else { @@ -79,51 +82,57 @@ function processConfigAnnotation(string name, function f) returns boolean { } // If the test function is not parallelizable, then print the reason for serial execution. - if !isSatisfiedParallelizableConditions && !(config.serialExecution == () ? false : true) && conMgr.isParallelExecutionEnabled() { - println("WARNING: Test function '" + name + "' cannot be parallelized, reason: " + string:'join(", ", ...reasonForSerialExecution)); + if !isSatisfiedParallelizableConditions && !(config?.serialExecution == () ? false : true) + && conMgr.isParallelExecutionEnabled() { + println(string `WARNING: Test function '${name}' cannot be parallelized, reason: ${string:'join(", ", ...reasonForSerialExecution)}`); } boolean enabled = config.enable && (filterGroups.length() == 0 ? true : hasGroup(config.groups, filterGroups)) - && (filterDisableGroups.length() == 0 ? true : !hasGroup(config.groups, filterDisableGroups)) && hasTest(name); + && (filterDisableGroups.length() == 0 ? true : !hasGroup(config.groups, filterDisableGroups)) + && hasTest(name); config.groups.forEach('group => groupStatusRegistry.incrementTotalTest('group, enabled)); dataDrivenTestParams[name] = params; testRegistry.addFunction(name = name, executableFunction = f, before = config.before, - after = config.after, groups = config.groups.cloneReadOnly(), diagnostics = diagnostics, dependsOn = config.dependsOn.cloneReadOnly(), - parallelizable = (!(config.serialExecution == () ? false : true) && isSatisfiedParallelizableConditions && conMgr.isParallelExecutionEnabled()), + after = config.after, groups = config.groups.cloneReadOnly(), diagnostics = diagnostics, + dependsOn = config.dependsOn.cloneReadOnly(), serialExecution = ((config?.serialExecution != ()) + || !isSatisfiedParallelizableConditions || !conMgr.isParallelExecutionEnabled()), config = config.cloneReadOnly()); - conMgr.createTestFunctionMetaData(functionName = name, dependsOnCount = config.dependsOn.length(), enabled = enabled); + conMgr.createTestFunctionMetaData(functionName = name, dependsOnCount = config.dependsOn.length(), + enabled = enabled); } return false; } function isBeforeAfterFuncSetIsolated(TestConfig config, string[] reasonForSerialExecution) returns boolean { boolean isBeforeAfterFunctionSetIsolated = true; - if config.before != () { - if !(config.before is isolated function) { + (function () returns any|error)? before = config.before; + if before !is () { + if before !is isolated function () returns any|error { isBeforeAfterFunctionSetIsolated = false; reasonForSerialExecution.push("non-isolated before function"); } } - if config.after != () { - if !(config.after is isolated function) { + (function () returns any|error)? after = config.after; + if after !is () { + if after !is isolated function () returns any|error { isBeforeAfterFunctionSetIsolated = false; reasonForSerialExecution.push("non-isolated after function"); } } foreach string 'group in config.groups { TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); - if beforeGroupFunctions != () { + if beforeGroupFunctions !is () { foreach TestFunction beforeGroupFunction in beforeGroupFunctions { - if !(beforeGroupFunction.executableFunction is isolated function) { + if beforeGroupFunction.executableFunction !is isolated function { isBeforeAfterFunctionSetIsolated = false; reasonForSerialExecution.push("non-isolated before-group function"); } } } TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); - if afterGroupFunctions != () { + if afterGroupFunctions !is () { foreach TestFunction afterGroupFunction in afterGroupFunctions { - if !(afterGroupFunction.executableFunction is isolated function) { + if afterGroupFunction.executableFunction !is isolated function { // TODO: move exaclamation before is isBeforeAfterFunctionSetIsolated = false; reasonForSerialExecution.push("non-isolated after-group function"); } @@ -132,14 +141,14 @@ function isBeforeAfterFuncSetIsolated(TestConfig config, string[] reasonForSeria } TestFunction[] beforeEachFunctions = beforeEachRegistry.getFunctions(); foreach TestFunction beforeEachFunction in beforeEachFunctions { - if !(beforeEachFunction.executableFunction is isolated function) { + if beforeEachFunction.executableFunction !is isolated function { isBeforeAfterFunctionSetIsolated = false; reasonForSerialExecution.push("non-isolated before-each function"); } } TestFunction[] afterEachFunctions = afterEachRegistry.getFunctions(); foreach TestFunction afterEachFunction in afterEachFunctions { - if !(afterEachFunction.executableFunction is isolated function) { + if afterEachFunction.executableFunction !is isolated function { isBeforeAfterFunctionSetIsolated = false; reasonForSerialExecution.push("non-isolated after-each function"); } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal index 3f7be3a726ad..cbeec74b3653 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotations.bal @@ -30,7 +30,7 @@ public type TestConfig record { function () returns (any|error) before?; function () returns (any|error) after?; function[] dependsOn = []; - serialExecutionType serialExecution = (); + true serialExecution?; }; # Configuration of the function to be mocked. diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal index 221c8a64f9d0..0ef70274366e 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal @@ -18,10 +18,8 @@ isolated function executeTestIso(TestFunction testFunction, DataProviderReturnTy if !isTestReadyToExecute(testFunction, testFunctionArgs) { return; } - executeBeforeGroupFunctionsIso(testFunction); executeBeforeEachFunctionsIso(); - boolean shouldSkipDependents = false; if !isSkipFunction(testFunction) { if (isDataDrivenTest(testFunctionArgs)) { @@ -53,23 +51,24 @@ isolated function executeBeforeEachFunctionsIso() => isolated function executeDataDrivenTestSetIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { string[] keys = []; - AnyOrErrorOrReadOnlyType[][] values = []; + AnyOrError[][] values = []; TestType testType = prepareDataSet(testFunctionArgs, keys, values); - map futures = {}; - + map futuresMap = {}; while keys.length() != 0 { string key = keys.remove(0); - ReadOnlyType[] value = values.remove(0); - future<()> futureResult = start prepareDataDrivenTestIso(testFunction, key, value.clone(), testType); - futures[key] = futureResult; + final readonly & readonly[] readOnlyVal = from any|error item in values.remove(0) + where item is readonly + select item; + future<()> futureResult = start prepareDataDrivenTestIso(testFunction, key, readOnlyVal, testType); + futuresMap[key] = futureResult; } - foreach [string, future] futureResult in futures.entries() { + foreach [string, future] futureResult in futuresMap.entries() { string suffix = futureResult[0]; any|error parallelDataProviderResult = wait futureResult[1]; if parallelDataProviderResult is error { reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + "]\n" + getErrorMessage(parallelDataProviderResult), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); + println(string `${"\n"}${testFunction.name}:${suffix} has failed.${"\n"}`); enableExit(); } } @@ -81,9 +80,7 @@ isolated function executeNonDataDrivenTestIso(TestFunction testFunction, DataPro reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); return true; } - boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunctionIso(testFunction, "", GENERAL_TEST)); - if executeAfterFunctionIso(testFunction) { return true; } @@ -112,7 +109,7 @@ isolated function executeFunctionsIso(TestFunction[] testFunctions, boolean skip } } -isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, ReadOnlyType[] value, TestType testType) { +isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { if executeBeforeFunctionIso(testFunction) { reportData.onSkipped(name = testFunction.name, testType = testType); } else { @@ -121,11 +118,10 @@ isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key } } -isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { +isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { if (skipDataDrivenTest(testFunction, suffix, testType)) { return; } - ExecutionError|boolean err = executeTestFunctionIso(testFunction, suffix, testType, params); handleDataDrivenTestOutput(err, testFunction, suffix, testType); } @@ -138,7 +134,7 @@ isolated function executeBeforeFunctionIso(TestFunction testFunction) returns bo return failed; } -isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { +isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { any|error output = params == () ? trap function:call(testFunction.executableFunction) : trap function:call(testFunction.executableFunction, ...params); return handleTestFuncOutput(output, testFunction, suffix, testType); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index df95231a46a4..1953683473d3 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -18,7 +18,7 @@ import ballerina/lang.'error as langError; isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; isolated int exitCode = 0; -isolated final ConcurrentExecutionManager conMgr = new (); +isolated final ConcurrentExecutionManager conMgr = new; map dataDrivenTestParams = {}; public function startSuite() returns int { @@ -28,7 +28,6 @@ public function startSuite() returns int { return exitCode; } } - if listGroups { string[] groupsList = groupStatusRegistry.getGroupsList(); if groupsList.length() == 0 { @@ -44,7 +43,6 @@ public function startSuite() returns int { return exitCode; } } - error? err = orderTests(); if err is error { enableExit(); @@ -68,25 +66,26 @@ public function startSuite() returns int { function executeTests() returns error? { decimal startTime = currentTimeInMillis(); foreach TestFunction testFunction in testRegistry.getFunctions() { - _ = testFunction.parallelizable ? conMgr.addInitialParallelTest(testFunction) : conMgr.addInitialSerialTest(testFunction); + if !testFunction.serialExecution { + conMgr.addInitialParallelTest(testFunction); + continue; + } + conMgr.addInitialSerialTest(testFunction); } while !conMgr.isExecutionDone() { - conMgr.populateExecutionQueues(); if conMgr.getSerialQueueLength() != 0 && conMgr.countTestInExecution() == 0 { TestFunction testFunction = conMgr.getSerialTest(); conMgr.addTestInExecution(testFunction); executeTest(testFunction); - } else if conMgr.getParallelQueueLength() != 0 && conMgr.getSerialQueueLength() == 0 { + } else if conMgr.getParallelQueueLength() != 0 { TestFunction testFunction = conMgr.getParallelTest(); conMgr.addTestInExecution(testFunction); DataProviderReturnType? testFunctionArgs = dataDrivenTestParams[testFunction.name]; - if testFunctionArgs is map|readonly[][] { - testFunctionArgs = testFunctionArgs.cloneReadOnly(); - } _ = start executeTestIso(testFunction, testFunctionArgs); } + conMgr.populateExecutionQueues(); } - println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n"); + println(string `${"\n"}${"\t"}${"\t"}Test execution time : ${currentTimeInMillis() - startTime}ms${"\n"}`); } function executeBeforeSuiteFunctions() { @@ -160,9 +159,9 @@ isolated function getErrorMessage(error err) returns string { } isolated function getTestType(DataProviderReturnType? params) returns TestType { - if (params is map) { + if (params is map) { return DATA_DRIVEN_MAP_OF_TUPLE; - } else if (params is AnyOrErrorOrReadOnlyType[][]) { + } else if (params is AnyOrError[][]) { return DATA_DRIVEN_TUPLE_OF_TUPLE; } return GENERAL_TEST; @@ -185,7 +184,7 @@ isolated function nestedEnabledDependentsAvailable(TestFunction[] dependents) re } isolated function isDataDrivenTest(DataProviderReturnType? params) returns boolean => - params is map || params is AnyOrErrorOrReadOnlyType[][]; + params is map || params is AnyOrError[][]; isolated function enableShouldSkip() { lock { @@ -210,11 +209,10 @@ isolated function isTestReadyToExecute(TestFunction testFunction, DataProviderRe conMgr.setExecutionSuspended(testFunction.name); return false; } - error? diagnoseError = testFunction.diagnostics; if diagnoseError is error { reportData.onFailed(name = testFunction.name, message = diagnoseError.message(), testType = getTestType(testFunctionArgs)); - println("\n" + testFunction.name + " has failed.\n"); + println(string `${"\n"}${testFunction.name} has failed.${"\n"}`); enableExit(); conMgr.setExecutionSuspended(testFunction.name); return false; @@ -228,7 +226,7 @@ isolated function finishTestExecution(TestFunction testFunction, boolean shouldS conMgr.setSkip(dependent.name); }); } - conMgr.setExecutionDone(testFunction.name); + conMgr.setExecutionCompleted(testFunction.name); } isolated function handleBeforeGroupOutput(TestFunction testFunction, string 'group, ExecutionError? err) { @@ -253,7 +251,7 @@ isolated function handleNonDataDrivenTestOutput(TestFunction testFunction, Execu if output is ExecutionError { failed = true; reportData.onFailed(name = testFunction.name, message = output.message(), testType = GENERAL_TEST); - println("\n" + testFunction.name + " has failed.\n"); + println(string `${"\n"}${testFunction.name} has failed.${"\n"}`); } else if output { failed = true; } @@ -280,7 +278,7 @@ isolated function handleDataDrivenTestOutput(ExecutionError|boolean err, TestFun if err is ExecutionError { reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name + "]\n" + getErrorMessage(err), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed.\n"); + println(string `${"\n"}${testFunction.name}:${suffix} has failed.${"\n"}`); enableExit(); } } @@ -307,7 +305,7 @@ isolated function handleTestFuncOutput(any|error output, TestFunction testFuncti if output is TestError { enableExit(); reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); - println("\n" + testFunction.name + ":" + suffix + " has failed\n"); + println(string `${"\n"}${testFunction.name}:${suffix} has failed.${"\n"}`); return true; } else if output is any { reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); @@ -319,17 +317,17 @@ isolated function handleTestFuncOutput(any|error output, TestFunction testFuncti } isolated function prepareDataSet(DataProviderReturnType? testFunctionArgs, string[] keys, - AnyOrErrorOrReadOnlyType[][] values) returns TestType { + AnyOrError[][] values) returns TestType { TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; - if testFunctionArgs is map { - foreach [string, AnyOrErrorOrReadOnlyType[]] entry in testFunctionArgs.entries() { + if testFunctionArgs is map { + foreach [string, AnyOrError[]] entry in testFunctionArgs.entries() { keys.push(entry[0]); values.push(entry[1]); } - } else if testFunctionArgs is AnyOrErrorOrReadOnlyType[][] { + } else if testFunctionArgs is AnyOrError[][] { testType = DATA_DRIVEN_TUPLE_OF_TUPLE; int i = 0; - foreach AnyOrErrorOrReadOnlyType[] entry in testFunctionArgs { + foreach AnyOrError[] entry in testFunctionArgs { keys.push(i.toString()); values.push(entry); i += 1; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index 3a1921b40627..d5b59d8178c1 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -20,6 +20,7 @@ boolean terminate = false; boolean listGroups = false; isolated final TestOptions testOptions = new (); +//TODO: remove public and see effect public function setTestOptions(string inTargetPath, string inPackageName, string inModuleName, string inReport, string inCoverage, string inGroups, string inDisableGroups, string inTests, string inRerunFailed, string inListGroups, string inIsParallelExecution) { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 9eded92c718b..38b44e2f75f7 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -15,15 +15,15 @@ // under the License. import ballerina/lang.array; -isolated final TestRegistry testRegistry = new (); -isolated final TestRegistry beforeSuiteRegistry = new (); -isolated final TestRegistry afterSuiteRegistry = new (); -isolated final TestRegistry beforeEachRegistry = new (); -isolated final TestRegistry afterEachRegistry = new (); +isolated final TestRegistry testRegistry = new; +isolated final TestRegistry beforeSuiteRegistry = new; +isolated final TestRegistry afterSuiteRegistry = new; +isolated final TestRegistry beforeEachRegistry = new; +isolated final TestRegistry afterEachRegistry = new; -isolated final GroupRegistry beforeGroupsRegistry = new (); -isolated final GroupRegistry afterGroupsRegistry = new (); -isolated final GroupStatusRegistry groupStatusRegistry = new (); +isolated final GroupRegistry beforeGroupsRegistry = new; +isolated final GroupRegistry afterGroupsRegistry = new; +isolated final GroupStatusRegistry groupStatusRegistry = new; type TestFunction record {| string name; @@ -34,7 +34,7 @@ type TestFunction record {| readonly & string[] groups = []; error? diagnostics = (); readonly & function[] dependsOn = []; - boolean parallelizable = true; + boolean serialExecution = false; TestConfig? config = (); |} & readonly; @@ -44,7 +44,7 @@ type TestFunctionMetaData record {| int dependsOnCount = 0; TestFunction[] dependents = []; boolean visited = false; - boolean isInExecutionQueue = false; + boolean isReadyToExecute = false; TestCompletionStatus executionCompletionStatus = YET_TO_COMPLETE; |}; @@ -53,15 +53,15 @@ isolated class ConcurrentExecutionManager { private final TestFunction[] serialTestExecutionList = []; private final TestFunction[] testsInExecution = []; private final map testMetaData = {}; - private boolean isParalleleExecutionEnabled = false; + private boolean parallelExecutionEnabled = false; - isolated function createTestFunctionMetaData(string functionName, *TestFunctionMetaData intialMetaData) { + isolated function createTestFunctionMetaData(string functionName, int dependsOnCount, boolean enabled) { // accept as readonly (str, str, int,boolean) param and remove clone usage lock { - self.testMetaData[functionName] = intialMetaData.clone(); + self.testMetaData[functionName] = {dependsOnCount: dependsOnCount, enabled: enabled}; } } - isolated function setExecutionDone(string functionName) { + isolated function setExecutionCompleted(string functionName) { lock { self.testMetaData[functionName].executionCompletionStatus = COMPLETED; } @@ -88,10 +88,7 @@ isolated class ConcurrentExecutionManager { isolated function isEnabled(string functionName) returns boolean { lock { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; - if testFunctionMetaData is TestFunctionMetaData { - return testFunctionMetaData.enabled; - } - return false; + return testFunctionMetaData is TestFunctionMetaData && testFunctionMetaData.enabled; } } @@ -104,20 +101,15 @@ isolated class ConcurrentExecutionManager { isolated function isVisited(string functionName) returns boolean { lock { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; - if testFunctionMetaData is TestFunctionMetaData { - return testFunctionMetaData.visited; - } - return false; + return testFunctionMetaData is TestFunctionMetaData && testFunctionMetaData.visited; } } isolated function isSkip(string functionName) returns boolean { lock { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[functionName]; - if testFunctionMetaData is TestFunctionMetaData { - return testFunctionMetaData.skip; - } - return false; + return testFunctionMetaData is TestFunctionMetaData && testFunctionMetaData.skip; + } } @@ -148,25 +140,13 @@ isolated class ConcurrentExecutionManager { isolated function setParallelExecutionStatus(boolean isParalleleExecutionEnabled) { lock { - self.isParalleleExecutionEnabled = isParalleleExecutionEnabled; + self.parallelExecutionEnabled = isParalleleExecutionEnabled; } } isolated function isParallelExecutionEnabled() returns boolean { lock { - return self.isParalleleExecutionEnabled; - } - } - - isolated function addParallelTest(TestFunction testFunction) { - lock { - self.parallelTestExecutionList.push(testFunction); - } - } - - isolated function addSerialTest(TestFunction testFunction) { - lock { - self.serialTestExecutionList.push(testFunction); + return self.parallelExecutionEnabled; } } @@ -234,7 +214,7 @@ isolated class ConcurrentExecutionManager { TestFunction testInProgress = self.testsInExecution[i]; TestFunctionMetaData? inProgressTestMetaData = self.testMetaData[testInProgress.name]; if inProgressTestMetaData == () { - return; + continue; } executionCompletionStatus = inProgressTestMetaData.executionCompletionStatus; if executionCompletionStatus == COMPLETED { @@ -243,7 +223,7 @@ isolated class ConcurrentExecutionManager { } else if executionCompletionStatus == SUSPENDED { _ = self.testsInExecution.remove(i); } else { - i = i + 1; + i += 1; } } } @@ -252,12 +232,18 @@ isolated class ConcurrentExecutionManager { private isolated function checkExecutionReadiness(TestFunction testFunction) { lock { TestFunctionMetaData? testFunctionMetaData = self.testMetaData[testFunction.name]; - if testFunctionMetaData is TestFunctionMetaData { - testFunctionMetaData.dependsOnCount -= 1; - if testFunctionMetaData.dependsOnCount == 0 && testFunctionMetaData.isInExecutionQueue != true { - testFunctionMetaData.isInExecutionQueue = true; - _ = testFunction.parallelizable ? self.parallelTestExecutionList.push(testFunction) : self.serialTestExecutionList.push(testFunction); - } + if testFunctionMetaData == () { + return; + } + testFunctionMetaData.dependsOnCount -= 1; + if testFunctionMetaData.dependsOnCount != 0 || testFunctionMetaData.isReadyToExecute { + return; + } + testFunctionMetaData.isReadyToExecute = true; + if !testFunction.serialExecution { + self.parallelTestExecutionList.push(testFunction); + } else { + self.serialTestExecutionList.push(testFunction); } } } @@ -281,7 +267,7 @@ isolated class TestOptions { isolated function getFilterSubTest(string key) returns string[] { lock { string[]? nullOrSubTests = self.filterSubTests[key]; - if nullOrSubTests is string[] { + if nullOrSubTests is string[] { // TODO: use ternary return nullOrSubTests.clone(); } return []; @@ -395,6 +381,7 @@ isolated class TestOptions { return self.hasFilteredTests; } } + } isolated class TestRegistry { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal index 2c757325fbce..c6768e0dd285 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal @@ -79,20 +79,22 @@ isolated class Result { } isolated class ReportData { - private ResultData[] passed = []; - private ResultData[] failed = []; - private ResultData[] skipped = []; + private final ResultData[] passed = []; + private final ResultData[] failed = []; + private final ResultData[] skipped = []; isolated function onPassed(*ResultData result) { lock { self.passed.push(result); } } + isolated function onFailed(*ResultData result) { lock { self.failed.push(result); } } + isolated function onSkipped(*ResultData result) { lock { self.skipped.push(result); @@ -104,11 +106,13 @@ isolated class ReportData { return self.passed.clone(); } } + isolated function failedCases() returns ResultData[] { lock { return self.failed.clone(); } } + function skippedCases() returns ResultData[] { lock { return self.skipped.clone(); @@ -120,17 +124,18 @@ isolated class ReportData { return self.passed.length(); } } + isolated function failedCount() returns int { lock { return self.failed.length(); } } + isolated function skippedCount() returns int { lock { return self.skipped.length(); } } - } isolated function consoleReport(ReportData data) { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal index 5f0d656c3841..63569daa49ce 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal @@ -54,12 +54,12 @@ function executeBeforeEachFunctions() => function executeDataDrivenTestSet(TestFunction testFunction) { DataProviderReturnType? params = dataDrivenTestParams[testFunction.name]; string[] keys = []; - AnyOrErrorOrReadOnlyType[][] values = []; + AnyOrError[][] values = []; TestType testType = prepareDataSet(params, keys, values); while keys.length() != 0 { string key = keys.remove(0); - AnyOrErrorOrReadOnlyType[] value = values.remove(0); + AnyOrError[] value = values.remove(0); prepareDataDrivenTest(testFunction, key, value, testType); } } @@ -71,7 +71,6 @@ function executeNonDataDrivenTest(TestFunction testFunction) returns boolean { return true; } boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunction(testFunction, "", GENERAL_TEST)); - if executeAfterFunction(testFunction) { return true; } @@ -100,7 +99,7 @@ function executeFunctions(TestFunction[] testFunctions, boolean skip = false) re } } -function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrErrorOrReadOnlyType[] value, TestType testType) { +function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { if executeBeforeFunction(testFunction) { reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); } else { @@ -109,7 +108,7 @@ function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError } } -function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[] params) { +function executeDataDrivenTest(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { if skipDataDrivenTest(testFunction, suffix, testType) { return; } @@ -125,7 +124,7 @@ function executeBeforeFunction(TestFunction testFunction) returns boolean { return failed; } -function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrErrorOrReadOnlyType[]? params = ()) returns ExecutionError|boolean { +function executeTestFunction(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { any|error output = params == () ? trap function:call(testFunction.executableFunction) : trap function:call(testFunction.executableFunction, ...params); return handleTestFuncOutput(output, testFunction, suffix, testType); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal index aca163b03031..8866d681eb94 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/types.bal @@ -15,19 +15,15 @@ // under the License. # Possible return types of the data provider function. -public type DataProviderReturnType error|map|AnyOrErrorOrReadOnlyType[][]; +public type DataProviderReturnType error|map|AnyOrError[][]; -public type AnyOrErrorOrReadOnlyType any|error|ReadOnlyType; - -public type ReadOnlyType readonly; +type AnyOrError any|error; # Represents errors generated by the Testerina module. public type TestError distinct error; type ExecutionError distinct error; -public type serialExecutionType true|(); - type ModuleRerunJson record {| string[] testNames; map testModuleNames; diff --git a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java index 509a6ec7011b..3022f1edc316 100644 --- a/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java +++ b/misc/testerina/modules/testerina-core/src/main/java/org/ballerinalang/testerina/natives/CommonUtils.java @@ -28,7 +28,7 @@ /** * Common utility functions for the Testerina module. * - * @since 2201.7.0 + * @since 2201.9.0 */ public class CommonUtils { public static BDecimal currentTimeInMillis() { @@ -40,10 +40,9 @@ public static Object isFunctionParamConcurrencySafe(BFunctionPointer func) { Parameter[] functionParameters = functionType.getParameters(); for (Parameter functionParameter : functionParameters) { Type parameterType = functionParameter.type; - if (isSubTypeOfReadOnly(parameterType)) { - continue; + if (!isSubTypeOfReadOnly(parameterType)) { + return false; } - return false; } return true; } diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index 3029b6125a11..cbe679ec2a9a 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -32,6 +32,7 @@ public class TestparallelizationTest extends BaseTestCase { + public static final String PARALLEL_FLAG = "--parallel"; private BMainInstance balClient; private String projectPath; @@ -43,7 +44,7 @@ public void setup() throws BallerinaTestException { @Test public void testParallelization() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-simple-test"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "parallelisation-simple-test"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -58,7 +59,7 @@ public void testParallelization() throws BallerinaTestException, IOException { @Test public void testNonParallelizable() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-serialExecution"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "parallelisation-serialExecution"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -75,7 +76,7 @@ public void testNonParallelizable() throws BallerinaTestException, IOException { @Test public void testParalallelizableTupleDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-tuple-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "parallelisation-tuple-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -90,7 +91,7 @@ public void testParalallelizableTupleDataProvider() throws BallerinaTestExceptio @Test public void testParalallelizableMapDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--parallel", "parallelisation-map-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "parallelisation-map-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -106,7 +107,7 @@ public void testParalallelizableMapDataProvider() throws BallerinaTestException, @Test public void testNonParalallelizableTupleDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-parallelisation-tuple-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-parallelisation-tuple-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -124,7 +125,7 @@ public void testNonParalallelizableTupleDataProvider() throws BallerinaTestExcep public void testNonIsolatedTestFunction() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'testAssertEquals*' cannot be parallelized, reason: " + "non-isolated test function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-tests"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-tests"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -144,7 +145,7 @@ public void testNonIsolatedTestFunction() throws BallerinaTestException, IOExcep public void testNonIsolatedTestParameter() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "unsafe test parameters"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-test-params"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-test-params"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -163,7 +164,7 @@ public void testNonIsolatedTestParameter() throws BallerinaTestException, IOExce public void testNonIsolatedDataProvider() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated data-provider function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -182,7 +183,7 @@ public void testNonIsolatedDataProvider() throws BallerinaTestException, IOExcep public void testNonIsolatedAfterEach() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated after-each function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-after-each"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-after-each"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -201,7 +202,7 @@ public void testNonIsolatedAfterEach() throws BallerinaTestException, IOExceptio public void testNonIsolatedBeforeEach() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated before-each function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-before-each"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-before-each"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -220,7 +221,7 @@ public void testNonIsolatedBeforeEach() throws BallerinaTestException, IOExcepti public void testNonIsolatedAfterFunc() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated after function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-after-func"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-after-func"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -239,7 +240,7 @@ public void testNonIsolatedAfterFunc() throws BallerinaTestException, IOExceptio public void testNonIsolatedBeforeFunc() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated before function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-before-func"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-before-func"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -258,7 +259,7 @@ public void testNonIsolatedBeforeFunc() throws BallerinaTestException, IOExcepti public void testNonIsolatedAfterGroup() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated after-group function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-after-grp"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-after-grp"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -277,7 +278,7 @@ public void testNonIsolatedAfterGroup() throws BallerinaTestException, IOExcepti public void testNonIsolatedBeforeGroup() throws BallerinaTestException, IOException { String warningDiagnostics = "WARNING: Test function 'mapDataProviderTest' cannot be parallelized, reason: " + "non-isolated before-group function"; - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-isolated-before-grp"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-isolated-before-grp"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -294,7 +295,7 @@ public void testNonIsolatedBeforeGroup() throws BallerinaTestException, IOExcept @Test public void testIsolatedSetUpTearDown() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--parallel", "isolated-set-up-tear-down"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "isolated-set-up-tear-down"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); @@ -310,7 +311,7 @@ public void testIsolatedSetUpTearDown() throws BallerinaTestException, IOExcepti @Test public void testNonParalallelizableMapDataProvider() throws BallerinaTestException, IOException { - String[] args = mergeCoverageArgs(new String[]{"--parallel", "non-parallelisation-map-data-provider"}); + String[] args = mergeCoverageArgs(new String[]{PARALLEL_FLAG, "non-parallelisation-map-data-provider"}); String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false); Float executionTimeW30 = getTimeForTestExecution(output); diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal index 3f180e290a4d..7be5d4866d43 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/isolated-set-up-tear-down/tests/data_driven_test.bal @@ -3,7 +3,6 @@ import ballerina/test; @test:BeforeSuite function setup() { - } @test:BeforeEach @@ -27,7 +26,6 @@ function mapDataProviderTest(int value1, int value2, string fruit) returns error } function beforeFunc() { - } function mapDataProvider() returns map<[int, int, string]>|error { @@ -78,12 +76,10 @@ function mapDataProvider() returns map<[int, int, string]>|error { } function afterFunc() { - } @test:AfterGroups {value: ["g1"]} function afterGroups1() { - } @test:AfterEach @@ -92,5 +88,4 @@ function afterEachFunc() { @test:AfterSuite function cleanup() { - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal index 691f5cb68c4a..e9c889a80f04 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-each/tests/data_driven_test.bal @@ -3,7 +3,6 @@ import ballerina/test; @test:BeforeSuite function setup() { - } @test:BeforeEach @@ -27,7 +26,6 @@ function mapDataProviderTest(int value1, int value2, string fruit) returns error } function beforeFunc() { - } function mapDataProvider() returns map<[int, int, string]>|error { @@ -83,7 +81,6 @@ function afterFunc() { @test:AfterGroups {value: ["g1"]} function afterGroups1() { - } @test:AfterEach @@ -92,5 +89,4 @@ public function afterEachFunc() { @test:AfterSuite function cleanup() { - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal index a42988819c18..0bffdce6d3bc 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-func/tests/data_driven_test.bal @@ -3,7 +3,6 @@ import ballerina/test; @test:BeforeSuite function setup() { - } @test:BeforeEach @@ -27,7 +26,6 @@ function mapDataProviderTest(int value1, int value2, string fruit) returns error } function beforeFunc() { - } function mapDataProvider() returns map<[int, int, string]>|error { @@ -92,5 +90,4 @@ function afterEachFunc() { @test:AfterSuite function cleanup() { - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal index 08b83b6293f9..0cc80f39db76 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-after-grp/tests/data_driven_test.bal @@ -3,7 +3,6 @@ import ballerina/test; @test:BeforeSuite function setup() { - } @test:BeforeEach @@ -27,7 +26,6 @@ function mapDataProviderTest(int value1, int value2, string fruit) returns error } function beforeFunc() { - } function mapDataProvider() returns map<[int, int, string]>|error { @@ -78,12 +76,10 @@ function mapDataProvider() returns map<[int, int, string]>|error { } function afterFunc() { - } @test:AfterGroups {value: ["g1"]} public function afterGroups1() { - } @test:AfterEach @@ -92,5 +88,4 @@ function afterEachFunc() { @test:AfterSuite function cleanup() { - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal index 67e4700c283e..c43e1ab9e7e3 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-each/tests/data_driven_test.bal @@ -3,7 +3,6 @@ import ballerina/test; @test:BeforeSuite function setup() { - } @test:BeforeEach @@ -27,7 +26,6 @@ function mapDataProviderTest(int value1, int value2, string fruit) returns error } function beforeFunc() { - } function mapDataProvider() returns map<[int, int, string]>|error { @@ -78,12 +76,10 @@ function mapDataProvider() returns map<[int, int, string]>|error { } function afterFunc() { - } @test:AfterGroups {value: ["g1"]} function afterGroups1() { - } @test:AfterEach @@ -92,5 +88,4 @@ function afterEachFunc() { @test:AfterSuite function cleanup() { - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal index 9b8ecbca2223..c29f456c9c4c 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-func/tests/data_driven_test.bal @@ -3,7 +3,6 @@ import ballerina/test; @test:BeforeSuite function setup() { - } @test:BeforeEach @@ -27,7 +26,6 @@ function mapDataProviderTest(int value1, int value2, string fruit) returns error } public function beforeFunc() { - } function mapDataProvider() returns map<[int, int, string]>|error { @@ -78,12 +76,10 @@ function mapDataProvider() returns map<[int, int, string]>|error { } function afterFunc() { - } @test:AfterGroups {value: ["g1"]} function afterGroups1() { - } @test:AfterEach @@ -92,5 +88,4 @@ function afterEachFunc() { @test:AfterSuite function cleanup() { - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal index 1902106b50e3..47fbcf04bf34 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-before-grp/tests/data_driven_test.bal @@ -3,7 +3,6 @@ import ballerina/test; @test:BeforeSuite function setup() { - } @test:BeforeEach @@ -27,7 +26,6 @@ function mapDataProviderTest(int value1, int value2, string fruit) returns error } function beforeFunc() { - } function mapDataProvider() returns map<[int, int, string]>|error { @@ -78,12 +76,10 @@ function mapDataProvider() returns map<[int, int, string]>|error { } function afterFunc() { - } @test:AfterGroups {value: ["g1"]} function afterGroups1() { - } @test:AfterEach @@ -92,5 +88,4 @@ function afterEachFunc() { @test:AfterSuite function cleanup() { - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-data-provider/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-test-params/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-isolated-tests/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-map-data-provider/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/tests/normalTest.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/tests/normalTest.bal index 677a7ea8ae3c..2a0a9d9c42a7 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/tests/normalTest.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-serialExecution/tests/normalTest.bal @@ -1,6 +1,5 @@ - -import ballerina/test; import ballerina/lang.runtime; +import ballerina/test; @test:BeforeSuite function beforeSuiteFunc() { diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-simple-test/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } diff --git a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal index a37ada63a432..6b71ef415c69 100644 --- a/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal +++ b/tests/testerina-integration-test/src/test/resources/project-based-tests/parallelisation-test/parallelisation-tuple-data-provider/main.bal @@ -1,55 +1,2 @@ -import ballerina/io; -import ballerina/lang.runtime; -import ballerina/time; - -int employedCount = 20000; -int[] quantities_ = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - - 23, - 24 -]; - public function main() { - decimal now = time:monotonicNow(); - int count = 10; - - while true { - if count > 0 { - future<()> f = start get_average(); - count -= 1; - } - // int avg = get_average(); - // io:println("Average: ", avg); - - } - -} - -function get_average() { - runtime:sleep(10); - io:println("Helllwo"); - } From cce17eecdb343057d3816ba21404d78ede8e563d Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Wed, 21 Feb 2024 20:19:42 +0530 Subject: [PATCH 35/37] Address the review --- .../project_b_100/resources/expectedDeps.toml | 30 ++++-- .../main/ballerina/annotation_processor.bal | 4 +- .../src/main/ballerina/concurrentExecuter.bal | 94 ++++++++++--------- .../src/main/ballerina/execute.bal | 80 ++++++++-------- .../src/main/ballerina/external.bal | 2 +- .../src/main/ballerina/filter.bal | 15 ++- .../src/main/ballerina/register.bal | 18 ++-- .../src/main/ballerina/report.bal | 10 +- .../src/main/ballerina/serialExecuter.bal | 4 +- .../testOutputs/balTestStrandDumpRegEx.txt | 22 ++--- ...iderTest-testArrayDataProviderWithFail.txt | 4 +- ...viderTest-testArrayDataRerunFailedTest.txt | 4 +- ...iderTest-testDataProviderSingleFailure.txt | 25 +++-- ...aProviderTest-testMapValueDataProvider.txt | 2 +- .../DataProviderTest-testRerunFailedTest.txt | 4 +- ...-testValidDataProviderWithAfterFailing.txt | 2 +- ...testValidDataProviderWithBeforeFailing.txt | 2 +- ...iderTest-testValidDataProviderWithFail.txt | 4 +- ...pingTest-testNonExistingGroupExclusion.txt | 4 +- .../GroupingTest-testSingleGroupExclusion.txt | 2 +- ...oviderTestCase-testInvalidDataProvider.txt | 35 ++++--- ...viderTestCase-testInvalidDataProvider2.txt | 35 ++++--- ...rTestCase-testInvalidTupleDataProvider.txt | 35 ++++--- .../ModuleExecutionWithInitStartFailures.txt | 2 +- ...racefulStopTest-test-listener-shutdown.txt | 6 +- .../unix/RerunFailedTest-testFullTest.txt | 4 +- .../RerunFailedTest-testRerunFailedTest.txt | 4 +- ...SingleFileTestExecutionWithInitFailure.txt | 2 +- ...ipTestsTestCase-testSkipWhenAfterFails.txt | 2 +- ...pTestsTestCase-testSkipWhenBeforeFails.txt | 2 +- ...TestCase-testSkipWhenBeforeGroupsFails.txt | 2 +- .../unix/TestExecutionWithInitFailures.txt | 8 +- ...tTest-testWarningForCoverageFormatFlag.txt | 2 +- ...stReportTest-testWarningForReportTools.txt | 2 +- ...iderTest-testArrayDataProviderWithFail.txt | 6 +- ...viderTest-testArrayDataRerunFailedTest.txt | 6 +- ...iderTest-testDataProviderSingleFailure.txt | 25 +++-- ...aProviderTest-testMapValueDataProvider.txt | 4 +- .../DataProviderTest-testRerunFailedTest.txt | 6 +- ...-testValidDataProviderWithAfterFailing.txt | 4 +- ...testValidDataProviderWithBeforeFailing.txt | 4 +- ...iderTest-testValidDataProviderWithFail.txt | 6 +- ...pingTest-testNonExistingGroupExclusion.txt | 6 +- .../GroupingTest-testSingleGroupExclusion.txt | 4 +- ...oviderTestCase-testInvalidDataProvider.txt | 35 ++++--- ...viderTestCase-testInvalidDataProvider2.txt | 37 +++++--- ...rTestCase-testInvalidTupleDataProvider.txt | 37 +++++--- .../ModuleExecutionWithInitStartFailures.txt | 4 +- ...racefulStopTest-test-listener-shutdown.txt | 8 +- .../windows/RerunFailedTest-testFullTest.txt | 88 ++++++++--------- .../RerunFailedTest-testRerunFailedTest.txt | 84 ++++++++--------- ...ipTestsTestCase-testSkipWhenAfterFails.txt | 92 +++++++++--------- ...pTestsTestCase-testSkipWhenBeforeFails.txt | 90 +++++++++--------- ...TestCase-testSkipWhenBeforeGroupsFails.txt | 4 +- .../windows/TestExecutionWithInitFailures.txt | 8 +- ...tTest-testWarningForCoverageFormatFlag.txt | 4 +- ...stReportTest-testWarningForReportTools.txt | 4 +- 57 files changed, 574 insertions(+), 465 deletions(-) diff --git a/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml b/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml index 703d373b52a3..379956ea2017 100644 --- a/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml +++ b/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml @@ -15,22 +15,39 @@ scope = "testOnly" [[package]] org = "ballerina" -name = "lang.error" +name = "lang.__internal" version = "0.0.0" scope = "testOnly" dependencies = [ - {org = "ballerina", name = "jballerina.java"} + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.object"} ] [[package]] org = "ballerina" -name = "lang.runtime" +name = "lang.array" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.__internal"} +] + +[[package]] +org = "ballerina" +name = "lang.error" version = "0.0.0" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"} ] +[[package]] +org = "ballerina" +name = "lang.object" +version = "0.0.0" +scope = "testOnly" + [[package]] org = "ballerina" name = "test" @@ -38,11 +55,12 @@ version = "0.0.0" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"}, - {org = "ballerina", name = "lang.error"}, - {org = "ballerina", name = "lang.runtime"} + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} ] modules = [ - {org = "ballerina", packageName = "test", moduleName = "test"} + {org = "ballerina", packageName = "test", moduleName = "test"}, + {org = "ballerina", packageName = "test", moduleName = "test.test"} ] [[package]] diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal index d0d51b01f192..9cde5d80f107 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/annotation_processor.bal @@ -227,13 +227,13 @@ isolated function hasTest(string name) returns boolean { int? testIndex = testOptions.getFilterTestIndex(testName); if testIndex == () { foreach string filter in testOptions.getFilterTests() { - if (filter.includes(WILDCARD)) { + if filter.includes(WILDCARD) { boolean|error wildCardMatch = matchWildcard(testName, filter); return (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)); } } return false; - } else if (matchModuleName(testName)) { + } else if matchModuleName(testName) { return true; } return false; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal index 0ef70274366e..fd4113c2798a 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/concurrentExecuter.bal @@ -14,142 +14,150 @@ // specific language governing permissions and limitations // under the License. -isolated function executeTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { +isolated function executeTestIsolated(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { if !isTestReadyToExecute(testFunction, testFunctionArgs) { return; } - executeBeforeGroupFunctionsIso(testFunction); - executeBeforeEachFunctionsIso(); + executeBeforeGroupFunctionsIsolated(testFunction); + executeBeforeEachFunctionsIsolated(); boolean shouldSkipDependents = false; if !isSkipFunction(testFunction) { - if (isDataDrivenTest(testFunctionArgs)) { - executeDataDrivenTestSetIso(testFunction, testFunctionArgs); + if isDataDrivenTest(testFunctionArgs) { + executeDataDrivenTestSetIsolated(testFunction, testFunctionArgs); } else { - shouldSkipDependents = executeNonDataDrivenTestIso(testFunction, testFunctionArgs); + shouldSkipDependents = executeNonDataDrivenTestIsolated(testFunction, testFunctionArgs); } } else { reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); shouldSkipDependents = true; } testFunction.groups.forEach('group => groupStatusRegistry.incrementExecutedTest('group)); - executeAfterEachFunctionsIso(); - executeAfterGroupFunctionsIso(testFunction); + executeAfterEachFunctionsIsolated(); + executeAfterGroupFunctionsIsolated(testFunction); finishTestExecution(testFunction, shouldSkipDependents); } -isolated function executeBeforeGroupFunctionsIso(TestFunction testFunction) { +isolated function executeBeforeGroupFunctionsIsolated(TestFunction testFunction) { foreach string 'group in testFunction.groups { TestFunction[]? beforeGroupFunctions = beforeGroupsRegistry.getFunctions('group); if beforeGroupFunctions != () && !groupStatusRegistry.firstExecuted('group) { - handleBeforeGroupOutput(testFunction, 'group, executeFunctionsIso(beforeGroupFunctions, getShouldSkip())); + handleBeforeGroupOutput(testFunction, 'group, executeFunctionsIsolated(beforeGroupFunctions, getShouldSkip())); } } } -isolated function executeBeforeEachFunctionsIso() => - handleBeforeEachOutput(executeFunctionsIso(beforeEachRegistry.getFunctions(), getShouldSkip())); +isolated function executeBeforeEachFunctionsIsolated() => + handleBeforeEachOutput(executeFunctionsIsolated(beforeEachRegistry.getFunctions(), getShouldSkip())); -isolated function executeDataDrivenTestSetIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { +isolated function executeDataDrivenTestSetIsolated(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) { string[] keys = []; AnyOrError[][] values = []; TestType testType = prepareDataSet(testFunctionArgs, keys, values); map futuresMap = {}; while keys.length() != 0 { string key = keys.remove(0); - final readonly & readonly[] readOnlyVal = from any|error item in values.remove(0) + AnyOrError[] value = values.remove(0); + final readonly & readonly[] readOnlyVal = from any|error item in value where item is readonly select item; - future<()> futureResult = start prepareDataDrivenTestIso(testFunction, key, readOnlyVal, testType); + if readOnlyVal.length() != value.length() { + reportData.onFailed(name = testFunction.name, suffix = key, message = string `[fail data provider for the function ${testFunction.name}]${"\n"} Data provider returned non-readonly values`, testType = testType); + println(string `${"\n"}${testFunction.name}:${key} has failed.${"\n"}`); + enableExit(); + } + future<()> futureResult = start prepareDataDrivenTestIsolated(testFunction, key, readOnlyVal, testType); futuresMap[key] = futureResult; } foreach [string, future] futureResult in futuresMap.entries() { string suffix = futureResult[0]; any|error parallelDataProviderResult = wait futureResult[1]; if parallelDataProviderResult is error { - reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name - + "]\n" + getErrorMessage(parallelDataProviderResult), testType = testType); + reportData.onFailed(name = testFunction.name, suffix = suffix, message = string `[fail data provider for the function ${testFunction.name}]${"\n"} ${getErrorMessage(parallelDataProviderResult)}`, testType = testType); println(string `${"\n"}${testFunction.name}:${suffix} has failed.${"\n"}`); enableExit(); } } } -isolated function executeNonDataDrivenTestIso(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { - if executeBeforeFunctionIso(testFunction) { +isolated function executeNonDataDrivenTestIsolated(TestFunction testFunction, DataProviderReturnType? testFunctionArgs) returns boolean { + if executeBeforeFunctionIsolated(testFunction) { conMgr.setSkip(testFunction.name); reportData.onSkipped(name = testFunction.name, testType = getTestType(testFunctionArgs)); return true; } - boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunctionIso(testFunction, "", GENERAL_TEST)); - if executeAfterFunctionIso(testFunction) { + boolean failed = handleNonDataDrivenTestOutput(testFunction, executeTestFunctionIsolated(testFunction, "", GENERAL_TEST)); + if executeAfterFunctionIsolated(testFunction) { return true; } return failed; } -isolated function executeAfterEachFunctionsIso() => - handleAfterEachOutput(executeFunctionsIso(afterEachRegistry.getFunctions(), getShouldSkip())); +isolated function executeAfterEachFunctionsIsolated() => + handleAfterEachOutput(executeFunctionsIsolated(afterEachRegistry.getFunctions(), getShouldSkip())); -isolated function executeAfterGroupFunctionsIso(TestFunction testFunction) { +isolated function executeAfterGroupFunctionsIsolated(TestFunction testFunction) { foreach string 'group in testFunction.groups { TestFunction[]? afterGroupFunctions = afterGroupsRegistry.getFunctions('group); if afterGroupFunctions != () && groupStatusRegistry.lastExecuted('group) { - ExecutionError? err = executeFunctionsIso(afterGroupFunctions, + ExecutionError? err = executeFunctionsIsolated(afterGroupFunctions, getShouldSkip() || groupStatusRegistry.getSkipAfterGroup('group)); handleAfterGroupOutput(err); } } } -isolated function executeFunctionsIso(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { +isolated function executeFunctionsIsolated(TestFunction[] testFunctions, boolean skip = false) returns ExecutionError? { foreach TestFunction testFunction in testFunctions { if !skip || testFunction.alwaysRun { - check executeFunctionIso(testFunction); + check executeFunctionIsolated(testFunction); } } } -isolated function prepareDataDrivenTestIso(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { - if executeBeforeFunctionIso(testFunction) { +isolated function prepareDataDrivenTestIsolated(TestFunction testFunction, string key, AnyOrError[] value, TestType testType) { + if executeBeforeFunctionIsolated(testFunction) { reportData.onSkipped(name = testFunction.name, testType = testType); } else { - executeDataDrivenTestIso(testFunction, key, testType, value); - var _ = executeAfterFunctionIso(testFunction); + executeDataDrivenTestIsolated(testFunction, key, testType, value); + _ = executeAfterFunctionIsolated(testFunction); } } -isolated function executeDataDrivenTestIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { - if (skipDataDrivenTest(testFunction, suffix, testType)) { +isolated function executeDataDrivenTestIsolated(TestFunction testFunction, string suffix, TestType testType, AnyOrError[] params) { + if skipDataDrivenTest(testFunction, suffix, testType) { return; } - ExecutionError|boolean err = executeTestFunctionIso(testFunction, suffix, testType, params); + ExecutionError|boolean err = executeTestFunctionIsolated(testFunction, suffix, testType, params); handleDataDrivenTestOutput(err, testFunction, suffix, testType); } -isolated function executeBeforeFunctionIso(TestFunction testFunction) returns boolean { +isolated function executeBeforeFunctionIsolated(TestFunction testFunction) returns boolean { boolean failed = false; if isBeforeFuncConditionMet(testFunction) { - failed = handleBeforeFunctionOutput(executeFunctionIso(testFunction.before)); + failed = handleBeforeFunctionOutput(executeFunctionIsolated(testFunction.before)); } return failed; } -isolated function executeTestFunctionIso(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { - any|error output = params == () ? trap function:call(testFunction.executableFunction) - : trap function:call(testFunction.executableFunction, ...params); +isolated function executeTestFunctionIsolated(TestFunction testFunction, string suffix, TestType testType, AnyOrError[]? params = ()) returns ExecutionError|boolean { + isolated function isolatedTestFunction = testFunction.executableFunction; + any|error output = params == () ? trap function:call(isolatedTestFunction) + : trap function:call(isolatedTestFunction, ...params); return handleTestFuncOutput(output, testFunction, suffix, testType); } -isolated function executeAfterFunctionIso(TestFunction testFunction) returns boolean { +isolated function executeAfterFunctionIsolated(TestFunction testFunction) returns boolean { boolean failed = false; if isAfterFuncConditionMet(testFunction) { - failed = handleAfterFunctionOutput(executeFunctionIso(testFunction.after)); + failed = handleAfterFunctionOutput(executeFunctionIsolated(testFunction.after)); } return failed; } -isolated function executeFunctionIso(TestFunction|function testFunction) returns ExecutionError? { - any|error output = trap function:call((testFunction is function ? testFunction : testFunction.executableFunction)); +isolated function executeFunctionIsolated(TestFunction|function testFunction) returns ExecutionError? { + isolated function isolatedTestFunction = (testFunction is function ? testFunction : testFunction.executableFunction); + // casting is done outside the function to avoid the error of casting inside the function and trapping it. + any|error output = trap function:call(isolatedTestFunction); if output is error { enableExit(); return error(getErrorMessage(output), functionName = testFunction is function ? "" : testFunction.name); diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal index d17b001f3e91..cbb5b8a4f270 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal @@ -18,7 +18,7 @@ import ballerina/lang.'error as langError; isolated boolean shouldSkip = false; boolean shouldAfterSuiteSkip = false; isolated int exitCode = 0; -isolated final ConcurrentExecutionManager conMgr = new; +final ConcurrentExecutionManager conMgr = new; map dataDrivenTestParams = {}; public function startSuite() returns int { @@ -81,7 +81,7 @@ function executeTests() returns error? { TestFunction testFunction = conMgr.getParallelTest(); conMgr.addTestInExecution(testFunction); DataProviderReturnType? testFunctionArgs = dataDrivenTestParams[testFunction.name]; - _ = start executeTestIso(testFunction, testFunctionArgs); + _ = start executeTestIsolated(testFunction, testFunctionArgs); } conMgr.populateExecutionQueues(); } @@ -108,11 +108,11 @@ function executeAfterSuiteFunctions() { function orderTests() returns error? { string[] descendants = []; - foreach TestFunction testFunction in testRegistry.getDependentFunctions() { - if !conMgr.isVisited(testFunction.name) && conMgr.isEnabled(testFunction.name) { - check restructureTest(testFunction, descendants); - } - } + from TestFunction testFunction in testRegistry.getDependentFunctions() + where !conMgr.isVisited(testFunction.name) && conMgr.isEnabled(testFunction.name) + do { + check restructureTest(testFunction, descendants); + }; } function restructureTest(TestFunction testFunction, string[] descendants) returns error? { @@ -124,7 +124,7 @@ function restructureTest(TestFunction testFunction, string[] descendants) return // dependsOnTestFunction.config?.enable is used instead of dependsOnTestFunction.enable to ensure that // the user has deliberately passed enable=false boolean? dependentEnabled = dependsOnTestFunction.config?.enable; - if dependentEnabled != () && !dependentEnabled { + if dependentEnabled == false { string errMsg = string `error: Test [${testFunction.name}] depends on function [${dependsOnTestFunction.name}], ` + string `but it is either disabled or not included.`; return error(errMsg); @@ -146,34 +146,37 @@ function restructureTest(TestFunction testFunction, string[] descendants) return _ = descendants.pop(); } -isolated function printExecutionError(ExecutionError err, string functionSuffix) - => println("\t[fail] " + err.detail().functionName + "[" + functionSuffix + "]" + ":\n\t " + formatFailedError(err.message(), 2)); +isolated function printExecutionError(ExecutionError err, string functionSuffix) { + println("\t[fail] " + err.detail().functionName + "[" + functionSuffix + "]" + ":\n\t " + formatFailedError(err.message(), 2)); +} isolated function getErrorMessage(error err) returns string { string message = err.toBalString(); string accumulatedTrace = ""; - foreach langError:StackFrame stackFrame in err.stackTrace() { + from langError:StackFrame stackFrame in err.stackTrace() + do { accumulatedTrace = accumulatedTrace + "\t" + stackFrame.toString() + "\n"; - } + }; return message + "\n" + accumulatedTrace; } isolated function getTestType(DataProviderReturnType? params) returns TestType { - if (params is map) { + if params is map { return DATA_DRIVEN_MAP_OF_TUPLE; - } else if (params is AnyOrError[][]) { + } + if params is AnyOrError[][] { return DATA_DRIVEN_TUPLE_OF_TUPLE; } return GENERAL_TEST; } isolated function nestedEnabledDependentsAvailable(TestFunction[] dependents) returns boolean { - if (dependents.length() == 0) { + if dependents.length() == 0 { return false; } TestFunction[] queue = []; foreach TestFunction dependent in dependents { - if (conMgr.isEnabled(dependent.name)) { + if conMgr.isEnabled(dependent.name) { return true; } foreach TestFunction superDependent in conMgr.getDependents(dependent.name) { @@ -276,8 +279,7 @@ isolated function handleAfterGroupOutput(ExecutionError? err) { isolated function handleDataDrivenTestOutput(ExecutionError|boolean err, TestFunction testFunction, string suffix, TestType testType) { if err is ExecutionError { - reportData.onFailed(name = testFunction.name, suffix = suffix, message = "[fail data provider for the function " + testFunction.name - + "]\n" + getErrorMessage(err), testType = testType); + reportData.onFailed(name = testFunction.name, suffix = suffix, message = string `[fail data provider for the function ${testFunction.name}]${"\n"} ${getErrorMessage(err)}`, testType = testType); println(string `${"\n"}${testFunction.name}:${suffix} has failed.${"\n"}`); enableExit(); } @@ -307,30 +309,28 @@ isolated function handleTestFuncOutput(any|error output, TestFunction testFuncti reportData.onFailed(name = testFunction.name, suffix = suffix, message = getErrorMessage(output), testType = testType); println(string `${"\n"}${testFunction.name}:${suffix} has failed.${"\n"}`); return true; - } else if output is any { + } + if output is any { reportData.onPassed(name = testFunction.name, suffix = suffix, testType = testType); return false; - } else { - enableExit(); - return error(getErrorMessage(output), functionName = testFunction.name); } + enableExit(); + return error(getErrorMessage(output), functionName = testFunction.name); } isolated function prepareDataSet(DataProviderReturnType? testFunctionArgs, string[] keys, AnyOrError[][] values) returns TestType { TestType testType = DATA_DRIVEN_MAP_OF_TUPLE; if testFunctionArgs is map { - foreach [string, AnyOrError[]] entry in testFunctionArgs.entries() { - keys.push(entry[0]); - values.push(entry[1]); + foreach [string, AnyOrError[]] [k, v] in testFunctionArgs.entries() { + keys.push(k); + values.push(v); } } else if testFunctionArgs is AnyOrError[][] { testType = DATA_DRIVEN_TUPLE_OF_TUPLE; - int i = 0; - foreach AnyOrError[] entry in testFunctionArgs { - keys.push(i.toString()); - values.push(entry); - i += 1; + foreach AnyOrError[] [k, v] in testFunctionArgs.enumerate() { + keys.push(k.toBalString()); + values.push(v); } } return testType; @@ -338,13 +338,13 @@ isolated function prepareDataSet(DataProviderReturnType? testFunctionArgs, strin isolated function skipDataDrivenTest(TestFunction testFunction, string suffix, TestType testType) returns boolean { string functionName = testFunction.name; - if (!testOptions.getHasFilteredTests()) { + if !testOptions.getHasFilteredTests() { return false; } TestFunction[] dependents = conMgr.getDependents(functionName); // if a dependent in a below level is enabled, this test should run - if (dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents)) { + if dependents.length() > 0 && nestedEnabledDependentsAvailable(dependents) { return false; } string functionKey = functionName; @@ -353,14 +353,14 @@ isolated function skipDataDrivenTest(TestFunction testFunction, string suffix, T boolean prefixMatch = testOptions.isFilterSubTestsContains(functionName); // if prefix matches to a wildcard - if (!prefixMatch && hasTest(functionName)) { + if !prefixMatch && hasTest(functionName) { // get the matching wildcard prefixMatch = true; foreach string filter in testOptions.getFilterTests() { - if (filter.includes(WILDCARD)) { + if filter.includes(WILDCARD) { boolean|error wildCardMatch = matchWildcard(functionKey, filter); - if (wildCardMatch is boolean && wildCardMatch && matchModuleName(filter)) { + if wildCardMatch is boolean && wildCardMatch && matchModuleName(filter) { functionKey = filter; break; } @@ -372,12 +372,12 @@ isolated function skipDataDrivenTest(TestFunction testFunction, string suffix, T boolean suffixMatch = !testOptions.isFilterSubTestsContains(functionKey); // if a subtest is found specified - if (!suffixMatch) { + if !suffixMatch { string[] subTests = testOptions.getFilterSubTest(functionKey); foreach string subFilter in subTests { string updatedSubFilter = subFilter; - if (testType == DATA_DRIVEN_MAP_OF_TUPLE) { - if (subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE)) { + if testType == DATA_DRIVEN_MAP_OF_TUPLE { + if subFilter.startsWith(SINGLE_QUOTE) && subFilter.endsWith(SINGLE_QUOTE) { updatedSubFilter = subFilter.substring(1, subFilter.length() - 1); } } @@ -387,11 +387,11 @@ isolated function skipDataDrivenTest(TestFunction testFunction, string suffix, T string updatedSuffix = decodedSuffix is string ? decodedSuffix : suffix; boolean wildCardMatchBoolean = false; - if (updatedSubFilter.includes(WILDCARD)) { + if updatedSubFilter.includes(WILDCARD) { boolean|error wildCardMatch = matchWildcard(updatedSuffix, updatedSubFilter); wildCardMatchBoolean = wildCardMatch is boolean && wildCardMatch; } - if ((updatedSubFilter == updatedSuffix) || wildCardMatchBoolean) { + if (updatedSubFilter == updatedSuffix) || wildCardMatchBoolean { suffixMatch = true; break; } diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal index a686e0afff45..3ef563ee034f 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/external.bal @@ -15,7 +15,7 @@ // under the License.` ` import ballerina/jballerina.java; -isolated handle outStreamObj = outStream(); +final handle outStreamObj = outStream(); isolated function print(handle printStream, any|error obj) = @java:Method { name: "print", diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal index d5b59d8178c1..d4dc23e83541 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/filter.bal @@ -18,9 +18,8 @@ string[] filterGroups = []; string[] filterDisableGroups = []; boolean terminate = false; boolean listGroups = false; -isolated final TestOptions testOptions = new (); +final TestOptions testOptions = new (); -//TODO: remove public and see effect public function setTestOptions(string inTargetPath, string inPackageName, string inModuleName, string inReport, string inCoverage, string inGroups, string inDisableGroups, string inTests, string inRerunFailed, string inListGroups, string inIsParallelExecution) { @@ -61,16 +60,16 @@ function filterKeyBasedTests(string packageName, string moduleName, string[] tes foreach string testName in tests { string updatedName = testName; string? prefix = (); - if (containsModulePrefix(packageName, moduleName, testName)) { + if containsModulePrefix(packageName, moduleName, testName) { int separatorIndex = updatedName.indexOf(MODULE_SEPARATOR); prefix = updatedName.substring(0, separatorIndex); updatedName = updatedName.substring(separatorIndex + 1); } - if (containsDataKeySuffix(updatedName)) { + if containsDataKeySuffix(updatedName) { int separatorIndex = updatedName.indexOf(DATA_KEY_SEPARATOR); string suffix = updatedName.substring(separatorIndex + 1); string testPart = updatedName.substring(0, separatorIndex); - if (testOptions.isFilterSubTestsContains(updatedName) && testOptions.getFilterSubTest(updatedName) is string[]) { + if testOptions.isFilterSubTestsContains(updatedName) && testOptions.getFilterSubTest(updatedName) is string[] { string[] subTestList = testOptions.getFilterSubTest(testPart); subTestList.push(suffix); testOptions.addFilterSubTest(testPart, subTestList); @@ -138,15 +137,15 @@ isolated function readRerunJson() returns map|error { } function containsModulePrefix(string packageName, string moduleName, string testName) returns boolean { - if (containsAPrefix(testName)) { + if containsAPrefix(testName) { return isPrefixInCorrectFormat(packageName, moduleName, testName); } return false; } function containsAPrefix(string testName) returns boolean { - if (testName.includes(MODULE_SEPARATOR)) { - if (containsDataKeySuffix(testName)) { + if testName.includes(MODULE_SEPARATOR) { + if containsDataKeySuffix(testName) { return testName.indexOf(MODULE_SEPARATOR) < testName.indexOf(DATA_KEY_SEPARATOR); } return true; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal index 38b44e2f75f7..70acd4171188 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/register.bal @@ -15,15 +15,15 @@ // under the License. import ballerina/lang.array; -isolated final TestRegistry testRegistry = new; -isolated final TestRegistry beforeSuiteRegistry = new; -isolated final TestRegistry afterSuiteRegistry = new; -isolated final TestRegistry beforeEachRegistry = new; -isolated final TestRegistry afterEachRegistry = new; - -isolated final GroupRegistry beforeGroupsRegistry = new; -isolated final GroupRegistry afterGroupsRegistry = new; -isolated final GroupStatusRegistry groupStatusRegistry = new; +final TestRegistry testRegistry = new; +final TestRegistry beforeSuiteRegistry = new; +final TestRegistry afterSuiteRegistry = new; +final TestRegistry beforeEachRegistry = new; +final TestRegistry afterEachRegistry = new; + +final GroupRegistry beforeGroupsRegistry = new; +final GroupRegistry afterGroupsRegistry = new; +final GroupStatusRegistry groupStatusRegistry = new; type TestFunction record {| string name; diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal index c6768e0dd285..8bd263c14a78 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/report.bal @@ -21,7 +21,7 @@ const string TESTS_CACHE_DIRECTORY = "tests_cache"; type ReportGenerate function (ReportData data); -isolated final ReportData reportData = new (); +final ReportData reportData = new (); ReportGenerate[] reportGenerators = [consoleReport, failedTestsReport]; @@ -139,7 +139,7 @@ isolated class ReportData { } isolated function consoleReport(ReportData data) { - if (!isSystemConsole()) { + if !isSystemConsole() { data.passedCases().forEach(isolated function(ResultData entrydata) { Result entry = new (entrydata); println("\t\t[pass] " + entry.fullName()); @@ -155,7 +155,7 @@ isolated function consoleReport(ReportData data) { int totalTestCount = data.passedCount() + data.failedCount() + data.skippedCount(); println("\n"); - if (totalTestCount == 0) { + if totalTestCount == 0 { println("\t\tNo tests found"); } else { println("\t\t" + data.passedCount().toString() + " passing"); @@ -187,8 +187,8 @@ isolated function failedTestsReport(ReportData data) { SINGLE_QUOTE + result.testSuffix() + SINGLE_QUOTE : result.testSuffix(); testNames.push(testPrefix); testModuleNames[testPrefix] = testOptions.getModuleName(); - if (result.isDataProvider()) { - if (subTestNames.hasKey(testPrefix) && subTestNames[testPrefix] is string[]) { + if result.isDataProvider() { + if subTestNames.hasKey(testPrefix) && subTestNames[testPrefix] is string[] { string[] subTestList = subTestNames[testPrefix]; subTestList.push(testSuffix); } else { diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal index 63569daa49ce..2a445209a4e0 100644 --- a/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal +++ b/misc/testerina/modules/testerina-core/src/main/ballerina/serialExecuter.bal @@ -24,7 +24,7 @@ function executeTest(TestFunction testFunction) { boolean shouldSkipDependents = false; if !isSkipFunction(testFunction) { - if (isDataDrivenTest(dataDrivenTestParams[testFunction.name])) { + if isDataDrivenTest(dataDrivenTestParams[testFunction.name]) { executeDataDrivenTestSet(testFunction); } else { shouldSkipDependents = executeNonDataDrivenTest(testFunction); @@ -104,7 +104,7 @@ function prepareDataDrivenTest(TestFunction testFunction, string key, AnyOrError reportData.onSkipped(name = testFunction.name, testType = getTestType(dataDrivenTestParams[testFunction.name])); } else { executeDataDrivenTest(testFunction, key, testType, value); - var _ = executeAfterFunction(testFunction); + _ = executeAfterFunction(testFunction); } } diff --git a/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt b/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt index 914eab9e0ff6..98f6ee85c227 100644 --- a/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt +++ b/tests/jballerina-integration-test/src/test/resources/troubleshoot/strandDump/testOutputs/balTestStrandDumpRegEx.txt @@ -4,21 +4,19 @@ Ballerina Strand Dump \[\d*/\d*/\d* \d*:\d*:\d*\] Total strand group count \t:\t3 Total strand count \t:\t9 Active strand group count\t:\t1 -Active strand count \t:\t9 +Active strand count \t:\t8 -group \d* \[QUEUED\]: \[9\] -\tstrand \d* "main" \[testOrg.testPackageWithModules.0:main\] \[WAITING\]: -\t\tat\tballerina.test.\d*.\d*.\d*:executeTests\(execute.bal:\d*\) +group \d* \[QUEUED\]: \[8\] +\tstrand \d* "main" \[testOrg.testPackageWithModules.0:main\] \[BLOCKED\]: +\t\tat\tballerina.lang.function.\d*.\d*.\d*:call\(function.bal:\d*\) +\t\t \tballerina.test.\d*.\d*.\d*:executeTestFunction\(serialExecuter.bal:\d*\) +\t\t \tballerina.test.\d*.\d*.\d*:executeNonDataDrivenTest\(serialExecuter.bal:\d*\) +\t\t \tballerina.test.\d*.\d*.\d*:executeTest\(serialExecuter.bal:\d*\) +\t\t \tballerina.test.\d*.\d*.\d*:executeTests\(execute.bal:\d*\) \t\t \tballerina.test.\d*.\d*.\d*:startSuite\(execute.bal:\d*\) \t\t \ttestOrg.testPackageWithModules.0.1.0:__execute__\(tests/test_execute-generated_1.bal:\d*\) \t\t \t\$moduleExecute -\tstrand \d* "serialWaiter" \[ballerina.test.\d*:executeTests\]\[\d*\] \[BLOCKED\]: -\t\tat\tballerina.lang.function.\d*.\d*.\d*:call\(function.bal:\d*\) -\t\t \tballerina.test.\d*.\d*.\d*:executeTestFunction\(execute.bal:\d*\) -\t\t \tballerina.test.\d*.\d*.\d*:executeNonDataDrivenTest\(execute.bal:\d*\) -\t\t \tballerina.test.\d*.\d*.\d*:executeTest\(execute.bal:\d*\) - \tstrand \d* \[ballerina.lang.function.\d*.\d*.\d*:call\]\[\d*\] \[WAITING\]: \t\tat\ttestOrg.testPackageWithModules.0.1.0:bar\(main.bal:52\) \t\t \ttestOrg.testPackageWithModules.0.1.0:foo\(main.bal:37\) @@ -52,6 +50,4 @@ group \d* \[QUEUED\]: \[9\] \t\t \ttestOrg.testPackageWithModules.anotherutils.0.1.0:\$lambda\$_0\(anotherutils.bal:34\) \tstrand \d* "w2" \[testOrg.testPackageWithModules.anotherutils.0:func3\]\[\d*\] \[BLOCKED ON WORKER MESSAGE RECEIVE\]: -\t\tat\ttestOrg.testPackageWithModules.anotherutils.0.1.0:\$lambda\$_1\(anotherutils.bal:42\) - -=========================================== +\t\tat\ttestOrg.testPackageWithModules.anotherutils.0.1.0:\$lambda\$_1\(anotherutils.bal:42\) \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt index 013213e6dc13..dce31585e22f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataProviderWithFail.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intArrayDataProviderTest:0 has failed +intArrayDataProviderTest:0 has failed. -intArrayDataProviderTest:1 has failed +intArrayDataProviderTest:1 has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt index 9c5574c7c718..c75c9294fb03 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testArrayDataRerunFailedTest.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intArrayDataProviderTest:0 has failed +intArrayDataProviderTest:0 has failed. -intArrayDataProviderTest:1 has failed +intArrayDataProviderTest:1 has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt index 48139cc61cf4..463fafcb6bee 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testDataProviderSingleFailure.txt @@ -6,6 +6,12 @@ WARNING [tests/new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +testDividingValuesNegative:2 has failed. + + + Test execution time :*****ms + [pass] testDividingValuesNegative#0 [pass] testDividingValuesNegative#1 [pass] testDividingValuesNegative#3 @@ -15,16 +21,18 @@ Running Tests with Coverage [fail] testDividingValuesNegative#2: [fail data provider for the function testDividingValuesNegative] - error {ballerina/test:0}ExecutionError ("error("{ballerina}DivisionByZero",message=" / by zero") + error {ballerina/test:0}ExecutionError ("error("{ballerina}DivisionByZero",message=" / by zero") callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 18 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 349 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 27 @@ -35,6 +43,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt index 25c1e4bf93dc..b97306f40b2a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testMapValueDataProvider.txt @@ -7,7 +7,7 @@ Running Tests with Coverage dataproviders -testGetState:1 has failed +testGetState:1 has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt index aab9e322afa6..9aeafa7c6abd 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testRerunFailedTest.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intDataProviderTest:Case1 has failed +intDataProviderTest:Case1 has failed. -intDataProviderTest:Case2 has failed +intDataProviderTest:Case2 has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt index ac01995d5ef0..ad1050cea3ea 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithAfterFailing.txt @@ -9,7 +9,7 @@ Running Tests with Coverage [fail] [after test function for the test]: error("{ballerina}DivisionByZero",message=" / by zero") callableName: afterFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 153 - callableName: afterFailsFunction$lambda84$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 + callableName: afterFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt index 8f3a0bf59c7a..b45267d98a5d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithBeforeFailing.txt @@ -9,7 +9,7 @@ Running Tests with Coverage [fail] [before test function for the test]: error("{ballerina}DivisionByZero",message=" / by zero") callableName: beforeFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 121 - callableName: beforeFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 + callableName: beforeFailsFunction$lambda62$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt index 18eb2ea3ce71..cf3fb86269d0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/DataProviderTest-testValidDataProviderWithFail.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intDataProviderTest:Case1 has failed +intDataProviderTest:Case1 has failed. -intDataProviderTest:Case2 has failed +intDataProviderTest:Case2 has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt index 182ddd929cb7..af7bfbb7990e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testNonExistingGroupExclusion.txt @@ -7,10 +7,10 @@ Running Tests groups-test.bal -testFunc4: has failed +testFunc4: has failed. -testFunc5: has failed +testFunc5: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt index 4ee078c548a4..50754fd18232 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/GroupingTest-testSingleGroupExclusion.txt @@ -7,7 +7,7 @@ Running Tests groups-test.bal -testFunc5: has failed +testFunc5: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt index 8714477b5792..c6d54b15e1de 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -8,25 +8,34 @@ Running Tests invalid-data-provider-test.bal +testInvalidDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider#0: [fail data provider for the function testInvalidDataProvider] - error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") + error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 336 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 129 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 346 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index 4138232d0ce0..32d49ce76d56 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -10,25 +10,34 @@ Running Tests invalid-data-provider-test2.bal +testInvalidDataProvider2:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider2#0: [fail data provider for the function testInvalidDataProvider2] - error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") + error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 336 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 129 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 346 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 5811136fd5b6..8f798ca85e6f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -7,25 +7,34 @@ Running Tests invalid-data-provider-test3.bal +testInvalidTupleDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidTupleDataProvider#0: [fail data provider for the function testInvalidTupleDataProvider] - error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") + error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 336 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 129 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 346 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt index d465ba49e9d7..56878002b26e 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleExecutionWithInitStartFailures.txt @@ -17,7 +17,7 @@ error: Error from start of moduleC moduleExecutionInitStartFailure.moduleB -test2: has failed +test2: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt index 25a55f23f2ee..4f39103c28cc 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/ModuleGracefulStopTest-test-listener-shutdown.txt @@ -9,10 +9,10 @@ Calling init for default module listener Calling start for moduleA listener Calling start for default module listener -main_negative_test1: has failed +main_negative_test1: has failed. -main_negative_test2: has failed +main_negative_test2: has failed. Test execution time :*****ms @@ -55,7 +55,7 @@ Calling stop for moduleA listener Calling init for moduleA listener Calling start for moduleA listener -negative_test1: has failed +negative_test1: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt index 1a0f10f7b292..d59b33f56051 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testFullTest.txt @@ -5,10 +5,10 @@ Running Tests with Coverage rerun_failed -testFunctionFail1: has failed +testFunctionFail1: has failed. -testFunctionFail2: has failed +testFunctionFail2: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt index 98aba3236a12..1d0030444837 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/RerunFailedTest-testRerunFailedTest.txt @@ -5,10 +5,10 @@ Running Tests with Coverage rerun_failed -testFunctionFail1: has failed +testFunctionFail1: has failed. -testFunctionFail2: has failed +testFunctionFail2: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SingleFileTestExecutionWithInitFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SingleFileTestExecutionWithInitFailure.txt index f7fda808fadd..bb342b37f8ea 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SingleFileTestExecutionWithInitFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SingleFileTestExecutionWithInitFailure.txt @@ -7,4 +7,4 @@ Running Tests bal-test-with-init-failure.bal error: {ballerina}DivisionByZero {"message":" / by zero"} -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt index 7b344e2d8571..cb173cadb38d 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -34,7 +34,7 @@ Running Tests callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuite fileName: skip-when-after-fails.bal lineNumber: 57 - callableName: afterSuite$lambda5$ fileName: skip-when-after-fails.bal lineNumber: 65 + callableName: afterSuite$lambda2$ fileName: skip-when-after-fails.bal lineNumber: 62 [pass] test1 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt index cba1fc4e65d5..1f36fc38fc65 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -34,7 +34,7 @@ Running Tests callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuite fileName: skip-when-before-fails.bal lineNumber: 54 - callableName: afterSuite$lambda5$ fileName: skip-when-before-fails.bal lineNumber: 62 + callableName: afterSuite$lambda2$ fileName: skip-when-before-fails.bal lineNumber: 59 [pass] test3 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt index 391227958cc7..b84349129fad 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt @@ -34,7 +34,7 @@ Running Tests callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuiteFunc fileName: skip-when-beforeGroups-fails.bal lineNumber: 81 - callableName: afterSuiteFunc$lambda9$ fileName: skip-when-beforeGroups-fails.bal lineNumber: 93 + callableName: afterSuiteFunc$lambda4$ fileName: skip-when-beforeGroups-fails.bal lineNumber: 88 [pass] testFunction [pass] testFunction2 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestExecutionWithInitFailures.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestExecutionWithInitFailures.txt index d17dbb2fdf17..2b2e90a311bd 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestExecutionWithInitFailures.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestExecutionWithInitFailures.txt @@ -4,6 +4,9 @@ Compiling source Running Tests testExecutionWithModuleInitFailure.moduleD + + Test execution time :*****ms + [pass] testFunc @@ -18,6 +21,9 @@ error: {ballerina}DivisionByZero {"message":" / by zero"} error: {ballerina}DivisionByZero {"message":" / by zero"} testExecutionWithModuleInitFailure.moduleC + + Test execution time :*****ms + [pass] testFunc @@ -27,4 +33,4 @@ error: {ballerina}DivisionByZero {"message":" / by zero"} testExecutionWithModuleInitFailure.moduleB error: {ballerina}DivisionByZero {"message":" / by zero"} -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt index 0578ef8f0060..2e5eeb8b2de5 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForCoverageFormatFlag.txt @@ -7,7 +7,7 @@ Running Tests foo -testMain: has failed +testMain: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt index af29ca0988dc..4db780f73348 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestReportTest-testWarningForReportTools.txt @@ -6,7 +6,7 @@ Running Tests with Coverage foo -testMain: has failed +testMain: has failed. Test execution time :*****ms diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt index d6b9021f01d0..c513557e1170 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataProviderWithFail.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intArrayDataProviderTest:0 has failed +intArrayDataProviderTest:0 has failed. -intArrayDataProviderTest:1 has failed +intArrayDataProviderTest:1 has failed. Test execution time :*****ms @@ -57,4 +57,4 @@ intArrayDataProviderTest:1 has failed Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt index ca6b41a8bd3e..775cf11480b3 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testArrayDataRerunFailedTest.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intArrayDataProviderTest:0 has failed +intArrayDataProviderTest:0 has failed. -intArrayDataProviderTest:1 has failed +intArrayDataProviderTest:1 has failed. Test execution time :*****ms @@ -56,4 +56,4 @@ intArrayDataProviderTest:1 has failed Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt index 08db5f4bd30c..950cecbca3f0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testDataProviderSingleFailure.txt @@ -6,6 +6,12 @@ WARNING [tests\new-data-provider-tests.bal:(153:9,153:21)] unused variable 'a' Running Tests with Coverage dataproviders + +testDividingValuesNegative:2 has failed. + + + Test execution time :*****ms + [pass] testDividingValuesNegative#0 [pass] testDividingValuesNegative#1 [pass] testDividingValuesNegative#3 @@ -15,16 +21,18 @@ Running Tests with Coverage [fail] testDividingValuesNegative#2: [fail data provider for the function testDividingValuesNegative] - error {ballerina/test:0}ExecutionError ("error("{ballerina}DivisionByZero",message=" / by zero") + error {ballerina/test:0}ExecutionError ("error("{ballerina}DivisionByZero",message=" / by zero") callableName: testDividingValuesNegative moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 191 callableName: testDividingValuesNegative$lambda14$ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 18 ",functionName="testDividingValuesNegative") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 346 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ moduleName: intg_tests.dataproviders$test.0.tests.test_execute-generated_*****lineNumber: 27 @@ -35,6 +43,9 @@ Running Tests with Coverage dataproviders.module1 + Test execution time :*****ms + + No tests found diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt index 0ce50e4c8357..509b141ac651 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testMapValueDataProvider.txt @@ -7,7 +7,7 @@ Running Tests with Coverage dataproviders -testGetState:1 has failed +testGetState:1 has failed. Test execution time :*****ms @@ -42,4 +42,4 @@ testGetState:1 has failed Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt index 326e653ce3c4..09441fb7c431 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testRerunFailedTest.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intDataProviderTest:Case1 has failed +intDataProviderTest:Case1 has failed. -intDataProviderTest:Case2 has failed +intDataProviderTest:Case2 has failed. Test execution time :*****ms @@ -56,4 +56,4 @@ intDataProviderTest:Case2 has failed Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt index 850bf54956e4..cedf1f5fc98c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithAfterFailing.txt @@ -9,7 +9,7 @@ Running Tests with Coverage [fail] [after test function for the test]: error("{ballerina}DivisionByZero",message=" / by zero") callableName: afterFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 153 - callableName: afterFailsFunction$lambda84$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 + callableName: afterFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 160 Test execution time :*****ms @@ -37,4 +37,4 @@ Running Tests with Coverage Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt index b4bee5dcaf75..cbf847fa465c 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithBeforeFailing.txt @@ -9,7 +9,7 @@ Running Tests with Coverage [fail] [before test function for the test]: error("{ballerina}DivisionByZero",message=" / by zero") callableName: beforeFailsFunction moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 121 - callableName: beforeFailsFunction$lambda72$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 + callableName: beforeFailsFunction$lambda62$ moduleName: intg_tests.dataproviders$test.0.tests.new-data-provider-tests fileName: tests/new-data-provider-tests.bal lineNumber: 127 Test execution time :*****ms @@ -36,4 +36,4 @@ Running Tests with Coverage Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt index 69a7549a761e..1b157e8c41d4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/DataProviderTest-testValidDataProviderWithFail.txt @@ -7,10 +7,10 @@ Running Tests with Coverage dataproviders -intDataProviderTest:Case1 has failed +intDataProviderTest:Case1 has failed. -intDataProviderTest:Case2 has failed +intDataProviderTest:Case2 has failed. Test execution time :*****ms @@ -57,4 +57,4 @@ intDataProviderTest:Case2 has failed Generating Test Report data-providers\target\report\test_results.json -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt index 105744904750..60b94a6f6ce0 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testNonExistingGroupExclusion.txt @@ -7,10 +7,10 @@ Running Tests groups-test.bal -testFunc4: has failed +testFunc4: has failed. -testFunc5: has failed +testFunc5: has failed. Test execution time :*****ms @@ -42,4 +42,4 @@ testFunc5: has failed 4 passing 2 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt index fb70d13eab0f..d8fe3d6c8d41 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/GroupingTest-testSingleGroupExclusion.txt @@ -7,7 +7,7 @@ Running Tests groups-test.bal -testFunc5: has failed +testFunc5: has failed. Test execution time :*****ms @@ -30,4 +30,4 @@ testFunc5: has failed 4 passing 1 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt index b843eb1c0939..73bac8d8aaa4 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider.txt @@ -8,25 +8,34 @@ Running Tests invalid-data-provider-test.bal +testInvalidDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider#0: [fail data provider for the function testInvalidDataProvider] - error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") + error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int)' cannot be passed to function expecting parameter list '(string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 336 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 129 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 ",functionName="testInvalidDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 346 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test.bal lineNumber: 37 diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index 76b7ee13ddf5..cd30dad4d291 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -10,25 +10,34 @@ Running Tests invalid-data-provider-test2.bal +testInvalidDataProvider2:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidDataProvider2#0: [fail data provider for the function testInvalidDataProvider2] - error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") + error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(int,int,int)' cannot be passed to function expecting parameter list '(string,string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 336 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 129 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 ",functionName="testInvalidDataProvider2") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 346 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test2.bal lineNumber: 39 @@ -36,4 +45,4 @@ Running Tests 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index e1f62e2ac8ad..0d50cc14f180 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -7,25 +7,34 @@ Running Tests invalid-data-provider-test3.bal +testInvalidTupleDataProvider:0 has failed. + + + Test execution time :*****ms + + [fail] testInvalidTupleDataProvider#0: [fail data provider for the function testInvalidTupleDataProvider] - error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") + error {ballerina/test:0}ExecutionError ("error("{ballerina/lang.function}IncompatibleArguments",message="arguments of incompatible types: argument list '(string,int)' cannot be passed to function expecting parameter list '(string,string)'") callableName: call moduleName: ballerina.lang.function.0 fileName: function.bal lineNumber: 37 - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 336 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 129 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 ",functionName="testInvalidTupleDataProvider") - callableName: executeTestFunction moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 346 - callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 136 - callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 123 - callableName: executeTest moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 83 - callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 58 - callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 48 + callableName: handleTestFuncOutput moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 318 + callableName: executeTestFunction moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 130 + callableName: executeDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 115 + callableName: prepareDataDrivenTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 106 + callableName: executeDataDrivenTestSet moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 63 + callableName: executeTest moduleName: ballerina.test.0 fileName: serialExecuter.bal lineNumber: 28 + callableName: executeTests moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 79 + callableName: startSuite moduleName: ballerina.test.0 fileName: execute.bal lineNumber: 52 callableName: __execute__ fileName: invalid-data-provider-test3.bal lineNumber: 36 @@ -33,4 +42,4 @@ Running Tests 0 passing 1 failing 0 skipped -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt index 87656dd15c94..1d105119cab1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleExecutionWithInitStartFailures.txt @@ -17,7 +17,7 @@ error: Error from start of moduleC moduleExecutionInitStartFailure.moduleB -test2: has failed +test2: has failed. Test execution time :*****ms @@ -41,4 +41,4 @@ test2: has failed 2 passing 1 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt index 419a5894b033..8b2c8a25287a 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/ModuleGracefulStopTest-test-listener-shutdown.txt @@ -9,10 +9,10 @@ Calling init for default module listener Calling start for moduleA listener Calling start for default module listener -main_negative_test1: has failed +main_negative_test1: has failed. -main_negative_test2: has failed +main_negative_test2: has failed. Test execution time :*****ms @@ -55,7 +55,7 @@ Calling stop for moduleA listener Calling init for moduleA listener Calling start for moduleA listener -negative_test1: has failed +negative_test1: has failed. Test execution time :*****ms @@ -79,4 +79,4 @@ negative_test1: has failed 1 failing 0 skipped Calling stop for moduleA listener -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt index 4b19adc087f0..3b8c591b532f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testFullTest.txt @@ -1,45 +1,45 @@ -Compiling source - intg_tests/rerun_failed:0.0.0 - -Running Tests with Coverage - - rerun_failed - -testFunctionFail1: has failed - - -testFunctionFail2: has failed - - - Test execution time :*****ms - - [pass] testFunctionPass1 - [pass] testFunctionPass2 - - [fail] testFunctionFail1: - - error {ballerina/test:0}TestError ("Failed!") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 - callableName: testFunctionFail1 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 35 - callableName: testFunctionFail1$lambda2$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 6 - - - [fail] testFunctionFail2: - - error {ballerina/test:0}TestError ("Failed!") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 - callableName: testFunctionFail2 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 41 - callableName: testFunctionFail2$lambda3$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 7 - - - - 2 passing - 2 failing - 0 skipped - -Generating Test Report - rerun-failed-tests\target\report\test_results.json - +Compiling source + intg_tests/rerun_failed:0.0.0 + +Running Tests with Coverage + + rerun_failed + +testFunctionFail1: has failed. + + +testFunctionFail2: has failed. + + + Test execution time :*****ms + + [pass] testFunctionPass1 + [pass] testFunctionPass2 + + [fail] testFunctionFail1: + + error {ballerina/test:0}TestError ("Failed!") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 + callableName: testFunctionFail1 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 35 + callableName: testFunctionFail1$lambda2$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 6 + + + [fail] testFunctionFail2: + + error {ballerina/test:0}TestError ("Failed!") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 + callableName: testFunctionFail2 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 41 + callableName: testFunctionFail2$lambda3$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 7 + + + + 2 passing + 2 failing + 0 skipped + +Generating Test Report + rerun-failed-tests\target\report\test_results.json + error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt index b97de420aca9..ba75ae0a1900 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/RerunFailedTest-testRerunFailedTest.txt @@ -1,43 +1,43 @@ -Compiling source - intg_tests/rerun_failed:0.0.0 - -Running Tests with Coverage - - rerun_failed - -testFunctionFail1: has failed - - -testFunctionFail2: has failed - - - Test execution time :*****ms - - - [fail] testFunctionFail1: - - error {ballerina/test:0}TestError ("Failed!") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 - callableName: testFunctionFail1 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 35 - callableName: testFunctionFail1$lambda2$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 6 - - - [fail] testFunctionFail2: - - error {ballerina/test:0}TestError ("Failed!") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 - callableName: testFunctionFail2 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 41 - callableName: testFunctionFail2$lambda3$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 7 - - - - 0 passing - 2 failing - 0 skipped - -Generating Test Report - rerun-failed-tests\target\report\test_results.json - +Compiling source + intg_tests/rerun_failed:0.0.0 + +Running Tests with Coverage + + rerun_failed + +testFunctionFail1: has failed. + + +testFunctionFail2: has failed. + + + Test execution time :*****ms + + + [fail] testFunctionFail1: + + error {ballerina/test:0}TestError ("Failed!") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 + callableName: testFunctionFail1 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 35 + callableName: testFunctionFail1$lambda2$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 6 + + + [fail] testFunctionFail2: + + error {ballerina/test:0}TestError ("Failed!") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertTrue moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 61 + callableName: testFunctionFail2 moduleName: intg_tests.rerun_failed$test.0.tests.main_test fileName: tests/main_test.bal lineNumber: 41 + callableName: testFunctionFail2$lambda3$ moduleName: intg_tests.rerun_failed$test.0.tests.test_execute-generated_*****lineNumber: 7 + + + + 0 passing + 2 failing + 0 skipped + +Generating Test Report + rerun-failed-tests\target\report\test_results.json + error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt index caf282f71d35..ad2fed3e7124 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenAfterFails.txt @@ -1,46 +1,46 @@ -Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... -warning: ignoring --includes flag since code coverage is not enabled -Compiling source - skip-when-after-fails.bal -WARNING [skip-when-after-fails.bal:(30:5,30:18)] unused variable 'i' - -Running Tests - - skip-when-after-fails.bal - [fail] [after test function for the test]: - error("{ballerina}DivisionByZero",message=" / by zero") - callableName: afterFunc fileName: skip-when-after-fails.bal lineNumber: 30 - callableName: afterFunc$lambda6$ fileName: skip-when-after-fails.bal lineNumber: 35 - - - Test execution time :*****ms - - [fail] afterSuite[after test suite function]: - error {ballerina/test:0}TestError ("Assertion Failed! - - expected: 'beforetestafterEachtestafterEach' - actual : 'beforetestafterEachafterEachtestafterEach' - - Diff : - - --- actual - +++ expected - - @@ -1,1 +1,1 @@ - - -beforetestafterEachafterEachtestafterEach - +beforetestafterEachtestafterEach - ") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 - callableName: afterSuite fileName: skip-when-after-fails.bal lineNumber: 57 - callableName: afterSuite$lambda5$ fileName: skip-when-after-fails.bal lineNumber: 65 - - [pass] test1 - [pass] test3 - - - 2 passing - 0 failing - 1 skipped -error: there are test failures +Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... +warning: ignoring --includes flag since code coverage is not enabled +Compiling source + skip-when-after-fails.bal +WARNING [skip-when-after-fails.bal:(30:5,30:18)] unused variable 'i' + +Running Tests + + skip-when-after-fails.bal + [fail] [after test function for the test]: + error("{ballerina}DivisionByZero",message=" / by zero") + callableName: afterFunc fileName: skip-when-after-fails.bal lineNumber: 30 + callableName: afterFunc$lambda6$ fileName: skip-when-after-fails.bal lineNumber: 35 + + + Test execution time :*****ms + + [fail] afterSuite[after test suite function]: + error {ballerina/test:0}TestError ("Assertion Failed! + + expected: 'beforetestafterEachtestafterEach' + actual : 'beforetestafterEachafterEachtestafterEach' + + Diff : + + --- actual + +++ expected + + @@ -1,1 +1,1 @@ + + -beforetestafterEachafterEachtestafterEach + +beforetestafterEachtestafterEach + ") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 + callableName: afterSuite fileName: skip-when-after-fails.bal lineNumber: 57 + callableName: afterSuite$lambda2$ fileName: skip-when-after-fails.bal lineNumber: 62 + + [pass] test1 + [pass] test3 + + + 2 passing + 0 failing + 1 skipped +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt index 6aadf6d5847f..1bbdf4d8fb9f 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeFails.txt @@ -1,45 +1,45 @@ -Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... -warning: ignoring --includes flag since code coverage is not enabled -Compiling source - skip-when-before-fails.bal -WARNING [skip-when-before-fails.bal:(28:5,28:18)] unused variable 'i' - -Running Tests - - skip-when-before-fails.bal - [fail] [before test function for the test]: - error("{ballerina}DivisionByZero",message=" / by zero") - callableName: before fileName: skip-when-before-fails.bal lineNumber: 28 - callableName: before$lambda6$ fileName: skip-when-before-fails.bal lineNumber: 32 - - - Test execution time :*****ms - - [fail] afterSuite[after test suite function]: - error {ballerina/test:0}TestError ("Assertion Failed! - - expected: 'beforetest3afterEach' - actual : 'beforeafterEachafterEachtest3afterEach' - - Diff : - - --- actual - +++ expected - - @@ -1,1 +1,1 @@ - - -beforeafterEachafterEachtest3afterEach - +beforetest3afterEach - ") - callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 - callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 - callableName: afterSuite fileName: skip-when-before-fails.bal lineNumber: 54 - callableName: afterSuite$lambda5$ fileName: skip-when-before-fails.bal lineNumber: 62 - - [pass] test3 - - - 1 passing - 0 failing - 2 skipped -error: there are test failures +Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run... +warning: ignoring --includes flag since code coverage is not enabled +Compiling source + skip-when-before-fails.bal +WARNING [skip-when-before-fails.bal:(28:5,28:18)] unused variable 'i' + +Running Tests + + skip-when-before-fails.bal + [fail] [before test function for the test]: + error("{ballerina}DivisionByZero",message=" / by zero") + callableName: before fileName: skip-when-before-fails.bal lineNumber: 28 + callableName: before$lambda6$ fileName: skip-when-before-fails.bal lineNumber: 32 + + + Test execution time :*****ms + + [fail] afterSuite[after test suite function]: + error {ballerina/test:0}TestError ("Assertion Failed! + + expected: 'beforetest3afterEach' + actual : 'beforeafterEachafterEachtest3afterEach' + + Diff : + + --- actual + +++ expected + + @@ -1,1 +1,1 @@ + + -beforeafterEachafterEachtest3afterEach + +beforetest3afterEach + ") + callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 + callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 + callableName: afterSuite fileName: skip-when-before-fails.bal lineNumber: 54 + callableName: afterSuite$lambda2$ fileName: skip-when-before-fails.bal lineNumber: 59 + + [pass] test3 + + + 1 passing + 0 failing + 2 skipped +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt index 007a069073c9..333ed93d8be7 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/SkipTestsTestCase-testSkipWhenBeforeGroupsFails.txt @@ -34,7 +34,7 @@ Running Tests callableName: createBallerinaError moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 41 callableName: assertEquals moduleName: ballerina.test.0 fileName: assert.bal lineNumber: 109 callableName: afterSuiteFunc fileName: skip-when-beforeGroups-fails.bal lineNumber: 81 - callableName: afterSuiteFunc$lambda9$ fileName: skip-when-beforeGroups-fails.bal lineNumber: 93 + callableName: afterSuiteFunc$lambda4$ fileName: skip-when-beforeGroups-fails.bal lineNumber: 88 [pass] testFunction [pass] testFunction2 @@ -43,4 +43,4 @@ Running Tests 2 passing 0 failing 3 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt index c235bb1dec8d..90b721b05ef1 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt @@ -4,6 +4,9 @@ Compiling source Running Tests testExecutionWithModuleInitFailure.moduleD + + Test execution time :*****ms + [pass] testFunc @@ -18,6 +21,9 @@ error: {ballerina}DivisionByZero {"message":" / by zero"} error: {ballerina}DivisionByZero {"message":" / by zero"} testExecutionWithModuleInitFailure.moduleC + + Test execution time :*****ms + [pass] testFunc @@ -27,4 +33,4 @@ error: {ballerina}DivisionByZero {"message":" / by zero"} testExecutionWithModuleInitFailure.moduleB error: {ballerina}DivisionByZero {"message":" / by zero"} -error: there are test failures +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt index f935e359e798..71bb5b5629d2 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForCoverageFormatFlag.txt @@ -7,7 +7,7 @@ Running Tests foo -testMain: has failed +testMain: has failed. Test execution time :*****ms @@ -50,4 +50,4 @@ testMain: has failed 2 passing 0 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt index 3752a560a25c..d7e5939484ed 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestReportTest-testWarningForReportTools.txt @@ -6,7 +6,7 @@ Running Tests with Coverage foo -testMain: has failed +testMain: has failed. Test execution time :*****ms @@ -53,4 +53,4 @@ testMain: has failed Generating Test Report*****project-based-tests\test-report-tests\target\report\test_results.json warning: Could not find the required HTML report tools for code coverage at \lib\tools\coverage\report.zip -error: there are test failures \ No newline at end of file +error: there are test failures From baee9813dcc59531a55723824e412cbe329d2116 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Thu, 22 Feb 2024 11:31:06 +0530 Subject: [PATCH 36/37] Fix build failure --- .../project_b_100/resources/expectedDeps.toml | 3 +-- .../ballerinalang/testerina/test/TestparallelizationTest.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml b/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml index 379956ea2017..46d0b588b37c 100644 --- a/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml +++ b/cli/ballerina-cli/src/test/resources/test-resources/balTestWithStickyFlag/project_b_100/resources/expectedDeps.toml @@ -59,8 +59,7 @@ dependencies = [ {org = "ballerina", name = "lang.error"} ] modules = [ - {org = "ballerina", packageName = "test", moduleName = "test"}, - {org = "ballerina", packageName = "test", moduleName = "test.test"} + {org = "ballerina", packageName = "test", moduleName = "test"} ] [[package]] diff --git a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java index cbe679ec2a9a..1b9e6b512940 100644 --- a/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java +++ b/tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java @@ -306,7 +306,7 @@ public void testIsolatedSetUpTearDown() throws BallerinaTestException, IOExcepti new HashMap<>(), projectPath, false); Float executionTimeW1 = getTimeForTestExecution(output); Assert.assertTrue(output.contains("40 passing") && output.contains("0 failing")); - Assert.assertTrue(executionTimeW1 > 2 * executionTimeW30 ); + Assert.assertTrue(executionTimeW1 > 2 * executionTimeW30); } @Test From 37f2a744f203d7f77237231be1f2db7bd462dcb2 Mon Sep 17 00:00:00 2001 From: Thevakumar-Luheerathan Date: Sun, 25 Feb 2024 20:30:50 -0800 Subject: [PATCH 37/37] Fix windows build failure --- .../InvalidDataProviderTestCase-testInvalidDataProvider2.txt | 2 +- ...InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt | 2 +- .../command-outputs/windows/TestExecutionWithInitFailures.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt index cd30dad4d291..17c85728d424 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidDataProvider2.txt @@ -45,4 +45,4 @@ testInvalidDataProvider2:0 has failed. 0 passing 1 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt index 0d50cc14f180..66aedc10ba64 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/InvalidDataProviderTestCase-testInvalidTupleDataProvider.txt @@ -42,4 +42,4 @@ testInvalidTupleDataProvider:0 has failed. 0 passing 1 failing 0 skipped -error: there are test failures \ No newline at end of file +error: there are test failures \ No newline at end of file diff --git a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt index 90b721b05ef1..aaa44aeeb4df 100644 --- a/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt +++ b/tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt @@ -33,4 +33,4 @@ error: {ballerina}DivisionByZero {"message":" / by zero"} testExecutionWithModuleInitFailure.moduleB error: {ballerina}DivisionByZero {"message":" / by zero"} -error: there are test failures \ No newline at end of file +error: there are test failures \ No newline at end of file