Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel support #1357

Closed
wants to merge 39 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
387a7b4
add support for --threads to runner & associated classes
boaty82 Apr 19, 2018
ce0814d
update classes to be thread safe
boaty82 Apr 19, 2018
dafc465
update HTMLFormatter to be thread safe
boaty82 Apr 19, 2018
3163156
update JSONFormatter to be threadsafe
boaty82 Apr 19, 2018
cc2fe4b
update PrettyFormatter to be threadsafe
boaty82 Apr 19, 2018
5fa1a0f
update UsageFormatter to be threadsafe
boaty82 Apr 19, 2018
93d95d2
update RerunFormatter to be threadsafe
boaty82 Apr 19, 2018
c6578cd
update JUnitFormatter to be threadsafe
boaty82 Apr 19, 2018
877a74a
improvement by not cloning node but moving it between documents
boaty82 Apr 19, 2018
6c6e181
update JUnitFormatter to not create new dom for each test case
boaty82 Apr 20, 2018
221b8e2
update TestNGFormatter to be threadsafe
boaty82 Apr 20, 2018
1359236
update JUNitFormatter. import order & remove duplicate code
boaty82 Apr 20, 2018
ea86ad7
update AndroidInstrumentationReporter to be threadsafe
boaty82 Apr 20, 2018
93ff064
update Formatter docs - inform that implementations should be threadsafe
boaty82 Apr 20, 2018
a6a5dd1
revert earlier changes as is threadsafe as instance not being shared
boaty82 Apr 20, 2018
c2b05b2
fix another class to be threadsafe
boaty82 Apr 20, 2018
b617c14
TODO: last thread issue, but it's a big one!
boaty82 Apr 21, 2018
cba4f1a
fix a TODO in a test
boaty82 Apr 21, 2018
1a13a7d
fix issue where no features found
boaty82 Apr 21, 2018
910682e
threadsafe ObjectFactory
boaty82 Apr 22, 2018
c504447
fix issue with Glue being shared by multiple threads.
boaty82 Apr 22, 2018
2b520f5
add test to ensure that Glue template isn't used during test execution
boaty82 Apr 22, 2018
92f611f
introduce a TimelineFormatter report
boaty82 Apr 23, 2018
41b9d99
correct some JS
boaty82 Apr 23, 2018
38c7e63
update usage instructions
boaty82 Apr 23, 2018
cc3b45b
add some TODO's to be worked on as new features/improvements
boaty82 Apr 23, 2018
17cb791
Use a queue so can run tests in a less sequential order
boaty82 Apr 23, 2018
a8f7e0d
add ability to run sync tests followed by remaining
boaty82 Apr 24, 2018
43a3e25
fix intermittent failing test - method execution order not guaranteed
boaty82 Apr 24, 2018
f8be9b1
improve some ThreadLocal usages
boaty82 Apr 24, 2018
2961b9e
improve some more ThreadLocal usages and code clean up
boaty82 Apr 25, 2018
c3e0ea7
add threading test for SpringFactory to ensure thread safety
boaty82 Apr 25, 2018
2e6782b
minor code change to pass smaller scope parameter data
boaty82 Apr 25, 2018
15322ab
fix test failing remotely
boaty82 Apr 25, 2018
4d34a6a
fix issue in Timeline report where times were wrong. EventBus sends …
boaty82 Apr 26, 2018
2af3320
add scenario navigator to top of report so can jump to time it ran
boaty82 Apr 27, 2018
d163cab
add tag filter to timeline
boaty82 May 1, 2018
c288668
Merge branch 'master' into parallel-support
boaty82 May 14, 2018
494fe55
Merge branch 'master' into parallel-support (forgot to 'git add .')
boaty82 May 14, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -83,27 +83,8 @@ static class StatusCodes {
*/
private int numberOfTests;

/**
* The severest step result of the current test execution.
* This might be a step or hook result.
*/
private Result severestResult;

/**
* The uri of the feature file of the current test case.
*/
private String currentUri;

/**
* The name of the current feature.
*/
private String currentFeatureName;

/**
* The name of the current test case.
*/
private String currentTestCaseName;

private ThreadLocal<CurrentFeature> featureUnderTest = new ThreadLocal<CurrentFeature>();

/**
* The event handler for the {@link TestSourceRead} events.
*/
Expand Down Expand Up @@ -172,40 +153,43 @@ void testSourceRead(final TestSourceRead event) {
}

void startTestCase(final TestCase testCase) {
if (!testCase.getUri().equals(currentUri)) {
currentUri = testCase.getUri();
currentFeatureName = testSources.getFeatureName(currentUri);
CurrentFeature currentFeature = featureUnderTest.get();
if (currentFeature == null || !testCase.getUri().equals(currentFeature.uri)) {
currentFeature = new CurrentFeature(testCase.getUri(), testSources.getFeatureName(testCase.getUri()));
featureUnderTest.set(currentFeature);
}
// Since the names of test cases are not guaranteed to be unique, we must check for unique names
currentTestCaseName = calculateUniqueTestName(testCase);
resetSeverestResult();
final Bundle testStart = createBundle(currentFeatureName, currentTestCaseName);
currentFeature.testCaseName = calculateUniqueTestName(testCase);
currentFeature.resetSeverestResult();
final Bundle testStart = createBundle(currentFeature.featureName, currentFeature.testCaseName);
instrumentation.sendStatus(StatusCodes.START, testStart);
}

void finishTestStep(final Result result) {
checkAndSetSeverestStepResult(result);
CurrentFeature currentFeature = featureUnderTest.get();
currentFeature.checkAndSetSeverestStepResult(result);
}

void finishTestCase() {
final Bundle testResult = createBundle(currentFeatureName, currentTestCaseName);
final CurrentFeature currentFeature = featureUnderTest.get();
final Bundle testResult = createBundle(currentFeature.featureName, currentFeature.testCaseName);

switch (severestResult.getStatus()) {
switch (currentFeature.severestResult.getStatus()) {
case FAILED:
if (severestResult.getError() instanceof AssertionError) {
testResult.putString(StatusKeys.STACK, severestResult.getErrorMessage());
if (currentFeature.severestResult.getError() instanceof AssertionError) {
testResult.putString(StatusKeys.STACK, currentFeature.severestResult.getErrorMessage());
instrumentation.sendStatus(StatusCodes.FAILURE, testResult);
} else {
testResult.putString(StatusKeys.STACK, getStackTrace(severestResult.getError()));
testResult.putString(StatusKeys.STACK, getStackTrace(currentFeature.severestResult.getError()));
instrumentation.sendStatus(StatusCodes.ERROR, testResult);
}
break;
case AMBIGUOUS:
testResult.putString(StatusKeys.STACK, getStackTrace(severestResult.getError()));
testResult.putString(StatusKeys.STACK, getStackTrace(currentFeature.severestResult.getError()));
instrumentation.sendStatus(StatusCodes.ERROR, testResult);
break;
case PENDING:
testResult.putString(StatusKeys.STACK, severestResult.getErrorMessage());
testResult.putString(StatusKeys.STACK, currentFeature.severestResult.getErrorMessage());
instrumentation.sendStatus(StatusCodes.ERROR, testResult);
break;
case PASSED:
Expand All @@ -217,7 +201,7 @@ void finishTestCase() {
instrumentation.sendStatus(StatusCodes.ERROR, testResult);
break;
default:
throw new IllegalStateException("Unexpected result status: " + severestResult.getStatus());
throw new IllegalStateException("Unexpected result status: " + currentFeature.severestResult.getStatus());
}
}

Expand Down Expand Up @@ -245,33 +229,6 @@ private String getLastSnippet() {
return runtime.getSnippets().get(runtime.getSnippets().size() - 1);
}

/**
* Resets the severest test result for the next scenario life cycle.
*/
private void resetSeverestResult() {
severestResult = null;
}

/**
* Checks if the given {@code result} is more severe than the current {@code severestResult} and
* updates the {@code severestResult} if that should be the case.
*
* @param result the {@link Result} to check
*/
private void checkAndSetSeverestStepResult(final Result result) {
final boolean firstResult = severestResult == null;
if (firstResult) {
severestResult = result;
return;
}

final boolean currentIsPassed = severestResult.is(Result.Type.PASSED);
final boolean nextIsNotPassed = !result.is(Result.Type.PASSED);
if (currentIsPassed && nextIsNotPassed) {
severestResult = result;
}
}

/**
* Creates a string representation of the given {@code throwable}'s stacktrace.
*
Expand Down Expand Up @@ -331,4 +288,43 @@ private String calculateUniqueTestName(TestCase testCase) {
return uniqueTestNameForTestCase.get(testCase);
}

private class CurrentFeature {
private final String uri;
private final String featureName;
private String testCaseName;
private Result severestResult;

CurrentFeature(final String uri, final String featureName) {
this.uri = uri;
this.featureName = featureName;
}

/**
* Resets the severest test result for the next scenario life cycle.
*/
private void resetSeverestResult() {
severestResult = null;
}

/**
* Checks if the given {@code result} is more severe than the current {@code severestResult} and
* updates the {@code severestResult} if that should be the case.
*
* @param result the {@link Result} to check
*/
private void checkAndSetSeverestStepResult(final Result result) {
final boolean firstResult = severestResult == null;
if (firstResult) {
severestResult = result;
return;
}

final boolean currentIsPassed = severestResult.is(Result.Type.PASSED);
final boolean nextIsNotPassed = !result.is(Result.Type.PASSED);
if (currentIsPassed && nextIsNotPassed) {
severestResult = result;
}
}
}

}
7 changes: 7 additions & 0 deletions core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,13 @@
<artifactId>webbit-rest</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-junit</artifactId>
<version>2.0.0.0</version>
<scope>test</scope>
</dependency>

</dependencies>

<build>
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/cucumber/api/CucumberOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,8 @@
*/
String[] junit() default {};

/**
* @return the number of parallel threads to run scenarios
*/
int threads() default 1;
}
15 changes: 15 additions & 0 deletions core/src/main/java/cucumber/api/event/TestGroupRunFinished.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package cucumber.api.event;

public class TestGroupRunFinished extends TimeStampedEvent {

private final String type;

public TestGroupRunFinished(final String type, final Long timeStamp) {
super(timeStamp);
this.type = type;
}

public String getType() {
return type;
}
}
27 changes: 27 additions & 0 deletions core/src/main/java/cucumber/api/event/TestGroupRunStarted.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package cucumber.api.event;

public class TestGroupRunStarted extends TimeStampedEvent {

private final String type;
private final int threadCount;
private final int featureCount;

public TestGroupRunStarted(final String type, final int threadCount, final int featureCount, final Long timeStamp) {
super(timeStamp);
this.type = type;
this.threadCount = threadCount;
this.featureCount = featureCount;
}

public String getType() {
return type;
}

public int getThreadCount() {
return threadCount;
}

public int getFeatureCount() {
return featureCount;
}
}
2 changes: 2 additions & 0 deletions core/src/main/java/cucumber/api/formatter/Formatter.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
* This is the interface you should implement if you want your own custom
* formatter.
*
* <b>NOTE: Implementations should be threadsafe to support '--threads' parameter</b>
*
* @see EventListener
* @see Plugin
*/
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package cucumber.api.formatter;

public class NiceRetrievableAppendable extends NiceAppendable {

private final StringBuilder builder;

public NiceRetrievableAppendable(StringBuilder out) {
super(out);
this.builder = out;
}

public String printAll() {
return builder.toString();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package cucumber.runner;

import cucumber.runtime.Glue;
import cucumber.runtime.StepDefinitionMatch;
import cucumber.runtime.UnreportedStepExecutor;
import gherkin.pickles.Argument;
import gherkin.pickles.PickleLocation;
import gherkin.pickles.PickleRow;
import gherkin.pickles.PickleStep;
import gherkin.pickles.PickleString;
import gherkin.pickles.PickleTable;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

//TODO: hmm not so sure about the Glue reference as have cloned() it since, but without some real usages I'm not sure at this point
public class DefaultUnreportedStepExecutor implements UnreportedStepExecutor {

private final Glue glue;

public DefaultUnreportedStepExecutor(final Glue glue) {
this.glue = glue;
}

//TODO: Maybe this should go into the cucumber step execution model and it should return the result of that execution!
@Override
public void runUnreportedStep(final String featurePath, final String language,
final String stepName, final int line,
final List<PickleRow> dataTableRows, final PickleString docString) throws Throwable {
List<Argument> arguments = new ArrayList<Argument>();
if (dataTableRows != null && !dataTableRows.isEmpty()) {
arguments.add(new PickleTable(dataTableRows));
} else if (docString != null) {
arguments.add(docString);
}
PickleStep step = new PickleStep(stepName, arguments, Collections.<PickleLocation>emptyList());

StepDefinitionMatch match = glue.stepDefinitionMatch(featurePath, step);
if (match == null) {
UndefinedStepException error = new UndefinedStepException(step);

StackTraceElement[] originalTrace = error.getStackTrace();
StackTraceElement[] newTrace = new StackTraceElement[originalTrace.length + 1];
newTrace[0] = new StackTraceElement("✽", "StepDefinition", featurePath, line);
System.arraycopy(originalTrace, 0, newTrace, 1, originalTrace.length);
error.setStackTrace(newTrace);

throw error;
}
match.runStep(language, null);
}
}
Loading