diff --git a/event-protocol/README.md b/event-protocol/README.md index 9bd64f99e4..17fc7fe289 100644 --- a/event-protocol/README.md +++ b/event-protocol/README.md @@ -85,6 +85,74 @@ Example (Java stack trace): [snippet](examples/events/004_attachment-stacktrace.json) ``` +### test-run-started {#test-run-started} + +Signals the start of a test run. Contains details of the context of the run, like the working directory, start time etc. + +Example: + +```json +[snippet](examples/events/005_test-run-started.json) +``` + +### test-case-prepared {#test-case-prepared} + +Contains the details of a test case that's ready to be executed. The `sourceLocation` should match the `uri` of preceeding `source` and `gherkin-document` events. + +Each `step` has an `actionLocation` pointing to the source of the glue code that will be invoked when the step is executed. The `sourceLocation` points to the source of the step in a Gherkin document. + +Example: + +```json +[snippet](examples/events/006_test-case-prepared.json) +``` + +### test-case-started + +Signals the start of executing a test case + +Example: + +```json +[snippet](examples/events/007_test-case-started.json) +``` + +### test-step-started + +Signals the start of executing a test step within a test case. The `index` locates the step within the array of steps sent in the preceeding `test-case-prepared` event. + +Example: + +```json +[snippet](examples/events/008_test-step-started.json) +``` + +### test-step-finished + +Signals the end of executing a test step. The result + +Example of a passing step: + +```json +[snippet](examples/events/009_test-step-finished.json) +``` + +Example of failing step: + +```json +[snippet](examples/events/011_test-step-finished.json) +``` + +Note that undefined is just a special case of failure. The `result` will always have an `exception` when the step has not passed. + +### test-case-finished + +Signals the end of executing a whole test case (scenario or example row). + +```json +[snippet](examples/events/012_test-case-finished.json) +``` + ## Implementation * Cucumber events are formatted as [Newline Delimited JSON](http://ndjson.org) @@ -128,7 +196,9 @@ We'll manage this by adding a version to a specific event, where needed. For now There are a few constraints about the order of events: * A [source](#event-source) event must be received before any - [attachment](#event-attachment) events referring to it + [attachment](#event-attachment), `test-case-*` or `test-step-*` events referring to it +* [test-case-started](#test-case-started), [test-step-started](#test-step-started), [test-step-finished](#test-step-finished), [test-case-finished](#test-case-finished) events are expected to be received in that order. They should be preceeded by the relevant [test-case-prepared](#test-case-prepared) event describing the test case and its steps. +* `test-*-started` events are optional ## Event validation {#validation} diff --git a/event-protocol/examples/events.ndjson b/event-protocol/examples/events.ndjson index 934edc315a..e5d3a332f9 100644 --- a/event-protocol/examples/events.ndjson +++ b/event-protocol/examples/events.ndjson @@ -3,3 +3,11 @@ {"pickle":{"language":"en","locations":[{"column":3,"line":2}],"name":"World","steps":[{"arguments":[],"locations":[{"column":11,"line":3}],"text":"a step"}],"tags":[]},"type":"pickle","uri":"features/hello.feature"} {"type":"attachment","source":{"uri":"features/hello.feature","start":{"line":3,"column":7}},"data":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAELUExURUdwTACoGAClFACqHACoFwCnGQCnFQCnGACoGQCoGQCoGACuGgCnFwCiFwCoGACoGACnFwClFgCpGACoFwD/AACnGAC2JACoGACnFwCqKgCoGACoGACqGQCoFwCnFwCoFwB/AACnGACoGAC/AACoGACoFgCmFwCnFwCnFwCqFgCmFgCZAACoGC64Ql3IbA2sJBuxMBiwLhewLR2yMgmrIOn36/z+/BGuJ9/04rHkuIDUjGHJcA6tJeD04wGoGQusIpnco+T257PlugSpHN704XDOfQeqHiCzNc7u0+z57t3z4HvSh7fmvj69UJzdpSi2PNjy3AKpGlrHahqxMBCtJpHZm83u0jS6R0vCXEWczDEAAAAsdFJOUwDIJQlYPSP7cFCqE1cW+mf4ImiBAXQHPu8GfP0zzoyOAp2wBOg4a2PqOU4FxgPjdgAAAK9JREFUGNNNj1UCwkAMRBcotLi7uw1S3N3d4f4nYSvIfO28bCYJIaI0OSabJ1+pFaDKpD4+zUJS1CHVWSwer+esDNhEQP+fV/f5uAd4XEIesNldl5WK0KSlQAtcDkc5xUcBA+y3tL/UKQIhCpLA+sTzw1K7UQVHQTyB23QyKraaNSAgTOGAQb9bqBeAoDg2EpMT4XdLm3rD8qrO7zFWO7UW8+86oodJp/zzRGUwyq83Mjcb8VXl0ZMAAAAASUVORK5CYII=","media":{"encoding":"base64","type":"image/png"}} {"type":"attachment","source":{"uri":"features/hello.feature","start":{"line":3,"column":7}},"data":"Exception in thread \"main\" java.lang.NullPointerException\n at com.example.myproject.Book.getTitle(Book.java:16)\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.stacktrace.java+plain"}} +{"type":"test-run-started","workingDirectory":"/Users/matt/projects/cucumber-ruby","timestamp":1489683458} +{"type":"test-case-prepared","sourceLocation":{"uri":"features/passing.feature","line":2},"steps":[{"actionLocation":{"uri":"/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb","line":28},"sourceLocation":{"uri":"/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb","line":28}},{"actionLocation":{"uri":"features/passing.feature","line":3},"sourceLocation":{"uri":"features/passing.feature","line":3}}]} +{"type":"test-case-started","sourceLocation":{"uri":"features/passing.feature","line":2}} +{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":0} +{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":0,"result":{"status":"passed","duration":3000}} +{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":1} +{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":1,"result":{"status":"undefined","exception":{"message":"Undefined step: \"this step passes\"","type":"Cucumber::Core::Test::Result::Undefined","stackTrace":["features/passing.feature:3:in `Given this step passes'"]}}} +{"type":"test-case-finished","sourceLocation":{"uri":"features/passing.feature","line":2},"result":{"status":"undefined","duration":13163000,"exception":{"message":"Undefined step: \"this step passes\"","type":"Cucumber::Core::Test::Result::Undefined","stackTrace":["features/passing.feature:3:in `Given this step passes'"]}}} diff --git a/event-protocol/examples/events/005_test-run-started.json b/event-protocol/examples/events/005_test-run-started.json new file mode 100644 index 0000000000..ea0bcfec01 --- /dev/null +++ b/event-protocol/examples/events/005_test-run-started.json @@ -0,0 +1,5 @@ +{ + "type":"test-run-started", + "workingDirectory":"/Users/matt/projects/cucumber-ruby", + "timestamp":1489683458 +} diff --git a/event-protocol/examples/events/006_test-case-prepared.json b/event-protocol/examples/events/006_test-case-prepared.json new file mode 100644 index 0000000000..75b0b7fe80 --- /dev/null +++ b/event-protocol/examples/events/006_test-case-prepared.json @@ -0,0 +1,29 @@ +{ + "type": "test-case-prepared", + "sourceLocation": { + "uri": "features/passing.feature", + "line": 2 + }, + "steps": [ + { + "actionLocation": { + "uri": "/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb", + "line": 28 + }, + "sourceLocation": { + "uri": "/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb", + "line": 28 + } + }, + { + "actionLocation": { + "uri": "features/passing.feature", + "line": 3 + }, + "sourceLocation": { + "uri": "features/passing.feature", + "line": 3 + } + } + ] +} diff --git a/event-protocol/examples/events/007_test-case-started.json b/event-protocol/examples/events/007_test-case-started.json new file mode 100644 index 0000000000..88ced740b8 --- /dev/null +++ b/event-protocol/examples/events/007_test-case-started.json @@ -0,0 +1,7 @@ +{ + "type": "test-case-started", + "sourceLocation": { + "uri": "features/passing.feature", + "line": 2 + } +} diff --git a/event-protocol/examples/events/008_test-step-started.json b/event-protocol/examples/events/008_test-step-started.json new file mode 100644 index 0000000000..e68b88c3fb --- /dev/null +++ b/event-protocol/examples/events/008_test-step-started.json @@ -0,0 +1,10 @@ +{ + "type": "test-step-started", + "testCase": { + "sourceLocation": { + "uri": "features/passing.feature", + "line": 2 + } + }, + "index": 0 +} diff --git a/event-protocol/examples/events/009_test-step-finished.json b/event-protocol/examples/events/009_test-step-finished.json new file mode 100644 index 0000000000..1084ee8097 --- /dev/null +++ b/event-protocol/examples/events/009_test-step-finished.json @@ -0,0 +1,14 @@ +{ + "type": "test-step-finished", + "testCase": { + "sourceLocation": { + "uri": "features/passing.feature", + "line": 2 + } + }, + "index": 0, + "result": { + "status": "passed", + "duration": 3000 + } +} diff --git a/event-protocol/examples/events/010_test-step-started.json b/event-protocol/examples/events/010_test-step-started.json new file mode 100644 index 0000000000..6d4069f101 --- /dev/null +++ b/event-protocol/examples/events/010_test-step-started.json @@ -0,0 +1,10 @@ +{ + "type": "test-step-started", + "testCase": { + "sourceLocation": { + "uri": "features/passing.feature", + "line": 2 + } + }, + "index": 1 +} diff --git a/event-protocol/examples/events/011_test-step-finished.json b/event-protocol/examples/events/011_test-step-finished.json new file mode 100644 index 0000000000..b6554f09b7 --- /dev/null +++ b/event-protocol/examples/events/011_test-step-finished.json @@ -0,0 +1,20 @@ +{ + "type": "test-step-finished", + "testCase": { + "sourceLocation": { + "uri": "features/passing.feature", + "line": 2 + } + }, + "index": 1, + "result": { + "status": "undefined", + "exception": { + "message": "Undefined step: \"this step passes\"", + "type": "Cucumber::Core::Test::Result::Undefined", + "stackTrace": [ + "features/passing.feature:3:in `Given this step passes'" + ] + } + } +} diff --git a/event-protocol/examples/events/012_test-case-finished.json b/event-protocol/examples/events/012_test-case-finished.json new file mode 100644 index 0000000000..eb6e744e0a --- /dev/null +++ b/event-protocol/examples/events/012_test-case-finished.json @@ -0,0 +1,18 @@ +{ + "type": "test-case-finished", + "sourceLocation": { + "uri": "features/passing.feature", + "line": 2 + }, + "result": { + "status": "undefined", + "duration": 13163000, + "exception": { + "message": "Undefined step: \"this step passes\"", + "type": "Cucumber::Core::Test::Result::Undefined", + "stackTrace": [ + "features/passing.feature:3:in `Given this step passes'" + ] + } + } +} diff --git a/event-protocol/schemas/attachment.json b/event-protocol/schemas/attachment.json index d3adb162f6..39fcefdea6 100644 --- a/event-protocol/schemas/attachment.json +++ b/event-protocol/schemas/attachment.json @@ -13,7 +13,7 @@ "uri": { "type": "string" }, - "start": { "$ref": "defs.json#/location" } + "start": { "$ref": "defs.json#/ast-location" } }, "required": [ "uri", diff --git a/event-protocol/schemas/defs.json b/event-protocol/schemas/defs.json index efa65351aa..c4c7cd3665 100644 --- a/event-protocol/schemas/defs.json +++ b/event-protocol/schemas/defs.json @@ -3,6 +3,23 @@ "title": "definitions", "description": "Re-usable parts of the schema, referenced by others", "location": { + "type": "object", + "properties": { + "line": { + "type": "integer", + "minimum": 1 + }, + "uri": { + "type": "string" + } + }, + "required": [ + "line", + "uri" + ], + "additionalProperties": false + }, + "gherkin-node-location": { "type": "object", "properties": { "line": { @@ -12,9 +29,6 @@ "column": { "type": "integer", "minimum": 0 - }, - "uri": { - "type": "string" } }, "required": [ @@ -73,5 +87,44 @@ "location" ], "additionalProperties": false + }, + "result": { + "type": "object", + "properties": { + "status": { + "enum": [ + "passed", + "failed", + "pending", + "skipped", + "undefined" + ] + }, + "duration": { + "type": "integer" + }, + "exception": { "$ref": "defs.json#/exception" } + }, + "required": [ "status" ], + "additionalProperties": false + }, + "exception": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "stackTrace": { + "type": "array", + "items": { + "type": "string" + } + }, + "type": { + "type": "string" + } + }, + "required": [ "message", "stackTrace" ], + "additionalProperties": false } } diff --git a/event-protocol/schemas/pickle.json b/event-protocol/schemas/pickle.json index 239e2027a7..76bd42f2e1 100644 --- a/event-protocol/schemas/pickle.json +++ b/event-protocol/schemas/pickle.json @@ -26,7 +26,7 @@ "locations": { "type": "array", "minLength": 1, - "items": { "$ref": "defs.json#/location" } + "items": { "$ref": "defs.json#/gherkin-node-location" } }, "steps": { "type": "array", @@ -43,7 +43,7 @@ "locations": { "type": "array", "minLength": 1, - "items": { "$ref": "defs.json#/location" } + "items": { "$ref": "defs.json#/gherkin-node-location" } } }, "required": [ diff --git a/event-protocol/schemas/test-case-finished.json b/event-protocol/schemas/test-case-finished.json new file mode 100644 index 0000000000..4d778fc652 --- /dev/null +++ b/event-protocol/schemas/test-case-finished.json @@ -0,0 +1,19 @@ +{ + "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-case-finished.json#", + "title": "test-case-finished", + "description": "A test case has finished executing", + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "sourceLocation": { + "$ref": "defs.json#/location" + }, + "result": { + "$ref": "defs.json#/result" + } + }, + "required": [ "sourceLocation", "result" ], + "additionalProperties": false +} diff --git a/event-protocol/schemas/test-case-prepared.json b/event-protocol/schemas/test-case-prepared.json new file mode 100644 index 0000000000..00d4cdd418 --- /dev/null +++ b/event-protocol/schemas/test-case-prepared.json @@ -0,0 +1,32 @@ +{ + "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-case-prepared.json#", + "title": "test-case-prepared", + "description": "Describes a test case that is about to be run", + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "sourceLocation": { + "$ref": "defs.json#/location" + }, + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "sourceLocation": { + "$ref": "defs.json#/location" + }, + "actionLocation": { + "$ref": "defs.json#/location" + } + } + }, + "required": [ "sourceLocation", "actionLocation" ], + "additionalProperties": false + } + }, + "required": [ "sourceLocation", "steps" ], + "additionalProperties": false +} diff --git a/event-protocol/schemas/test-case-started.json b/event-protocol/schemas/test-case-started.json new file mode 100644 index 0000000000..efe815b119 --- /dev/null +++ b/event-protocol/schemas/test-case-started.json @@ -0,0 +1,16 @@ +{ + "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-case-started.json#", + "title": "test-case-started", + "description": "Signals the start of executing a test case", + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "sourceLocation": { + "$ref": "defs.json#/location" + } + }, + "required": [ "sourceLocation" ], + "additionalProperties": false +} diff --git a/event-protocol/schemas/test-run-started.json b/event-protocol/schemas/test-run-started.json new file mode 100644 index 0000000000..6fa0e391ee --- /dev/null +++ b/event-protocol/schemas/test-run-started.json @@ -0,0 +1,18 @@ +{ + "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-run-started.json#", + "title": "test-run-started", + "description": "Signals the start of a test run", + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "workingDirectory": { + "type": "string" + }, + "timestamp": { + "type": "number" + } + }, + "additionalProperties": false +} diff --git a/event-protocol/schemas/test-step-finished.json b/event-protocol/schemas/test-step-finished.json new file mode 100644 index 0000000000..9450579026 --- /dev/null +++ b/event-protocol/schemas/test-step-finished.json @@ -0,0 +1,29 @@ +{ + "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-step-finished.json#", + "title": "test-step-finished", + "description": "A step of a test case has finished executing", + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "testCase": { + "type": "object", + "properties": { + "sourceLocation": { + "$ref": "defs.json#/location" + } + }, + "required": [ "sourceLocation" ], + "additionalProperties": false + }, + "index": { + "type": "number" + }, + "result": { + "$ref": "defs.json#/result" + } + }, + "required": [ "testCase", "index", "result" ], + "additionalProperties": false +} diff --git a/event-protocol/schemas/test-step-started.json b/event-protocol/schemas/test-step-started.json new file mode 100644 index 0000000000..847b2fccd7 --- /dev/null +++ b/event-protocol/schemas/test-step-started.json @@ -0,0 +1,26 @@ +{ + "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-step-started.json#", + "title": "test-step-started", + "description": "A step of a test case is about to be executed", + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "testCase": { + "type": "object", + "properties": { + "sourceLocation": { + "$ref": "defs.json#/location" + } + }, + "required": [ "sourceLocation" ], + "additionalProperties": false + }, + "index": { + "type": "number" + } + }, + "required": [ "testCase", "index" ], + "additionalProperties": false +}