event-protocol: Add events & schemas for test runs

event-protocol/README.md

@@ -85,6 +85,74 @@ Example (Java stack trace):
 [snippet](examples/events/004_attachment-stacktrace.json)
 ```
 
+### test-run-started {#test-run-started}
+
+Signals the start of a test run. Contains details of the context of the run, like the working directory, start time etc.
+
+Example:
+
+```json
+[snippet](examples/events/005_test-run-started.json)
+```
+
+### test-case-prepared {#test-case-prepared}
+
+Contains the details of a test case that's ready to be executed. The `sourceLocation` should match the `uri` of preceeding `source` and `gherkin-document` events.
+
+Each `step` has an `actionLocation` pointing to the source of the glue code that will be invoked when the step is executed. The `sourceLocation` points to the source of the step in a Gherkin document.
+
+Example:
+
+```json
+[snippet](examples/events/006_test-case-prepared.json)
+```
+
+### test-case-started
+
+Signals the start of executing a test case
+
+Example:
+
+```json
+[snippet](examples/events/007_test-case-started.json)
+```
+
+### test-step-started
+
+Signals the start of executing a test step within a test case. The `index` locates the step within the array of steps sent in the preceeding `test-case-prepared` event.
+
+Example:
+
+```json
+[snippet](examples/events/008_test-step-started.json)
+```
+
+### test-step-finished
+
+Signals the end of executing a test step. The result
+
+Example of a passing step:
+
+```json
+[snippet](examples/events/009_test-step-finished.json)
+```
+
+Example of failing step:
+
+```json
+[snippet](examples/events/011_test-step-finished.json)
+```
+
+Note that undefined is just a special case of failure. The `result` will always have an `exception` when the step has not passed.
+
+### test-case-finished
+
+Signals the end of executing a whole test case (scenario or example row).
+
+```json
+[snippet](examples/events/012_test-case-finished.json)
+```
+
 ## Implementation
 
 * Cucumber events are formatted as [Newline Delimited JSON](http://ndjson.org)
@@ -128,7 +196,9 @@ We'll manage this by adding a version to a specific event, where needed. For now
 There are a few constraints about the order of events:
 
 * A [source](#event-source) event must be received before any
-  [attachment](#event-attachment) events referring to it
+  [attachment](#event-attachment), `test-case-*` or `test-step-*` events referring to it
+* [test-case-started](#test-case-started), [test-step-started](#test-step-started), [test-step-finished](#test-step-finished), [test-case-finished](#test-case-finished) events are expected to be received in that order. They should be preceeded by the relevant [test-case-prepared](#test-case-prepared) event describing the test case and its steps.
+* `test-*-started` events are optional
 
 ## Event validation {#validation}
 

event-protocol/examples/events.ndjson

@@ -3,3 +3,11 @@
 {"pickle":{"language":"en","locations":[{"column":3,"line":2}],"name":"World","steps":[{"arguments":[],"locations":[{"column":11,"line":3}],"text":"a step"}],"tags":[]},"type":"pickle","uri":"features/hello.feature"}
 {"type":"attachment","source":{"uri":"features/hello.feature","start":{"line":3,"column":7}},"data":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAELUExURUdwTACoGAClFACqHACoFwCnGQCnFQCnGACoGQCoGQCoGACuGgCnFwCiFwCoGACoGACnFwClFgCpGACoFwD/AACnGAC2JACoGACnFwCqKgCoGACoGACqGQCoFwCnFwCoFwB/AACnGACoGAC/AACoGACoFgCmFwCnFwCnFwCqFgCmFgCZAACoGC64Ql3IbA2sJBuxMBiwLhewLR2yMgmrIOn36/z+/BGuJ9/04rHkuIDUjGHJcA6tJeD04wGoGQusIpnco+T257PlugSpHN704XDOfQeqHiCzNc7u0+z57t3z4HvSh7fmvj69UJzdpSi2PNjy3AKpGlrHahqxMBCtJpHZm83u0jS6R0vCXEWczDEAAAAsdFJOUwDIJQlYPSP7cFCqE1cW+mf4ImiBAXQHPu8GfP0zzoyOAp2wBOg4a2PqOU4FxgPjdgAAAK9JREFUGNNNj1UCwkAMRBcotLi7uw1S3N3d4f4nYSvIfO28bCYJIaI0OSabJ1+pFaDKpD4+zUJS1CHVWSwer+esDNhEQP+fV/f5uAd4XEIesNldl5WK0KSlQAtcDkc5xUcBA+y3tL/UKQIhCpLA+sTzw1K7UQVHQTyB23QyKraaNSAgTOGAQb9bqBeAoDg2EpMT4XdLm3rD8qrO7zFWO7UW8+86oodJp/zzRGUwyq83Mjcb8VXl0ZMAAAAASUVORK5CYII=","media":{"encoding":"base64","type":"image/png"}}
 {"type":"attachment","source":{"uri":"features/hello.feature","start":{"line":3,"column":7}},"data":"Exception in thread \"main\" java.lang.NullPointerException\n        at com.example.myproject.Book.getTitle(Book.java:16)\n        at com.example.myproject.Author.getBookTitles(Author.java:25)\n        at com.example.myproject.Bootstrap.main(Bootstrap.java:14)\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.stacktrace.java+plain"}}
+{"type":"test-run-started","workingDirectory":"/Users/matt/projects/cucumber-ruby","timestamp":1489683458}
+{"type":"test-case-prepared","sourceLocation":{"uri":"features/passing.feature","line":2},"steps":[{"actionLocation":{"uri":"/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb","line":28},"sourceLocation":{"uri":"/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb","line":28}},{"actionLocation":{"uri":"features/passing.feature","line":3},"sourceLocation":{"uri":"features/passing.feature","line":3}}]}
+{"type":"test-case-started","sourceLocation":{"uri":"features/passing.feature","line":2}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":0}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":0,"result":{"status":"passed","duration":3000}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":1}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":1,"result":{"status":"undefined","exception":{"message":"Undefined step: \"this step passes\"","type":"Cucumber::Core::Test::Result::Undefined","stackTrace":["features/passing.feature:3:in `Given this step passes'"]}}}
+{"type":"test-case-finished","sourceLocation":{"uri":"features/passing.feature","line":2},"result":{"status":"undefined","duration":13163000,"exception":{"message":"Undefined step: \"this step passes\"","type":"Cucumber::Core::Test::Result::Undefined","stackTrace":["features/passing.feature:3:in `Given this step passes'"]}}}

event-protocol/examples/events/005_test-run-started.json

@@ -0,0 +1,5 @@
+{
+  "type":"test-run-started",
+  "workingDirectory":"/Users/matt/projects/cucumber-ruby",
+  "timestamp":1489683458
+}

event-protocol/examples/events/006_test-case-prepared.json

@@ -0,0 +1,29 @@
+{
+  "type": "test-case-prepared",
+  "sourceLocation": {
+    "uri": "features/passing.feature",
+    "line": 2
+  },
+  "steps": [
+    {
+      "actionLocation": {
+        "uri": "/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb",
+        "line": 28
+      },
+      "sourceLocation": {
+        "uri": "/Users/matt/projects/cucumber-ruby/lib/cucumber/filters/prepare_world.rb",
+        "line": 28
+      }
+    },
+    {
+      "actionLocation": {
+        "uri": "features/passing.feature",
+        "line": 3
+      },
+      "sourceLocation": {
+        "uri": "features/passing.feature",
+        "line": 3
+      }
+    }
+  ]
+}

event-protocol/examples/events/007_test-case-started.json

@@ -0,0 +1,7 @@
+{
+  "type": "test-case-started",
+  "sourceLocation": {
+    "uri": "features/passing.feature",
+    "line": 2
+  }
+}

event-protocol/examples/events/008_test-step-started.json

@@ -0,0 +1,10 @@
+{
+  "type": "test-step-started",
+  "testCase": {
+    "sourceLocation": {
+      "uri": "features/passing.feature",
+      "line": 2
+    }
+  },
+  "index": 0
+}

event-protocol/examples/events/009_test-step-finished.json

@@ -0,0 +1,14 @@
+{
+  "type": "test-step-finished",
+  "testCase": {
+    "sourceLocation": {
+      "uri": "features/passing.feature",
+      "line": 2
+    }
+  },
+  "index": 0,
+  "result": {
+    "status": "passed",
+    "duration": 3000
+  }
+}

event-protocol/examples/events/010_test-step-started.json

@@ -0,0 +1,10 @@
+{
+  "type": "test-step-started",
+  "testCase": {
+    "sourceLocation": {
+      "uri": "features/passing.feature",
+      "line": 2
+    }
+  },
+  "index": 1
+}

event-protocol/examples/events/011_test-step-finished.json

@@ -0,0 +1,20 @@
+{
+  "type": "test-step-finished",
+  "testCase": {
+    "sourceLocation": {
+      "uri": "features/passing.feature",
+      "line": 2
+    }
+  },
+  "index": 1,
+  "result": {
+    "status": "undefined",
+    "exception": {
+      "message": "Undefined step: \"this step passes\"",
+      "type": "Cucumber::Core::Test::Result::Undefined",
+      "stackTrace": [
+        "features/passing.feature:3:in `Given this step passes'"
+      ]
+    }
+  }
+}

event-protocol/examples/events/012_test-case-finished.json

@@ -0,0 +1,18 @@
+{
+  "type": "test-case-finished",
+  "sourceLocation": {
+    "uri": "features/passing.feature",
+    "line": 2
+  },
+  "result": {
+    "status": "undefined",
+    "duration": 13163000,
+    "exception": {
+      "message": "Undefined step: \"this step passes\"",
+      "type": "Cucumber::Core::Test::Result::Undefined",
+      "stackTrace": [
+        "features/passing.feature:3:in `Given this step passes'"
+      ]
+    }
+  }
+}

event-protocol/schemas/attachment.json

@@ -13,7 +13,7 @@
         "uri": {
           "type": "string"
         },
-        "start": { "$ref": "defs.json#/location" }
+        "start": { "$ref": "defs.json#/ast-location" }
       },
       "required": [
         "uri",

event-protocol/schemas/defs.json

@@ -3,6 +3,23 @@
   "title": "definitions",
   "description": "Re-usable parts of the schema, referenced by others",
   "location": {
+    "type": "object",
+    "properties": {
+      "line": {
+        "type": "integer",
+        "minimum": 1
+      },
+      "uri": {
+        "type": "string"
+      }
+    },
+    "required": [
+      "line",
+      "uri"
+    ],
+    "additionalProperties": false
+  },
+  "gherkin-node-location": {
     "type": "object",
     "properties": {
       "line": {
@@ -12,9 +29,6 @@
       "column": {
         "type": "integer",
         "minimum": 0
-      },
-      "uri": {
-        "type": "string"
       }
     },
     "required": [
@@ -73,5 +87,44 @@
       "location"
     ],
     "additionalProperties": false
+  },
+  "result": {
+    "type": "object",
+    "properties": {
+      "status": {
+        "enum": [
+          "passed",
+          "failed",
+          "pending",
+          "skipped",
+          "undefined"
+        ]
+      },
+      "duration": {
+        "type": "integer"
+      },
+      "exception": { "$ref": "defs.json#/exception" }
+    },
+    "required": [ "status" ],
+    "additionalProperties": false
+  },
+  "exception": {
+    "type": "object",
+    "properties": {
+      "message": {
+        "type": "string"
+      },
+      "stackTrace": {
+        "type": "array",
+        "items": {
+          "type": "string"
+        }
+      },
+      "type": {
+        "type": "string"
+      }
+    },
+    "required": [ "message", "stackTrace" ],
+    "additionalProperties": false
   }
 }

event-protocol/schemas/pickle.json

@@ -26,7 +26,7 @@
         "locations": {
           "type": "array",
           "minLength": 1,
-          "items": { "$ref": "defs.json#/location" }
+          "items": { "$ref": "defs.json#/gherkin-node-location" }
         },
         "steps": {
           "type": "array",
@@ -43,7 +43,7 @@
               "locations": {
                 "type": "array",
                 "minLength": 1,
-                "items": { "$ref": "defs.json#/location" }
+                "items": { "$ref": "defs.json#/gherkin-node-location" }
               }
             },
             "required": [

event-protocol/schemas/test-case-finished.json

@@ -0,0 +1,19 @@
+{
+  "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-case-finished.json#",
+  "title": "test-case-finished",
+  "description": "A test case has finished executing",
+  "type": "object",
+  "properties": {
+    "type": {
+      "type": "string"
+    },
+    "sourceLocation": { 
+      "$ref": "defs.json#/location"
+    },
+    "result": {
+      "$ref": "defs.json#/result"
+    }
+  },
+  "required": [ "sourceLocation", "result" ],
+  "additionalProperties": false
+}

event-protocol/schemas/test-case-prepared.json

@@ -0,0 +1,32 @@
+{
+  "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-case-prepared.json#",
+  "title": "test-case-prepared",
+  "description": "Describes a test case that is about to be run",
+  "type": "object",
+  "properties": {
+    "type": {
+      "type": "string"
+    },
+    "sourceLocation": { 
+      "$ref": "defs.json#/location"
+    },
+    "steps": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "sourceLocation": { 
+            "$ref": "defs.json#/location"
+          },
+          "actionLocation": { 
+            "$ref": "defs.json#/location"
+          }
+        }
+      },
+      "required": [ "sourceLocation", "actionLocation" ],
+      "additionalProperties": false
+    }
+  },
+  "required": [ "sourceLocation", "steps" ],
+  "additionalProperties": false
+}

event-protocol/schemas/test-case-started.json

@@ -0,0 +1,16 @@
+{
+  "id": "https://raw.github.com/cucumber/cucumber/master/event-protocol/schemas/test-case-started.json#",
+  "title": "test-case-started",
+  "description": "Signals the start of executing a test case",
+  "type": "object",
+  "properties": {
+    "type": {
+      "type": "string"
+    },
+    "sourceLocation": { 
+      "$ref": "defs.json#/location"
+    }
+  },
+  "required": [ "sourceLocation" ],
+  "additionalProperties": false
+}