🏗 [test-case-reporting] Implement PoC for e2e test reporter (#29768)

* Implement PoC for e2e test reporter

* Improve wording in comment & err message

* Lint

* Refactor for clarity, fix fullTitle typo

* Move reporters to e2e folder, allowing imports to break for now

* Fix broken imports

* Rename mocha ci reporter, breaking imports

* Make ci reporter that joins together the reporters

* Rename KarmaMimicReporter in files

* Rename karma-mimic-reporter file

* Rename private _karmaMimicReporter

* Simplify event insertion

* Only write json report if report flag set

build-system/tasks/e2e/index.js

@@ -17,8 +17,9 @@
 'use strict';
 
 const argv = require('minimist')(process.argv.slice(2));
-const ciReporter = require('../mocha-ci-reporter');
+const ciReporter = require('./mocha-ci-reporter');
 const config = require('../../test-configs/config');
+const dotsReporter = require('./mocha-dots-reporter');
 const glob = require('glob');
 const log = require('fancy-log');
 const Mocha = require('mocha');
@@ -52,11 +53,20 @@ async function cleanUp_() {
 }
 
 function createMocha_() {
+  let reporter;
+  if (argv.testnames || argv.watch) {
+    reporter = '';
+  } else if (argv.report) {
+    reporter = ciReporter;
+  } else {
+    reporter = dotsReporter;
+  }
+
   const mocha = new Mocha({
     // e2e tests have a different standard for when a test is too slow,
     // so we set a non-default threshold.
     slow: SLOW_TEST_THRESHOLD_MS,
-    reporter: argv.testnames || argv.watch ? '' : ciReporter,
+    reporter,
     retries: TEST_RETRIES,
     fullStackTrace: true,
   });
@@ -163,4 +173,5 @@ e2e.flags = {
     'Options are `puppeteer` or `selenium`. Default: `selenium`',
   'headless': '  Runs the browser in headless mode',
   'debug': '  Prints debugging information while running tests',
+  'report': '  Write test result report to a local file',
 };

build-system/tasks/e2e/mocha-ci-reporter.js

@@ -0,0 +1,30 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const JsonReporter = require('./mocha-custom-json-reporter');
+const mocha = require('mocha');
+const MochaDotsReporter = require('./mocha-dots-reporter');
+const {Base} = mocha.reporters;
+
+function ciReporter(runner, options) {
+  Base.call(this, runner, options);
+  this._mochaDotsReporter = new MochaDotsReporter(runner, options);
+  this._jsonReporter = new JsonReporter(runner, options);
+  return this;
+}
+ciReporter.prototype.__proto__ = Base.prototype;
+
+module.exports = ciReporter;

build-system/tasks/e2e/mocha-custom-json-reporter.js

@@ -0,0 +1,87 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+'use strict';
+
+const fs = require('fs').promises;
+const {
+  EVENT_TEST_PASS,
+  EVENT_TEST_FAIL,
+  EVENT_RUN_END,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END,
+} = require('mocha').Runner;
+const {Base} = require('mocha').reporters;
+const {inherits} = require('mocha').utils;
+
+async function writeOutput(output, filename) {
+  try {
+    await fs.writeFile(filename, JSON.stringify(output, null, 4));
+  } catch (error) {
+    process.stdout.write(
+      Base.color(
+        'fail',
+        `Could not write test result report to file '${filename}'`
+      )
+    );
+  }
+}
+
+/**
+ * Custom Mocha reporter for CI builds.
+ * Mimics the structured karma reporter, but for Mocha.
+ * @param {*} runner
+ */
+function JsonReporter(runner) {
+  Base.call(this, runner);
+  const testEvents = [];
+  let suiteList = [];
+
+  runner.on(EVENT_SUITE_BEGIN, function (suite) {
+    suiteList.push(suite.title);
+  });
+
+  runner.on(EVENT_SUITE_END, function () {
+    // We need a fresh copy every time we make a new suite,
+    // so we can't use pop here (or the suiteList info of previous
+    // tests would be changed)
+    suiteList = suiteList.slice(0, -1);
+  });
+
+  [EVENT_TEST_PASS, EVENT_TEST_FAIL, EVENT_TEST_PENDING].forEach((event) => {
+    runner.on(event, (test) => {
+      testEvents.push({test, suiteList, event});
+    });
+  });
+
+  runner.on(EVENT_RUN_END, async function () {
+    const testResults = testEvents.map(({test, suiteList, event}) => ({
+      description: test.title,
+      suite: suiteList,
+      success: event === EVENT_TEST_PASS,
+      skipped: event === EVENT_TEST_PENDING,
+      time: test.duration, // in milliseconds
+    }));
+
+    // Apparently we'll need to add a --no-exit flag when calling this
+    // to allow for the asynchronous reporter.
+    // See https://github.com/mochajs/mocha/issues/812
+    await writeOutput({testResults}, `result-reports/e2e.json`);
+  });
+}
+
+inherits(JsonReporter, Base);
+module.exports = JsonReporter;

build-system/tasks/mocha-ci-reporter.js → build-system/tasks/e2e/mocha-dots-reporter.js

@@ -17,15 +17,15 @@
 
 const {Base} = require('mocha').reporters;
 const {inherits} = require('mocha').utils;
-const {reportTestFinished} = require('./report-test-status');
-const {symbols} = require('./karma.conf').mochaReporter;
+const {reportTestFinished} = require('../report-test-status');
+const {symbols} = require('../karma.conf').mochaReporter;
 
 /**
  * Custom Mocha reporter for CI builds.
  * Mimics the style of the Karma reporter on Travis.
  * @param {*} runner
  */
-function ciReporter(runner) {
+function MochaDotsReporter(runner) {
   Base.call(this, runner);
   const self = this;
 
@@ -62,5 +62,5 @@ function ciReporter(runner) {
   }
 }
 
-inherits(ciReporter, Base);
-module.exports = ciReporter;
+inherits(MochaDotsReporter, Base);
+module.exports = MochaDotsReporter;