Merge pull request #337 from jpmorganchase/test-stability

Test stability

.travis.yml

@@ -31,5 +31,4 @@ install:
 script:
   - yarn lint
   - PSP_DOCKER=1 yarn build
-  - PSP_DOCKER=1 yarn test:build
-  - yarn test:quietrun
+  - yarn test --docker --quiet

docs/development.md

@@ -110,6 +110,12 @@ build the test suite for every package and run them.
 yarn test
 ```
 
+A Test name regex can be passed to `jest` via the same `-t` flag:
+
+```bash
+yarn test -t 'button test (A|B)'
+``` 
+
 The test suite is composed of two sections:  a Node.js test which asserts
 behavior of the `@jpmorganchase/perspective` library, and a suite of 
 [Puppeteer](https://developers.google.com/web/tools/puppeteer/) tests which 
@@ -131,7 +137,7 @@ screenshots reflect your change, you can update the new hashes manually in the
 environment variable:
 
 ```bash
-WRITE_TESTS=1 yarn test
+yarn test --write
 ```
 
 ## Benchmarking

package.json

@@ -41,19 +41,11 @@
         "bench:run": "lerna exec ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} -- npm run bench:run",
         "bench": "npm-run-all --silent bench:build bench:run",
         "docs": "lerna run docs --silent --stream",
-        "pretest:build": "lerna exec -- mkdir -p build",
-        "test:build": "bash -c '[[ -z \"${PSP_DOCKER}\" ]] && npm run --silent _build_test || npm run --silent _emsdk -- npm run --silent _build_test'",
-        "test:run": "npm run --silent clean:screenshots && npm run --silent _puppeteer -- node_modules/.bin/lerna exec --concurrency 1 --no-bail ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} -- yarn --silent test:run",
-        "test:quietrun:jest": "output=$(jest --color --silent 2>&1); ret=$?; echo \"${output}\"; exit $ret",
-        "test:quietrun": "yarn --silent _puppeteer yarn --silent test:quietrun:jest",
-        "test:quickrun":  "yarn --silent clean:screenshots && yarn --silent _puppeteer yarn --silent jest --color --silent 2>&1",
-        "quicktest": "npm-run-all --silent lint test:build test:quickrun",
-        "test": "npm-run-all --silent lint test:build test:run",
+        "test": "node scripts/test.js",
         "clean": "find obj -mindepth 1 -delete && lerna run clean ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream",
         "preclean:screenshots": "lerna exec -- mkdir -p screenshots",
         "clean:screenshots": "lerna run clean:screenshots ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}}",
         "start": "lerna run start --stream --scope",
-        "write_tests": "WRITE_TESTS=1 npm run test:run",
         "precommit": "npm run lint",
         "lint": "npm-run-all --silent lint:*",
         "lint:eslint": "eslint packages/*/src/**/*.js packages/*/test/**/*.js examples/*/*.js",
@@ -66,7 +58,6 @@
         "fix:json": "prettier --tab-width 4 --write **/package.json",
         "fix": "npm-run-all --silent fix:*",
         "_build": "lerna run build ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream",
-        "_build_test": "lerna run test:build ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream",
         "_emsdk": "docker run --rm -it ${PSP_CPU_COUNT:+--cpus=\"${PSP_CPU_COUNT}.0\"} -v $(pwd):/src -e PACKAGE=${PACKAGE} perspective/emsdk",
         "_puppeteer": "docker run -it --rm --shm-size=2g ${PSP_CPU_COUNT:+--cpus=\"${PSP_CPU_COUNT}.0\"} -u root -e WRITE_TESTS=${WRITE_TESTS} -e PACKAGE=${PACKAGE} -v $(pwd):/src -w /src perspective/puppeteer"
     },

packages/perspective-viewer-highcharts/test/js/axis_tests.js

@@ -11,7 +11,7 @@ exports.default = function() {
     describe("axis tests", () => {
         test.capture("sets a category X axis when pivoted by a datetime.", async page => {
             const viewer = await page.$("perspective-viewer");
-            await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+            await page.shadow_click("perspective-viewer", "#config_button");
             await page.evaluate(element => element.setAttribute("row-pivots", '["Order Date"]'), viewer);
             await page.waitForSelector("perspective-viewer:not([updating])");
             await page.evaluate(element => element.setAttribute("columns", '["State","Sales"]'), viewer);
@@ -22,7 +22,7 @@ exports.default = function() {
 
         test.capture("sets a category axis when the axis type is a string.", async page => {
             const viewer = await page.$("perspective-viewer");
-            await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+            await page.shadow_click("perspective-viewer", "#config_button");
             await page.evaluate(element => element.setAttribute("columns", '["State","Sales"]'), viewer);
             await page.waitForSelector("perspective-viewer:not([updating])");
         });

packages/perspective-viewer-highcharts/test/js/bar.spec.js

@@ -25,13 +25,9 @@ utils.with_server({}, () => {
                     await utils.invoke_tooltip(bar, page);
                 });
 
-                test.capture("tooltip shows column label.", async page => {
-                    await utils.invoke_tooltip(bar, page);
-                });
-
                 test.capture("tooltip shows proper column labels based on hover target.", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     // set a new column
                     await page.evaluate(element => element.setAttribute("columns", '["Sales", "Profit"]'), viewer);
                     await page.waitForSelector("perspective-viewer:not([updating])");
@@ -40,7 +36,7 @@ utils.with_server({}, () => {
 
                 test.capture("tooltip shows proper column labels based on hover target, pt2.", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     // set a new column
                     await page.evaluate(element => element.setAttribute("columns", '["Sales", "Profit"]'), viewer);
                     await page.waitForSelector("perspective-viewer:not([updating])");
@@ -49,7 +45,7 @@ utils.with_server({}, () => {
 
                 test.capture("tooltip shows pivot labels.", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     // set a row pivot and a column pivot
                     await page.evaluate(element => element.setAttribute("row-pivots", '["State"]'), viewer);
                     await page.waitForSelector("perspective-viewer:not([updating])");
@@ -73,7 +69,7 @@ utils.with_server({}, () => {
                 "render warning should show above size limit.",
                 async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await page.evaluate(element => {
                         window.getPlugin("y_bar").max_size = 50;
                         element.setAttribute("columns", '["col_b"]');
@@ -96,7 +92,7 @@ utils.with_server({}, () => {
                 "dismissing render warning should trigger render.",
                 async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await page.evaluate(element => {
                         window.getPlugin("y_bar").max_size = 50;
                         element.setAttribute("columns", '["col_b"]');
@@ -121,7 +117,7 @@ utils.with_server({}, () => {
                 "selecting 'do not show again' should stop render warnings.",
                 async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await page.evaluate(element => {
                         window.getPlugin("y_bar").max_size = 50;
                         element.setAttribute("columns", '["col_a", "col_b"]');
@@ -153,7 +149,7 @@ utils.with_server({}, () => {
                     });
 
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await page.evaluate(element => {
                         element.setAttribute("columns", '["col_a", "col_b"]');
                     }, viewer);

packages/perspective-viewer-highcharts/test/js/line.spec.js

@@ -26,20 +26,13 @@ utils.with_server({}, () => {
                 const series = "path.highcharts-graph";
 
                 test.capture("tooltip shows on hover.", async page => {
-                    const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
-                    await utils.invoke_tooltip(series, page);
-                });
-
-                test.capture("tooltip shows proper column labels.", async page => {
-                    const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await utils.invoke_tooltip(series, page);
                 });
 
                 test.capture("tooltip shows pivot labels.", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
 
                     // set a row pivot and a column pivot
                     await page.evaluate(element => element.setAttribute("row-pivots", '["State"]'), viewer);

packages/perspective-viewer-highcharts/test/js/scatter.spec.js

@@ -26,20 +26,13 @@ utils.with_server({}, () => {
                 const point = "path.highcharts-point";
 
                 test.capture("tooltip shows on hover.", async page => {
-                    const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
-                    await utils.invoke_tooltip(point, page);
-                });
-
-                test.capture("tooltip shows proper column labels.", async page => {
-                    const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await utils.invoke_tooltip(point, page);
                 });
 
                 test.capture("tooltip shows pivot labels.", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
 
                     // set a row pivot and a column pivot
                     await page.evaluate(element => element.setAttribute("row-pivots", '["State"]'), viewer);

packages/perspective-viewer-highcharts/test/js/treemap.spec.js

@@ -21,7 +21,7 @@ utils.with_server({}, () => {
             describe("tooltip tests", () => {
                 test.capture("tooltip shows on hover.", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
 
                     // set a row pivot and a column pivot so the graph will render
                     await page.evaluate(element => element.setAttribute("row-pivots", '["State"]'), viewer);

packages/perspective-viewer-hypergrid/test/js/regressions.spec.js

@@ -31,7 +31,7 @@ utils.with_server({}, () => {
         () => {
             test.capture("empty grids do not explode", async page => {
                 const viewer = await page.$("perspective-viewer");
-                await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                await page.shadow_click("perspective-viewer", "#config_button");
                 await page.waitFor("perspective-viewer:not([updating])");
                 await capture_update(page, viewer, () => page.evaluate(element => element.update([{x: 3}]), viewer));
                 await page.waitFor("perspective-viewer:not([updating])");
@@ -46,14 +46,14 @@ utils.with_server({}, () => {
             describe("Updates", () => {
                 test.capture("regular updates", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await capture_update(page, viewer, () => page.evaluate(element => element.update([{x: 3, y: "Updated!"}]), viewer));
                     await page.waitFor("perspective-viewer:not([updating])");
                 });
 
                 test.capture("saving a computed column does not interrupt update rendering", async page => {
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer);
                     await page.evaluate(element => {
                         let com = element.shadowRoot.querySelector("perspective-computed-column");

packages/perspective-viewer-hypergrid/test/js/superstore.spec.js

@@ -49,7 +49,7 @@ utils.with_server({}, () => {
                 test.capture("resets viewable area when the logical size expands.", async page => {
                     await set_lazy(page);
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await page.evaluate(element => element.setAttribute("column-pivots", '["Category"]'), viewer);
                     await page.waitForSelector("perspective-viewer:not([updating])");
                     await page.evaluate(element => element.setAttribute("row-pivots", '["City"]'), viewer);
@@ -58,11 +58,11 @@ utils.with_server({}, () => {
                 test.capture("resets viewable area when the physical size expands.", async page => {
                     await set_lazy(page);
                     const viewer = await page.$("perspective-viewer");
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                     await page.evaluate(element => element.setAttribute("row-pivots", '["Category"]'), viewer);
                     await page.waitForSelector("perspective-viewer:not([updating])");
                     await page.evaluate(element => element.setAttribute("row-pivots", "[]"), viewer);
-                    await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer);
+                    await page.shadow_click("perspective-viewer", "#config_button");
                 });
             });
         },