Merge pull request #870 from finos/optional-webpack-plugin

Optional `@finos/perspective-webpack-plugin`

.gitignore

@@ -159,4 +159,5 @@ python/perspective/perspective/node/assets/*
 docs/static/js/logo.js
 docs/static/js/logo.js.map
 python/perspective/pip-wheel-metadata
-results.darwin.json
+darwin.json
+darwin.interactive.json

package.json

@@ -17,6 +17,10 @@
         "docs",
         "python/perspective"
     ],
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/finos/perspective"
+    },
     "devDependencies": {
         "@babel/cli": "^7.2.3",
         "@babel/core": "^7.1.0",

packages/perspective-jupyterlab/src/config/plugin.config.js

@@ -8,15 +8,13 @@
  */
 
 const path = require("path");
-const PerspectivePlugin = require("@finos/perspective-webpack-plugin");
 const webpack = require("webpack");
 
 module.exports = {
     mode: process.env.PSP_NO_MINIFY || process.env.PSP_DEBUG ? "development" : process.env.NODE_ENV || "production",
     entry: {
         index: "./src/ts/index.ts"
     },
-    devtool: "cheap-eval-source-map",
     resolveLoader: {
         alias: {
             "file-worker-loader": "@finos/perspective-webpack-plugin/src/js/psp-worker-loader.js"
@@ -32,7 +30,7 @@ module.exports = {
     },
     externals: /\@jupyterlab|\@phosphor|\@jupyter-widgets/,
     stats: {modules: false, hash: false, version: false, builtAt: false, entrypoints: false},
-    plugins: [new webpack.ContextReplacementPlugin(/moment[\/\\]locale$/, /(en|es|fr)$/), new PerspectivePlugin()],
+    plugins: [new webpack.ContextReplacementPlugin(/moment[\/\\]locale$/, /(en|es|fr)$/)],
     module: {
         rules: [
             {

packages/perspective-jupyterlab/src/ts/widget.ts

@@ -12,17 +12,6 @@ import {DOMWidgetView} from "@jupyter-widgets/base";
 import {PerspectiveViewerOptions} from "@finos/perspective-viewer";
 import {PerspectiveWidget, PerspectiveWidgetOptions} from "@finos/perspective-phosphor";
 
-import perspective from "@finos/perspective";
-
-import * as wasm from "@finos/perspective/dist/umd/psp.async.wasm";
-import * as worker from "!!file-worker-loader?inline=true!@finos/perspective/dist/umd/perspective.wasm.worker.js";
-
-if (perspective) {
-    perspective.override({wasm, worker});
-} else {
-    console.warn("Perspective was undefined in jlab - wasm load errors may occur");
-}
-
 export type PerspectiveJupyterWidgetOptions = {
     view: DOMWidgetView;
 };

packages/perspective-phosphor/package.json

@@ -23,6 +23,7 @@
         "build:styles": "webpack --color --config config/webpack.theme.config.js",
         "build:umd": "webpack --color --config config/webpack.umd.config.js",
         "clean": "rimraf dist",
+        "clean:screenshots": "rimraf \"screenshots/**/*.@(failed|diff).png\"",
         "test": "yarn test:run",
         "test:run": "npm-run-all test:unit test:integration",
         "test:unit": "jest  --color --silent --config=./config/jest.unit.config.js 2>&1",

packages/perspective-test/src/js/index.js

@@ -21,9 +21,7 @@ const cp = require("child_process");
 
 const {WebSocketServer} = require("@finos/perspective");
 
-const IS_LOCAL_PUPPETEER = fs.existsSync(path.join(__dirname, "..", "..", "..", "..", "node_modules", "puppeteer"));
-const LOCAL_RESULTS_FILENAME = `results.${process.platform}.json`;
-const RESULTS_FILENAME = IS_LOCAL_PUPPETEER ? LOCAL_RESULTS_FILENAME : "results.json";
+const {IS_LOCAL_PUPPETEER, RESULTS_TAGNAME, RESULTS_FILENAME} = require("./paths.js");
 
 let __PORT__;
 
@@ -127,7 +125,7 @@ beforeAll(async done => {
 
         if (results.__GIT_COMMIT__) {
             const hash = execSync(`git cat-file -e ${results.__GIT_COMMIT__}`);
-            if (!hash || hash.length == 0) {
+            if (!hash || hash.toString().length != 0) {
                 private_console.error(`-- WARNING - Test results generated from non-existent commit ${results.__GIT_COMMIT__}.`);
             }
         }
@@ -181,7 +179,7 @@ function mkdirSyncRec(targetDir) {
 describe.page = (url, body, {reload_page = true, name, root} = {}) => {
     let _url = url ? url : page_url;
     test_root = root ? root : test_root;
-    const dir_name = path.join(test_root, "screenshots", _url.replace(".html", ""));
+    const dir_name = path.join(test_root, "screenshots", RESULTS_TAGNAME, _url.replace(".html", ""));
     if (!fs.existsSync(dir_name)) {
         mkdirSyncRec(dir_name);
     }
@@ -196,7 +194,7 @@ describe.page = (url, body, {reload_page = true, name, root} = {}) => {
         return result;
     });
 
-    if (IS_LOCAL_PUPPETEER && !fs.existsSync(path.join(test_root, "test", "results", LOCAL_RESULTS_FILENAME)) && !process.env.WRITE_TESTS) {
+    if (IS_LOCAL_PUPPETEER && !fs.existsSync(path.join(test_root, "test", "results", RESULTS_FILENAME)) && !process.env.WRITE_TESTS) {
         throw new Error(`
         
 ERROR: Running in puppeteer tests without "${RESULTS_FILENAME}"
@@ -315,7 +313,7 @@ test.capture = function capture(name, body, {timeout = 60000, viewport = null, w
                     .update(screenshot)
                     .digest("hex");
 
-                const filename = path.join(test_root, "screenshots", `${_url.replace(".html", "")}`, `${name.replace(/ /g, "_").replace(/[\.']/g, "")}`);
+                const filename = path.join(test_root, "screenshots", RESULTS_TAGNAME, `${_url.replace(".html", "")}`, `${name.replace(/ /g, "_").replace(/[\.']/g, "")}`);
 
                 if (hash === results[_url + "/" + name]) {
                     fs.writeFileSync(filename + ".png", screenshot);

packages/perspective-test/src/js/paths.js

@@ -0,0 +1,15 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+module.exports.IS_LOCAL_PUPPETEER = fs.existsSync(path.join(__dirname, "..", "..", "..", "..", "node_modules", "puppeteer"));
+module.exports.RESULTS_TAGNAME = `${process.platform}${module.exports.IS_LOCAL_PUPPETEER ? (process.env.PSP_PAUSE_ON_FAILURE ? ".interactive" : "") : ".docker"}`;
+module.exports.RESULTS_FILENAME = `${module.exports.RESULTS_TAGNAME}.json`;

packages/perspective-test/src/js/reporter.js

@@ -8,6 +8,7 @@
  */
 
 const cp = require("child_process");
+const paths = require("./paths.js");
 const fs = require("fs");
 const termimg = require("term-img");
 
@@ -48,7 +49,7 @@ module.exports = class ImageViewerReporter {
                 const ancestors = test.ancestorTitles.filter(x => x.indexOf(".html") > -1).map(x => x.replace(".html", "").replace(/ /g, "_"));
                 const desc = ancestors.join("/");
                 const name = test.title.replace(/ /g, "_").replace(/[\.']/g, "");
-                const filename = `${testRunConfig.path.split("/test")[0]}/screenshots/${desc}/${name}.diff.png`;
+                const filename = `${testRunConfig.path.split("/test")[0]}/screenshots/${paths.RESULTS_TAGNAME}/${desc}/${name}.diff.png`;
                 const alt_filename = `screenshots/${desc}/${name}.diff.png`;
                 if (filename) {
                     this.write_img(test.title, ancestors, filename);

packages/perspective-webpack-plugin/index.js

@@ -30,8 +30,35 @@ class PerspectiveWebpackPlugin {
     }
 
     apply(compiler) {
+        const compilerOptions = compiler.options;
+        const moduleOptions = compilerOptions.module || (compilerOptions.module = {});
+
         const rules = [];
 
+        if (!this.options.inline) {
+            rules.push({
+                test: /perspective\.inline\.js/,
+                include: this.options.load_path,
+                use: [
+                    {
+                        loader: require.resolve("./src/js/switch-inline-loader.js")
+                    }
+                ]
+            });
+
+            if (compilerOptions.target !== "node") {
+                rules.push({
+                    test: /__node\.js$/,
+                    include: this.options.load_path,
+                    use: [
+                        {
+                            loader: require.resolve("./src/js/null-loader.js")
+                        }
+                    ]
+                });
+            }
+        }
+
         if (this.options.build_worker) {
             rules.push({
                 test: /perspective\.wasm\.js$/,
@@ -89,8 +116,6 @@ class PerspectiveWebpackPlugin {
             }
         });
 
-        const compilerOptions = compiler.options;
-        const moduleOptions = compilerOptions.module || (compilerOptions.module = {});
         moduleOptions.rules = (moduleOptions.rules || []).concat(rules);
     }
 }

packages/perspective-webpack-plugin/src/js/null-loader.js

@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+exports.default = function pitch(request) {
+    return request;
+};
+
+exports.pitch = function() {
+    return `module.exports.default = function() {}`;
+};
+
+exports.raw = true;

packages/perspective-webpack-plugin/src/js/psp-worker-loader.js

@@ -44,20 +44,19 @@ exports.default = function loader(content) {
 
     if (!options.compiled) {
         var inputPath = this.resourcePath;
-        if (!options.inline) {
-            inputPath = inputPath
-                .replace(path.join("perspective", "dist", "esm"), path.join("perspective", "dist", "umd"))
-                .replace(path.join("perspective", "dist", "cjs"), path.join("perspective", "dist", "umd"))
-                .replace(/\.js/, ".worker.js")
-                .replace(path.join("dist", "esm"), path.join("dist", "umd"));
-        }
+        inputPath = inputPath
+            .replace(path.join("perspective", "dist", "esm"), path.join("perspective", "dist", "umd"))
+            .replace(path.join("perspective", "dist", "cjs"), path.join("perspective", "dist", "umd"))
+            .replace(/\.js/, ".worker.js")
+            .replace(path.join("dist", "esm"), path.join("dist", "umd"));
         content = fs.readFileSync(inputPath).toString();
         if (!options.inline) {
-            this.emitFile(emitPath, "" + content);
             const map_file = `${inputPath}.map`;
             if (fs.existsSync(map_file)) {
                 const map_content = fs.readFileSync(map_file).toString();
-                this.emitFile(`${emitPath}.map`, "" + map_content);
+                this.emitFile(emitPath, "" + content, map_content);
+            } else {
+                this.emitFile(emitPath, "" + content);
             }
         }
     }

packages/perspective-webpack-plugin/src/js/switch-inline-loader.js

@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+exports.default = function pitch(request) {
+    return request;
+};
+
+exports.pitch = function(request) {
+    return `module.exports = require("${request.replace("umd/perspective.inline", "esm/perspective.parallel")}");`;
+};
+
+exports.raw = true;

packages/perspective-webpack-plugin/src/js/wasm-loader.js

@@ -20,11 +20,18 @@ var schema = {
         regExp: {},
         context: {
             type: "string"
+        },
+        inline: {
+            type: "boolean"
         }
     },
     additionalProperties: true
 };
 
+function ab2str(buf) {
+    return buf.toString("base64");
+}
+
 exports.default = function loader() {};
 
 exports.pitch = function pitch(request) {
@@ -39,14 +46,28 @@ exports.pitch = function pitch(request) {
         regExp: options.regExp
     });
 
-    var outputPath = JSON.stringify(emitPath);
-    this.emitFile(emitPath, content);
+    if (options.inline) {
+        return `
+module.exports = (
+    function (base64Data) {
+        var isBrowser = typeof window !== 'undefined' && typeof window.atob === 'function';
+        var binary = isBrowser ? window.atob(base64Data) : Buffer.from(base64Data, 'base64').toString('binary');
+        var bytes = new Uint8Array(binary.length);         
+        for (var i = 0; i < binary.length; ++i) {
+            bytes[i] = binary.charCodeAt(i);
+        }
+        return bytes.buffer
+    }
+)("${ab2str(content)}")`;
+    } else {
+        var outputPath = JSON.stringify(emitPath);
+        this.emitFile(emitPath, content);
 
-    const utils_path = JSON.stringify(`!!${path.join(__dirname, "utils.js")}`);
-    return `
-    var utils = require(${utils_path});
-    module.exports = utils.publicPath(__webpack_public_path__) + ${outputPath};
-    `;
+        const utils_path = JSON.stringify(`!!${path.join(__dirname, "utils.js")}`);
+        return `
+var utils = require(${utils_path});
+module.exports = utils.publicPath(__webpack_public_path__) + ${outputPath};`;
+    }
 };
 
 exports.raw = true;

packages/perspective/package.json

@@ -4,8 +4,8 @@
     "description": "Perspective.js",
     "main": "dist/umd/perspective.node.js",
     "unpkg": "dist/umd/perspective.js",
-    "browser": "dist/esm/perspective.parallel.js",
-    "module": "dist/esm/perspective.parallel.js",
+    "browser": "dist/umd/perspective.inline.js",
+    "module": "dist/umd/perspective.inline.js",
     "publishConfig": {
         "access": "public"
     },
@@ -17,11 +17,11 @@
     "typings": "index.d.ts",
     "scripts": {
         "prebuild": "mkdirp dist && mkdirp obj",
-        "cpp": "npm-run-all build:cpp test:cpp",
-        "build": "npm-run-all build:babel:* build:webpack",
+        "build": "npm-run-all build:babel:* build:webpack build:webpack:umd:inline",
         "build:babel:js": "babel src/js --source-maps --out-dir dist/esm",
         "build:babel:emsdk": "babel obj --source-maps --out-dir dist/esm",
-        "build:webpack": "npm-run-all -p build:webpack:*",
+        "build:webpack": "npm-run-all -p build:webpack:* ",
+        "build:webpack:umd:inline": "webpack --color --config src/config/perspective.inline.config.js",
         "build:webpack:umd": "webpack --color --config src/config/perspective.config.js",
         "build:webpack:node": "webpack --color --config src/config/perspective.node.config.js",
         "docs": "npm-run-all docs:jsdoc docs:deploy",

packages/perspective/src/config/common.config.js

@@ -4,8 +4,8 @@ const PerspectivePlugin = require("@finos/perspective-webpack-plugin");
 const TerserPlugin = require("terser-webpack-plugin");
 const plugins = [new webpack.ContextReplacementPlugin(/moment[\/\\]locale$/, /(en|es|fr)$/)];
 
-function common({build_worker, no_minify} = {}) {
-    plugins.push(new PerspectivePlugin({build_worker: build_worker}));
+function common({build_worker, no_minify, inline} = {}) {
+    plugins.push(new PerspectivePlugin({build_worker: build_worker, workerLoaderOptions: {inline, name: "[name].worker.js"}, wasmLoaderOptions: {inline, name: "[name]"}}));
     return {
         mode: process.env.PSP_NO_MINIFY || process.env.PSP_DEBUG || no_minify ? "development" : process.env.NODE_ENV || "production",
         plugins: plugins,

packages/perspective/src/config/minimizer.js

@@ -0,0 +1,24 @@
+const TerserPlugin = require("terser-webpack-plugin");
+
+module.exports.minimizer = [
+    new TerserPlugin({
+        cache: true,
+        parallel: true,
+        test: /\.js(\?.*)?$/i,
+        exclude: /wasm/,
+        sourceMap: true,
+        terserOptions: {
+            keep_infinity: true
+        }
+    }),
+    new TerserPlugin({
+        cache: true,
+        parallel: true,
+        test: /wasm/,
+        sourceMap: true,
+        terserOptions: {
+            mangle: false,
+            keep_infinity: true
+        }
+    })
+];