Update source-map worker to use source-map 0.7.x (#7071)

* Remove unused wrapper function in Webpack config generation.
* Update yarn.lock for recent package.json changes.
* Add a general concept of assets to the source-map worker.
* Upgrade the source-map module for WASM performance.
* Centralize sourcemap consumer construction.

.travis.yml

@@ -13,7 +13,7 @@ env:
   global:
     - DISPLAY=':99.0'
     - YARN_VERSION='1.9.4'
-    - MC_COMMIT='156f9442db84' # https://hg.mozilla.org/mozilla-central/shortlog
+    - MC_COMMIT='13b0256be449' # https://hg.mozilla.org/mozilla-central/shortlog
 
 notifications:
   slack:

bin/copy-assets.js

@@ -1,6 +1,7 @@
 const {
   tools: { makeBundle, symlinkTests, copyFile }
 } = require("devtools-launchpad/index");
+const sourceMapAssets = require("devtools-source-map/assets");
 const path = require("path");
 const minimist = require("minimist");
 var fs = require("fs");
@@ -210,13 +211,6 @@ function start() {
     { cwd: projectPath }
   );
 
-  console.log("[copy-assets] - dwarf_to_json.wasm");
-  copyFile(
-    path.join(projectPath, "./assets/wasm/dwarf_to_json.wasm"),
-    path.join(mcPath, "devtools/client/shared/source-map/dwarf_to_json.wasm"),
-    { cwd: projectPath }
-  );
-
   // Ensure /dist path exists.
   const bundlePath = "devtools/client/debugger/new/dist";
   shell.mkdir("-p", path.join(mcPath, bundlePath));
@@ -274,6 +268,13 @@ function onBundleFinish({mcPath, bundlePath, projectPath}) {
     path.join(mcPath, "devtools/client/shared/source-map/worker.js"),
     {cwd: projectPath}
   );
+  for (const filename of Object.keys(sourceMapAssets)) {
+    moveFile(
+      path.join(mcPath, bundlePath, "source-map-worker-assets", filename),
+      path.join(mcPath, "devtools/client/shared/source-map/assets", filename),
+      {cwd: projectPath}
+    );
+  }
 
   moveFile(
     path.join(mcPath, bundlePath, "source-map-index.js"),

bin/dev-server.js

@@ -16,7 +16,6 @@ let { app } = toolbox.startDevServer(envConfig, webpackConfig, __dirname);
 
 app.use("/integration/examples", express.static("src/test/mochitest/examples"));
 app.use("/images", serve(path.join(__dirname, "../assets/images")));
-app.use("/wasm", serve(path.join(__dirname, "../assets/wasm")));
 
 // Serve devtools-reps images
 app.use("/devtools-reps/images/", serve(path.join(__dirname, "../src/shared/images")));

package.json

@@ -114,6 +114,7 @@
     "babel-preset-react": "^6.24.1",
     "chalk": "^2.1.0",
     "copy-paste": "^1.3.0",
+    "copy-webpack-plugin": "^4.5.2",
     "devtools-license-check": "^0.7.0",
     "documentation": "^5.2.1",
     "enzyme": "^3.3.0",

packages/devtools-source-map/README.md

@@ -12,4 +12,6 @@ This is used in multiple contexts:
 # Application Requirements
 
 This package assumes that an application using this code will make the
-`worker.js` file available at application specified URL `workers.sourceMapURL`.
+`worker.js`, and `dwarf_to_json.wasm` files available and call
+`startSourceMapWorker(workerURL, wasmRoot)` to initialize the worker and specify
+the location of the wasm asset.

packages/devtools-source-map/assets.js

@@ -0,0 +1,9 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+// @flow
+
+module.exports = {
+  "dwarf_to_json.wasm": require.resolve("./wasm/dwarf_to_json.wasm"),
+  "source-map-mappings.wasm": require.resolve("source-map/lib/mappings.wasm")
+};

packages/devtools-source-map/package.json

@@ -9,6 +9,9 @@
   "homepage":
     "https://github.com/devtools-html/debugger.html/tree/master/packages/source-maps#readme",
   "main": "src/index.js",
+  "browser": {
+    "./src/utils/wasmAsset.js": "./src/utils/wasmAssetBrowser.js"
+  },
   "scripts": {
     "license-check": "devtools-license-check",
     "test": "jest --projects jest.config.js"
@@ -19,6 +22,6 @@
     "devtools-utils": "0.0.14",
     "md5": "^2.2.1",
     "regenerator-runtime": "^0.10.3",
-    "source-map": "^0.6.1"
+    "source-map": "^0.7.3"
   }
 }

packages/devtools-source-map/src/index.js

@@ -17,6 +17,7 @@ const {
 
 const dispatcher = new WorkerDispatcher();
 
+const setAssetRootURL = dispatcher.task("setAssetRootURL");
 const getOriginalURLs = dispatcher.task("getOriginalURLs");
 const getOriginalRanges = dispatcher.task("getOriginalRanges");
 const getGeneratedRanges = dispatcher.task("getGeneratedRanges", {
@@ -53,6 +54,9 @@ module.exports = {
   applySourceMap,
   clearSourceMaps,
   getOriginalStackFrames,
-  startSourceMapWorker: dispatcher.start.bind(dispatcher),
+  startSourceMapWorker(url: string, assetRoot: string) {
+    dispatcher.start(url);
+    setAssetRootURL(assetRoot);
+  },
   stopSourceMapWorker: dispatcher.stop.bind(dispatcher)
 };

packages/devtools-source-map/src/source-map.js

@@ -12,6 +12,7 @@
 const { networkRequest } = require("devtools-utils");
 const { SourceMapConsumer, SourceMapGenerator } = require("source-map");
 
+const { createConsumer } = require("./utils/createConsumer");
 const assert = require("./utils/assert");
 const { fetchSourceMap } = require("./utils/fetchSourceMap");
 const {
@@ -170,17 +171,36 @@ async function getGeneratedLocation(
     return location;
   }
 
-  const { line, column } = map.generatedPositionFor({
+  const positions = map.allGeneratedPositionsFor({
     source: originalSource.url,
     line: location.line,
-    column: location.column == null ? 0 : location.column,
-    bias: SourceMapConsumer.LEAST_UPPER_BOUND
+    column: location.column == null ? 0 : location.column
   });
 
+  // Prior to source-map 0.7, the source-map module returned the earliest
+  // generated location in the file when there were multiple generated
+  // locations. The current comparison fn in 0.7 does not appear to take
+  // generated location into account properly.
+  let match;
+  for (const pos of positions) {
+    if (!match || pos.line < match.line || pos.column < match.column) {
+      match = pos;
+    }
+  }
+
+  if (!match) {
+    match = map.generatedPositionFor({
+      source: originalSource.url,
+      line: location.line,
+      column: location.column == null ? 0 : location.column,
+      bias: SourceMapConsumer.LEAST_UPPER_BOUND
+    });
+  }
+
   return {
     sourceId: generatedSourceId,
-    line,
-    column
+    line: match.line,
+    column: match.column
   };
 }
 
@@ -305,7 +325,7 @@ function applySourceMap(
   mappings.forEach(mapping => generator.addMapping(mapping));
   generator.setSourceContent(url, code);
 
-  const map = SourceMapConsumer(generator.toJSON());
+  const map = createConsumer(generator.toJSON());
   setSourceMap(generatedId, Promise.resolve(map));
 }
 

packages/devtools-source-map/src/tests/wasm-source-map.js

@@ -5,7 +5,7 @@
 // Test WasmRemap
 
 const { WasmRemap } = require("../utils/wasmRemap");
-const { SourceMapConsumer } = require("source-map");
+const { createConsumer } = require("../utils/createConsumer");
 
 jest.mock("devtools-utils/src/network-request");
 
@@ -26,7 +26,7 @@ describe("wasm source maps", () => {
       { offset: 17, line: 2, column: 18 }
     ];
 
-    const map1 = new SourceMapConsumer(testMap1);
+    const map1 = await createConsumer(testMap1);
     const remap1 = new WasmRemap(map1);
 
     expect(remap1.file).toEqual("min.js");
@@ -81,7 +81,7 @@ describe("wasm source maps", () => {
       sourcesContent: ["//test"]
     };
 
-    const map2 = new SourceMapConsumer(testMap2);
+    const map2 = await createConsumer(testMap2);
     const remap2 = new WasmRemap(map2);
     expect(remap2.file).toEqual("none.js");
     expect(remap2.hasContentsOfAllSources()).toEqual(true);

packages/devtools-source-map/src/utils/convertToJSON.js

@@ -4,6 +4,8 @@
 
 /* eslint camelcase: 0*/
 
+const { getDwarfToWasmData } = require("./wasmAsset.js");
+
 let cachedWasmModule;
 let utf8Decoder;
 
@@ -38,16 +40,18 @@ function convertDwarf(wasm, instance) {
 }
 
 async function convertToJSON(buffer: ArrayBuffer): any {
-  if (!cachedWasmModule) {
-    const isFirefoxPanel =
-      typeof location !== "undefined" && location.protocol === "resource:";
-    const wasmPath = `${isFirefoxPanel ? "." : "/wasm"}/dwarf_to_json.wasm`;
-    const wasm = await (await fetch(wasmPath)).arrayBuffer();
-    const imports = {};
-    const { instance } = await WebAssembly.instantiate(wasm, imports);
-    cachedWasmModule = instance;
-  }
-  return convertDwarf(buffer, cachedWasmModule);
+  // Note: We don't 'await' here because it could mean that multiple
+  // calls to 'convertToJSON' could cause multiple fetches to be started.
+  cachedWasmModule = cachedWasmModule || loadConverterModule();
+
+  return convertDwarf(buffer, await cachedWasmModule);
+}
+
+async function loadConverterModule() {
+  const wasm = await getDwarfToWasmData();
+  const imports = {};
+  const { instance } = await WebAssembly.instantiate(wasm, imports);
+  return instance;
 }
 
 module.exports = {

packages/devtools-source-map/src/utils/createConsumer.js

@@ -0,0 +1,14 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+
+// @flow
+const { SourceMapConsumer } = require("source-map");
+
+async function createConsumer(map: any, sourceMapUrl: any): SourceMapConsumer {
+  return new SourceMapConsumer(map, sourceMapUrl);
+}
+
+module.exports = {
+  createConsumer
+};

packages/devtools-source-map/src/utils/fetchSourceMap.js

@@ -9,6 +9,7 @@ const { getSourceMap, setSourceMap } = require("./sourceMapRequests");
 const { WasmRemap } = require("./wasmRemap");
 const { SourceMapConsumer } = require("source-map");
 const { convertToJSON } = require("./convertToJSON");
+const { createConsumer } = require("./createConsumer");
 
 import type { Source } from "debugger-html";
 
@@ -44,7 +45,7 @@ async function _resolveAndFetch(generatedSource: Source): SourceMapConsumer {
   }
 
   // Create the source map and fix it up.
-  let map = new SourceMapConsumer(fetched.content, baseURL);
+  let map = await createConsumer(fetched.content, baseURL);
   if (generatedSource.isWasm) {
     map = new WasmRemap(map);
     // Check if experimental scope info exists.

packages/devtools-source-map/src/utils/wasmAsset.js

@@ -0,0 +1,30 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+// @flow
+
+const fs = require("fs");
+
+const assets = require("../../assets");
+
+function setAssetRootURL(assetRoot: string): void {
+  // No-op on Node
+}
+
+async function getDwarfToWasmData(): Promise<ArrayBuffer> {
+  const data = await new Promise((res, rej) => {
+    fs.readFile(assets["dwarf_to_json.wasm"], (err, result) => {
+      if (err) {
+        return rej(err);
+      }
+      res(result);
+    });
+  });
+
+  return data.buffer;
+}
+
+module.exports = {
+  setAssetRootURL,
+  getDwarfToWasmData
+};

packages/devtools-source-map/src/utils/wasmAssetBrowser.js

@@ -0,0 +1,31 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+// @flow
+
+const { SourceMapConsumer } = require("source-map");
+
+let root;
+function setAssetRootURL(assetRoot: string): void {
+  // Remove any trailing slash so we don't generate a double-slash below.
+  root = assetRoot.replace(/\/$/, "");
+
+  SourceMapConsumer.initialize({
+    "lib/mappings.wasm": `${root}/source-map-mappings.wasm`
+  });
+}
+
+async function getDwarfToWasmData(name: string): Promise<ArrayBuffer> {
+  if (!root) {
+    throw new Error(`No wasm path - Unable to resolve ${name}`);
+  }
+
+  const response = await fetch(`${root}/dwarf_to_json.wasm`);
+
+  return response.arrayBuffer();
+}
+
+module.exports = {
+  setAssetRootURL,
+  getDwarfToWasmData
+};

packages/devtools-source-map/src/worker.js

@@ -1,6 +1,7 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+// @flow
 
 const {
   getOriginalURLs,
@@ -10,13 +11,13 @@ const {
   getAllGeneratedLocations,
   getOriginalLocation,
   getOriginalSourceText,
-  getLocationScopes,
   hasMappedSource,
   clearSourceMaps,
   applySourceMap
 } = require("./source-map");
 
 const { getOriginalStackFrames } = require("./utils/getOriginalStackFrames");
+const { setAssetRootURL } = require("./utils/wasmAsset");
 
 const {
   workerUtils: { workerHandler }
@@ -25,13 +26,13 @@ const {
 // The interface is implemented in source-map to be
 // easier to unit test.
 self.onmessage = workerHandler({
+  setAssetRootURL,
   getOriginalURLs,
   getOriginalRanges,
   getGeneratedRanges,
   getGeneratedLocation,
   getAllGeneratedLocations,
   getOriginalLocation,
-  getLocationScopes,
   getOriginalSourceText,
   getOriginalStackFrames,
   hasMappedSource,

src/utils/bootstrap.js

@@ -68,7 +68,11 @@ export function bootstrapWorkers() {
 
   if (isDevelopment()) {
     // When used in Firefox, the toolbox manages the source map worker.
-    startSourceMapWorker(`${workerPath}/source-map-worker.js`);
+    startSourceMapWorker(
+      `${workerPath}/source-map-worker.js`,
+      // This is relative to the worker itself.
+      "./source-map-worker-assets/"
+    );
   }
 
   prettyPrint.start(`${workerPath}/pretty-print-worker.js`);