From 2bcddb10845183074dbf5c709d9a70dadbce6dcb Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Sat, 8 May 2021 00:05:50 -0700 Subject: [PATCH] feat(compartment-mapper): Introduce rudimentary bundler --- packages/compartment-mapper/NEWS.md | 3 + packages/compartment-mapper/src/bundle.js | 334 ++++++++++++++++++ packages/compartment-mapper/src/main.js | 1 + packages/compartment-mapper/src/types.js | 2 +- .../node_modules/bundle/import-all-from-me.js | 6 + .../bundle/import-and-export-all.js | 3 + .../import-and-reexport-name-from-me.js | 1 + .../bundle/import-default-export-from-me.js | 1 + .../bundle/import-named-export-and-rename.js | 1 + .../bundle/import-named-exports-from-me.js | 4 + .../test/node_modules/bundle/main.js | 20 ++ .../test/node_modules/bundle/package.json | 5 + .../test/node_modules/bundle/reexport-all.js | 1 + .../test/node_modules/bundle/reexport-name.js | 1 + .../compartment-mapper/test/test-bundle.js | 58 +++ 15 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 packages/compartment-mapper/src/bundle.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/import-all-from-me.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/import-and-export-all.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/import-and-reexport-name-from-me.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/import-default-export-from-me.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/import-named-export-and-rename.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/import-named-exports-from-me.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/main.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/package.json create mode 100644 packages/compartment-mapper/test/node_modules/bundle/reexport-all.js create mode 100644 packages/compartment-mapper/test/node_modules/bundle/reexport-name.js create mode 100644 packages/compartment-mapper/test/test-bundle.js diff --git a/packages/compartment-mapper/NEWS.md b/packages/compartment-mapper/NEWS.md index 1a34832648..d159a69b52 100644 --- a/packages/compartment-mapper/NEWS.md +++ b/packages/compartment-mapper/NEWS.md @@ -6,6 +6,9 @@ User-visible changes to the compartment mapper: Babel. * The Compartment Mapper now produces archives containing SES-shim pre-compiled StaticModuleRecords for ESM instead of the source. +* The Compartment Mapper can now produce bundles of concatenated modules but + without Compartments and only supporting ESM but not supporting live + bindings. * *BREAKING*: Archives created for the previous version will no longer work. The `importArchive` feature only supports pre-compiled ESM and CJS. * *BREAKING*: This release parallels a breaking upgrade for SES to version diff --git a/packages/compartment-mapper/src/bundle.js b/packages/compartment-mapper/src/bundle.js new file mode 100644 index 0000000000..742cd5ab8c --- /dev/null +++ b/packages/compartment-mapper/src/bundle.js @@ -0,0 +1,334 @@ +// @ts-check +/* eslint no-shadow: 0 */ + +import { resolve } from './node-module-specifier.js'; +import { compartmentMapForNodeModules } from './node-modules.js'; +import { search } from './search.js'; +import { link } from './assemble.js'; +import { makeImportHookMaker } from './import-hook.js'; +import { parseJson } from './parse-json.js'; +import { parseArchiveCjs } from './parse-archive-cjs.js'; +import { parseArchiveMjs } from './parse-archive-mjs.js'; +import { parseLocatedJson } from './json.js'; + +const textEncoder = new TextEncoder(); + +/** quotes strings */ +const q = JSON.stringify; + +/** @type {Record} */ +export const parserForLanguage = { + mjs: parseArchiveMjs, + cjs: parseArchiveCjs, + json: parseJson, +}; + +/** + * @param {Record} compartmentDescriptors + * @param {Record} compartmentSources + * @param {Record} compartmentResolvers + * @param {string} entryCompartmentName + * @param {string} entryModuleSpecifier + */ +const sortedModules = ( + compartmentDescriptors, + compartmentSources, + compartmentResolvers, + entryCompartmentName, + entryModuleSpecifier, +) => { + const modules = []; + const seen = new Set(); + + /** + * @param {string} compartmentName + * @param {string} moduleSpecifier + */ + const recur = (compartmentName, moduleSpecifier) => { + const key = `${compartmentName}#${moduleSpecifier}`; + if (seen.has(key)) { + return key; + } + seen.add(key); + + const resolve = compartmentResolvers[compartmentName]; + const source = compartmentSources[compartmentName][moduleSpecifier]; + if (source) { + const { record, parser } = source; + if (record) { + const { imports = [], reexports = [] } = record; + const resolvedImports = {}; + for (const importSpecifier of [...imports, ...reexports]) { + const resolvedSpecifier = resolve(importSpecifier, moduleSpecifier); + resolvedImports[importSpecifier] = recur( + compartmentName, + resolvedSpecifier, + ); + } + + modules.push({ + key, + compartmentName, + moduleSpecifier, + parser, + record, + resolvedImports, + }); + + return key; + } + } else { + const descriptor = + compartmentDescriptors[compartmentName].modules[moduleSpecifier]; + if (descriptor) { + const { + compartment: aliasCompartmentName, + module: aliasModuleSpecifier, + } = descriptor; + if ( + aliasCompartmentName !== undefined && + aliasModuleSpecifier !== undefined + ) { + return recur(aliasCompartmentName, aliasModuleSpecifier); + } + } + } + + throw new Error( + `Cannot bundle: cannot follow module import ${moduleSpecifier} in compartment ${compartmentName}`, + ); + }; + + recur(entryCompartmentName, entryModuleSpecifier); + + return modules; +}; + +/** + * @param {ReadFn} read + * @param {string} moduleLocation + * @param {Object} [options] + * @param {ModuleTransforms} [options.moduleTransforms] + * @returns {Promise} + */ +export const makeBundle = async (read, moduleLocation, options) => { + const { moduleTransforms } = options || {}; + const { + packageLocation, + packageDescriptorText, + packageDescriptorLocation, + moduleSpecifier, + } = await search(read, moduleLocation); + + /** @type {Set} */ + const tags = new Set(); + + const packageDescriptor = parseLocatedJson( + packageDescriptorText, + packageDescriptorLocation, + ); + const compartmentMap = await compartmentMapForNodeModules( + read, + packageLocation, + tags, + packageDescriptor, + moduleSpecifier, + ); + + const { + compartments, + entry: { compartment: entryCompartmentName, module: entryModuleSpecifier }, + } = compartmentMap; + /** @type {Sources} */ + const sources = {}; + + const makeImportHook = makeImportHookMaker( + read, + packageLocation, + sources, + compartments, + ); + + // Induce importHook to record all the necessary modules to import the given module specifier. + const { compartment, resolvers } = link(compartmentMap, { + resolve, + makeImportHook, + moduleTransforms, + parserForLanguage, + }); + await compartment.load(entryModuleSpecifier); + + const modules = sortedModules( + compartmentMap.compartments, + sources, + resolvers, + entryCompartmentName, + entryModuleSpecifier, + ); + + // Create an index of modules so we can resolve import specifiers to the + // index of the corresponding functor. + const modulesByKey = {}; + for (let index = 0; index < modules.length; index += 1) { + const module = modules[index]; + module.index = index; + modulesByKey[module.key] = module; + } + for (const module of modules) { + module.indexedImports = Object.fromEntries( + Object.entries(module.resolvedImports).map(([importSpecifier, key]) => [ + importSpecifier, + modulesByKey[key].index, + ]), + ); + } + + // Only support mjs format. + const problems = modules + .filter(module => module.parser !== 'premjs') + .map( + ({ moduleSpecifier, compartmentName, parser }) => + `module ${moduleSpecifier} in compartment ${compartmentName} in language ${parser}`, + ); + if (problems.length) { + throw new Error( + `Can only bundle applications that only have ESM (.mjs-type) modules, got ${problems.join( + ', ', + )}`, + ); + } + + const bundle = `\ +(functors => { + function cell(name, value = undefined) { + const observers = []; + function set(newValue) { + value = newValue; + for (const observe of observers) { + observe(value); + } + } + function get() { + return value; + } + function observe(observe) { + observers.push(observe); + observe(value); + } + return { get, set, observe, enumerable: true }; + } + + const cells = [${''.concat( + ...modules.map( + ({ record: { __fixedExportMap__, __liveExportMap__ } }) => `{ + ${''.concat( + ...Object.keys(__fixedExportMap__).map( + exportName => `${exportName}: cell(${q(exportName)}),\n`, + ), + )} + ${''.concat( + ...Object.keys(__liveExportMap__).map( + exportName => `${exportName}: cell(${q(exportName)}),\n`, + ), + )} + },`, + ), + )}]; + + ${''.concat( + ...modules.flatMap(({ index, indexedImports, record: { reexports } }) => + reexports.map( + (/* @type {string} */ importSpecifier) => `\ + Object.defineProperties(cells[${index}], Object.getOwnPropertyDescriptors(cells[${indexedImports[importSpecifier]}])); + `, + ), + ), + )} + + const namespaces = cells.map(cells => Object.create(null, cells)); + + for (let index = 0; index < namespaces.length; index += 1) { + cells[index]['*'] = cell('*', namespaces[index]); + } + + ${''.concat( + ...modules.map( + ({ + index, + indexedImports, + record: { __liveExportMap__, __fixedExportMap__ }, + }) => `\ + functors[${index}]({ + imports(map) { + ${''.concat( + ...Object.entries(indexedImports).map( + ([importName, importIndex]) => `\ + for (const [name, observers] of map.get(${q( + importName, + )}).entries()) { + const cell = cells[${importIndex}][name]; + if (cell === undefined) { + throw new ReferenceError(\`Cannot import name \${name}\`); + } + for (const observer of observers) { + cell.observe(observer); + } + } + `, + ), + )} + }, + liveVar: { + ${''.concat( + ...Object.entries(__liveExportMap__).map( + ([exportName, [importName]]) => `\ + ${importName}: cells[${index}].${exportName}.set, + `, + ), + )} + }, + onceVar: { + ${''.concat( + ...Object.entries(__fixedExportMap__).map( + ([exportName, [importName]]) => `\ + ${importName}: cells[${index}].${exportName}.set, + `, + ), + )} + }, + }); + `, + ), + )} + +})([ + ${''.concat( + ...modules.map( + ({ record: { __syncModuleProgram__ } }) => + `${__syncModuleProgram__}\n,\n`, + ), + )} +]); +`; + + return bundle; +}; + +/** + * @param {WriteFn} write + * @param {ReadFn} read + * @param {string} bundleLocation + * @param {string} moduleLocation + * @param {ArchiveOptions} [options] + */ +export const writeBundle = async ( + write, + read, + bundleLocation, + moduleLocation, + options, +) => { + const bundleString = await makeBundle(read, moduleLocation, options); + const bundleBytes = textEncoder.encode(bundleString); + await write(bundleLocation, bundleBytes); +}; diff --git a/packages/compartment-mapper/src/main.js b/packages/compartment-mapper/src/main.js index 312b2956a1..0995039587 100644 --- a/packages/compartment-mapper/src/main.js +++ b/packages/compartment-mapper/src/main.js @@ -4,3 +4,4 @@ export { makeArchive, writeArchive } from './archive.js'; export { parseArchive, loadArchive, importArchive } from './import-archive.js'; export { search } from './search.js'; export { compartmentMapForNodeModules } from './node-modules.js'; +export { makeBundle, writeBundle } from './bundle.js'; diff --git a/packages/compartment-mapper/src/types.js b/packages/compartment-mapper/src/types.js index 8cff089a2e..637b4b78a4 100644 --- a/packages/compartment-mapper/src/types.js +++ b/packages/compartment-mapper/src/types.js @@ -42,7 +42,7 @@ * package.json, there is a corresponding module descriptor. * * @typedef {Object} ModuleDescriptor - * @property {string} [compartment] + * @property {string=} [compartment] * @property {string} [module] * @property {string} [location] * @property {Language} [parser] diff --git a/packages/compartment-mapper/test/node_modules/bundle/import-all-from-me.js b/packages/compartment-mapper/test/node_modules/bundle/import-all-from-me.js new file mode 100644 index 0000000000..b80e10c305 --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/import-all-from-me.js @@ -0,0 +1,6 @@ +export const c = 'sea'; +export const i = 'eye'; +export const k = 'que'; +export const q = 'cue'; +export const u = 'you'; +export const y = 'why'; diff --git a/packages/compartment-mapper/test/node_modules/bundle/import-and-export-all.js b/packages/compartment-mapper/test/node_modules/bundle/import-and-export-all.js new file mode 100644 index 0000000000..d6a01e5e38 --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/import-and-export-all.js @@ -0,0 +1,3 @@ +export const red = '#f00'; +export const green = '#0f0'; +export const blue = '#00f'; diff --git a/packages/compartment-mapper/test/node_modules/bundle/import-and-reexport-name-from-me.js b/packages/compartment-mapper/test/node_modules/bundle/import-and-reexport-name-from-me.js new file mode 100644 index 0000000000..3949cd84f0 --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/import-and-reexport-name-from-me.js @@ -0,0 +1 @@ +export const qux = 'qux'; diff --git a/packages/compartment-mapper/test/node_modules/bundle/import-default-export-from-me.js b/packages/compartment-mapper/test/node_modules/bundle/import-default-export-from-me.js new file mode 100644 index 0000000000..d02ba545bd --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/import-default-export-from-me.js @@ -0,0 +1 @@ +export default 'foo'; diff --git a/packages/compartment-mapper/test/node_modules/bundle/import-named-export-and-rename.js b/packages/compartment-mapper/test/node_modules/bundle/import-named-export-and-rename.js new file mode 100644 index 0000000000..96f7c8f6b3 --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/import-named-export-and-rename.js @@ -0,0 +1 @@ +export const color = 'blue'; diff --git a/packages/compartment-mapper/test/node_modules/bundle/import-named-exports-from-me.js b/packages/compartment-mapper/test/node_modules/bundle/import-named-exports-from-me.js new file mode 100644 index 0000000000..01dc097333 --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/import-named-exports-from-me.js @@ -0,0 +1,4 @@ +const fizz = 'fizz'; +const bizz = 'bizz'; +const buzz = 'buzz'; +export { fizz, bizz, buzz }; diff --git a/packages/compartment-mapper/test/node_modules/bundle/main.js b/packages/compartment-mapper/test/node_modules/bundle/main.js new file mode 100644 index 0000000000..78fe317d0d --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/main.js @@ -0,0 +1,20 @@ +/* global print */ + +import foo from './import-default-export-from-me.js'; +print(foo); + +import * as bar from './import-all-from-me.js'; +print(bar); + +import { fizz, buzz } from './import-named-exports-from-me.js'; +print(fizz); +print(buzz); + +import { color as colour } from './import-named-export-and-rename.js'; +print(colour); + +import { qux } from './reexport-name.js'; +print(qux); + +import * as colors from './reexport-all.js'; +print(colors); diff --git a/packages/compartment-mapper/test/node_modules/bundle/package.json b/packages/compartment-mapper/test/node_modules/bundle/package.json new file mode 100644 index 0000000000..6b23559bb5 --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/package.json @@ -0,0 +1,5 @@ +{ + "name": "bundle", + "parsers": {"js": "mjs"}, + "main": "./main.js" +} diff --git a/packages/compartment-mapper/test/node_modules/bundle/reexport-all.js b/packages/compartment-mapper/test/node_modules/bundle/reexport-all.js new file mode 100644 index 0000000000..a6cfa5001f --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/reexport-all.js @@ -0,0 +1 @@ +export * from './import-and-export-all.js'; diff --git a/packages/compartment-mapper/test/node_modules/bundle/reexport-name.js b/packages/compartment-mapper/test/node_modules/bundle/reexport-name.js new file mode 100644 index 0000000000..9109467e8f --- /dev/null +++ b/packages/compartment-mapper/test/node_modules/bundle/reexport-name.js @@ -0,0 +1 @@ +export { qux } from './import-and-reexport-name-from-me.js'; diff --git a/packages/compartment-mapper/test/test-bundle.js b/packages/compartment-mapper/test/test-bundle.js new file mode 100644 index 0000000000..6383cc49f8 --- /dev/null +++ b/packages/compartment-mapper/test/test-bundle.js @@ -0,0 +1,58 @@ +import 'ses'; +import fs from 'fs'; +import test from 'ava'; +import { makeBundle, makeArchive, parseArchive } from '../src/main.js'; + +const fixture = new URL( + 'node_modules/bundle/main.js', + import.meta.url, +).toString(); + +const read = async location => fs.promises.readFile(new URL(location).pathname); + +const expectedLog = [ + 'foo', + { + c: 'sea', + i: 'eye', + q: 'cue', + k: 'que', + u: 'you', + y: 'why', + }, + 'fizz', + 'buzz', + 'blue', + 'qux', + { + red: '#f00', + green: '#0f0', + blue: '#00f', + }, +]; + +test('bundles work', async t => { + const bundle = await makeBundle(read, fixture); + t.log(bundle); + const log = []; + const print = entry => { + log.push(entry); + }; + const compartment = new Compartment({ print }); + compartment.evaluate(bundle); + t.deepEqual(log, expectedLog); +}); + +test('equivalent archive behaves the same as bundle', async t => { + const log = []; + const print = entry => { + log.push(entry); + }; + + const archive = await makeArchive(read, fixture); + const application = await parseArchive(archive, fixture); + await application.import({ + globals: { print }, + }); + t.deepEqual(log, expectedLog); +});