Skip to content

Commit

Permalink
Native ES modules
Browse files Browse the repository at this point in the history 
This is a major refactor of internal compiler.

Before: JS and TS both were sent through the typescript compiler where
their imports were parsed and handled. Both compiled to AMD JS and
finally sent to V8

Now: JS is sent directly into V8. TS is sent through the typescript
compiler, but tsc generates ES modules now instead of AMD. This
generated JS is then dumped into V8.

This should much faster for pure JS code. It may improve TS compilation
speed.

In the future this allows us to separate TS out of the runtime heap and
into its own dedicated snapshot. This will result in a smaller runtime
heap, and thus should be faster.

Many tests had to be disabled to make this happen. Apologies - but as
you can see this is a very complex patch - and I had to sledge hammer it
into place in a few places.

Also worth noting that this is necessary to support WASM

This patch also adds the basic framework for WebWorkers.
  • Loading branch information
@ry
ry committed Jan 4, 2019
1 parent ba7c993 commit fd4b392
Show file tree
Hide file tree
Showing 25 changed files with 750 additions and 281 deletions.
1 change: 1 addition & 0 deletions BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ ts_sources = [
"js/url.ts",
"js/url_search_params.ts",
"js/util.ts",
"js/workers.ts",
"js/write_file.ts",
"tsconfig.json",

Expand Down
169 changes: 35 additions & 134 deletions js/compiler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,34 +3,20 @@ import * as ts from "typescript";
import { MediaType } from "gen/msg_generated";
import { assetSourceCode } from "./assets";
import * as os from "./os";
// tslint:disable-next-line:no-circular-imports
import * as deno from "./deno";
import { globalEval } from "./global_eval";
import { assert, log, notImplemented } from "./util";

const window = globalEval("this");
// tslint:disable-next-line:no-circular-imports
// import * as deno from "./deno";

const EOL = "\n";
const ASSETS = "$asset$";
const LIB_RUNTIME = "lib.deno_runtime.d.ts";

// tslint:disable:no-any
type AmdCallback = (...args: any[]) => void;
type AmdErrback = (err: any) => void;
export type AmdFactory = (...args: any[]) => object | void;
// tslint:enable:no-any
export type AmdDefine = (deps: ModuleSpecifier[], factory: AmdFactory) => void;
type AMDRequire = (
deps: ModuleSpecifier[],
callback: AmdCallback,
errback: AmdErrback
) => void;

/** The location that a module is being loaded from. This could be a directory,
* like `.`, or it could be a module specifier like
* `http://gist.github.com/somefile.ts`
*/
type ContainingFile = string;
export type ContainingFile = string;
/** The internal local filename of a compiled module. It will often be something
* like `/home/ry/.deno/gen/f7b4605dfbc4d3bb356e98fda6ceb1481e4a8df5.js`
*/
Expand All @@ -42,7 +28,7 @@ type ModuleId = string;
/** The external name of a module - could be a URL or could be a relative path.
* Examples `http://gist.github.com/somefile.ts` or `./somefile.ts`
*/
type ModuleSpecifier = string;
export type ModuleSpecifier = string;
/** The compiled source code which is cached in `.deno/gen/` */
type OutputCode = string;
/** The original source code */
Expand Down Expand Up @@ -78,7 +64,6 @@ export interface Ts {
export class ModuleMetaData implements ts.IScriptSnapshot {
public deps?: ModuleFileName[];
public exports = {};
public factory?: AmdFactory;
public gatheringDeps = false;
public hasRun = false;
public scriptVersion = "";
Expand Down Expand Up @@ -149,8 +134,6 @@ export class DenoCompiler
ContainingFile,
Map<ModuleSpecifier, ModuleFileName>
>();
// A reference to global eval, so it can be monkey patched during testing
private _globalEval = globalEval;
// A reference to the log utility, so it can be monkey patched during testing
private _log = log;
// A map of module file names to module meta data
Expand All @@ -163,7 +146,7 @@ export class DenoCompiler
private readonly _options: ts.CompilerOptions = {
allowJs: true,
checkJs: true,
module: ts.ModuleKind.AMD,
module: ts.ModuleKind.ESNext,
outDir: "$deno$",
resolveJsonModule: true,
sourceMap: true,
Expand All @@ -184,9 +167,6 @@ export class DenoCompiler
// A reference to `typescript` module so it can be monkey patched during
// testing
private _ts: Ts = ts;
// A reference to the global scope so it can be monkey patched during
// testing
private _window = window;
// Flags forcing recompilation of TS code
public recompile = false;

Expand All @@ -200,21 +180,7 @@ export class DenoCompiler
);
let moduleMetaData: ModuleMetaData | undefined;
while ((moduleMetaData = this._runQueue.shift())) {
assert(
moduleMetaData.factory != null,
"Cannot run module without factory."
);
assert(moduleMetaData.hasRun === false, "Module has already been run.");
// asserts not tracked by TypeScripts, so using not null operator
const exports = moduleMetaData.factory!(
...this._getFactoryArguments(moduleMetaData)
);
// For JSON module support and potential future features.
// TypeScript always imports `exports` and mutates it directly, but the
// AMD specification allows values to be returned from the factory.
if (exports != null) {
moduleMetaData.exports = exports;
}
moduleMetaData.hasRun = true;
}
}
Expand All @@ -233,33 +199,6 @@ export class DenoCompiler
if (moduleMetaData.hasRun) {
return;
}

this._window.define = this._makeDefine(moduleMetaData);
this._globalEval(this.compile(moduleMetaData));
this._window.define = undefined;
}

/** Retrieve the arguments to pass a module's factory function. */
// tslint:disable-next-line:no-any
private _getFactoryArguments(moduleMetaData: ModuleMetaData): any[] {
if (!moduleMetaData.deps) {
throw new Error("Cannot get arguments until dependencies resolved.");
}
return moduleMetaData.deps.map(dep => {
if (dep === "require") {
return this._makeLocalRequire(moduleMetaData);
}
if (dep === "exports") {
return moduleMetaData.exports;
}
if (dep in DenoCompiler._builtins) {
return DenoCompiler._builtins[dep];
}
const dependencyMetaData = this._getModuleMetaData(dep);
assert(dependencyMetaData != null, `Missing dependency "${dep}".`);
// TypeScript does not track assert, therefore using not null operator
return dependencyMetaData!.exports;
});
}

/** The TypeScript language service often refers to the resolved fileName of
Expand All @@ -270,76 +209,14 @@ export class DenoCompiler
* TypeScript compiler, but the TypeScript compiler shouldn't be asking about
* external modules that we haven't told it about yet.
*/
private _getModuleMetaData(
fileName: ModuleFileName
): ModuleMetaData | undefined {
getModuleMetaData(fileName: ModuleFileName): ModuleMetaData | undefined {
return this._moduleMetaDataMap.has(fileName)
? this._moduleMetaDataMap.get(fileName)
: fileName.startsWith(ASSETS)
? this.resolveModule(fileName, "")
: undefined;
}

/** Create a localized AMD `define` function and return it. */
private _makeDefine(moduleMetaData: ModuleMetaData): AmdDefine {
return (deps: ModuleSpecifier[], factory: AmdFactory): void => {
this._log("compiler.localDefine", moduleMetaData.fileName);
moduleMetaData.factory = factory;
// when there are circular dependencies, we need to skip recursing the
// dependencies
moduleMetaData.gatheringDeps = true;
// we will recursively resolve the dependencies for any modules
moduleMetaData.deps = deps.map(dep => {
if (
dep === "require" ||
dep === "exports" ||
dep in DenoCompiler._builtins
) {
return dep;
}
const dependencyMetaData = this.resolveModule(
dep,
moduleMetaData.fileName
);
if (!dependencyMetaData.gatheringDeps) {
this._gatherDependencies(dependencyMetaData);
}
return dependencyMetaData.fileName;
});
moduleMetaData.gatheringDeps = false;
if (!this._runQueue.includes(moduleMetaData)) {
this._runQueue.push(moduleMetaData);
}
};
}

/** Returns a require that specifically handles the resolution of a transpiled
* emit of a dynamic ES `import()` from TypeScript.
*/
private _makeLocalRequire(moduleMetaData: ModuleMetaData): AMDRequire {
return (
deps: ModuleSpecifier[],
callback: AmdCallback,
errback: AmdErrback
): void => {
log("localRequire", deps);
assert(
deps.length === 1,
"Local require requires exactly one dependency."
);
const [moduleSpecifier] = deps;
try {
const requiredMetaData = this.run(
moduleSpecifier,
moduleMetaData.fileName
);
callback(requiredMetaData.exports);
} catch (e) {
errback(e);
}
};
}

/** Given a `moduleSpecifier` and `containingFile` retrieve the cached
* `fileName` for a given module. If the module has yet to be resolved
* this will return `undefined`.
Expand Down Expand Up @@ -496,6 +373,13 @@ export class DenoCompiler
return dependencies;
}

getOutput(filename: ModuleFileName): OutputCode {
const moduleMetaData = this.getModuleMetaData(filename)!;
assert(moduleMetaData != null, `Module not loaded: "${filename}"`);
this._scriptFileNames = [moduleMetaData.fileName];
return this.compile(moduleMetaData);
}

/** Given a `moduleSpecifier` and `containingFile`, resolve the module and
* return the `ModuleMetaData`.
*/
Expand Down Expand Up @@ -543,6 +427,7 @@ export class DenoCompiler
}
assert(moduleId != null, "No module ID.");
assert(fileName != null, "No file name.");
assert(sourceCode ? sourceCode.length > 0 : false, "No source code.");
assert(
mediaType !== MediaType.Unknown,
`Unknown media type for: "${moduleSpecifier}" from "${containingFile}".`
Expand Down Expand Up @@ -588,6 +473,20 @@ export class DenoCompiler
return moduleMetaData;
}

getSource(filename: ModuleFileName): SourceCode {
const moduleMetaData = this.getModuleMetaData(filename)!;
assert(moduleMetaData != null, `Module not loaded: "${filename}"`);
return moduleMetaData.sourceCode;
}

getJavaScriptSource(filename: ModuleFileName): OutputCode {
let s = this.getOutput(filename);
if (!s) {
s = this.getSource(filename);
}
return s;
}

// TypeScript Language Service and Format Diagnostic Host API

getCanonicalFileName(fileName: string): string {
Expand All @@ -613,7 +512,7 @@ export class DenoCompiler

getScriptKind(fileName: ModuleFileName): ts.ScriptKind {
this._log("getScriptKind()", fileName);
const moduleMetaData = this._getModuleMetaData(fileName);
const moduleMetaData = this.getModuleMetaData(fileName);
if (moduleMetaData) {
switch (moduleMetaData.mediaType) {
case MediaType.TypeScript:
Expand All @@ -632,13 +531,13 @@ export class DenoCompiler

getScriptVersion(fileName: ModuleFileName): string {
this._log("getScriptVersion()", fileName);
const moduleMetaData = this._getModuleMetaData(fileName);
const moduleMetaData = this.getModuleMetaData(fileName);
return (moduleMetaData && moduleMetaData.scriptVersion) || "";
}

getScriptSnapshot(fileName: ModuleFileName): ts.IScriptSnapshot | undefined {
this._log("getScriptSnapshot()", fileName);
return this._getModuleMetaData(fileName);
return this.getModuleMetaData(fileName);
}

getCurrentDirectory(): string {
Expand All @@ -664,7 +563,7 @@ export class DenoCompiler
}

fileExists(fileName: string): boolean {
const moduleMetaData = this._getModuleMetaData(fileName);
const moduleMetaData = this.getModuleMetaData(fileName);
const exists = moduleMetaData != null;
this._log("fileExists()", fileName, exists);
return exists;
Expand Down Expand Up @@ -711,11 +610,13 @@ export class DenoCompiler
* Placed as a private static otherwise we get use before
* declared with the `DenoCompiler`
*/
// tslint:disable-next-line:no-any
// tslint:disable-next-line:no-any,no-unused-variable
/*
private static _builtins: { [mid: string]: any } = {
typescript: ts,
deno
};
*/

private static _instance: DenoCompiler | undefined;

Expand Down
6 changes: 6 additions & 0 deletions js/globals.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import * as textEncoding from "./text_encoding";
import * as timers from "./timers";
import * as url from "./url";
import * as urlSearchParams from "./url_search_params";
import * as workers from "./workers";

// These imports are not exposed and therefore are fine to just import the
// symbols required.
Expand Down Expand Up @@ -78,3 +79,8 @@ window.TextEncoder = textEncoding.TextEncoder;
export type TextEncoder = textEncoding.TextEncoder;
window.TextDecoder = textEncoding.TextDecoder;
export type TextDecoder = textEncoding.TextDecoder;

window.workerMain = workers.workerMain;
// TODO These shouldn't be available in main isolate.
window.postMessage = workers.postMessage;
window.close = workers.workerClose;
Loading

0 comments on commit fd4b392

Please sign in to comment.