From ac6f55681eaff028c9069aef21d7d5a5f5fc208f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Feb 2022 00:23:35 +0100 Subject: [PATCH 01/27] compat: CJS/ESM interoperability --- cli/proc_state.rs | 17 ++++++++++++++++- imported.js | 1 + main.mjs | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 imported.js create mode 100644 main.mjs diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 320e20ac4d2180..49be96f04391fd 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -41,6 +41,7 @@ use deno_core::SharedArrayBufferStore; use deno_graph::create_graph; use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; +use deno_graph::source::ResolveResponse; use deno_graph::source::Loader; use deno_graph::ModuleKind; use deno_graph::Resolved; @@ -497,6 +498,14 @@ impl ProcState { is_dynamic ); + let mut needs_cjs_esm_translation = false; + if let Some(resolver) = &self.maybe_resolver { + if let Some(referrer) = maybe_referrer { + let response = resolver.resolve(specifier.as_str(), &referrer); + needs_cjs_esm_translation = matches!(response, ResolveResponse::CommonJs(_)); + } + } + let graph_data = self.graph_data.read(); let found = graph_data.follow_redirect(&specifier); match graph_data.get(&found) { @@ -508,7 +517,13 @@ impl ProcState { | MediaType::Unknown | MediaType::Cjs | MediaType::Mjs - | MediaType::Json => code.as_ref().clone(), + | MediaType::Json => { + if needs_cjs_esm_translation { + panic!("{} needs to be translated from CJS to ESM", specifier); + } else { + code.as_ref().clone() + } + }, MediaType::Dts => "".to_string(), _ => { let emit_path = self diff --git a/imported.js b/imported.js new file mode 100644 index 00000000000000..4c167df400512f --- /dev/null +++ b/imported.js @@ -0,0 +1 @@ +console.log(exports) diff --git a/main.mjs b/main.mjs new file mode 100644 index 00000000000000..6fbed1b7c059cf --- /dev/null +++ b/main.mjs @@ -0,0 +1 @@ +import "./imported.js"; From 3ec042c7df0b8c1765eb8920f42dd12e0d83f77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Feb 2022 00:29:12 +0100 Subject: [PATCH 02/27] example translation --- imported.js | 7 +++++++ imported.mjs | 7 +++++++ main.mjs | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 imported.mjs diff --git a/imported.js b/imported.js index 4c167df400512f..74d862938928cb 100644 --- a/imported.js +++ b/imported.js @@ -1 +1,8 @@ +exports = { + a: "A", + b: "B", +} +exports.foo = "foo"; +exports.bar = "bar"; + console.log(exports) diff --git a/imported.mjs b/imported.mjs new file mode 100644 index 00000000000000..5172daa5b20862 --- /dev/null +++ b/imported.mjs @@ -0,0 +1,7 @@ +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); +const mod = require("./imported.js"); + +export default mod; +export const foo = mod.foo; +export const bar = mod.bar; \ No newline at end of file diff --git a/main.mjs b/main.mjs index 6fbed1b7c059cf..058f209562bf23 100644 --- a/main.mjs +++ b/main.mjs @@ -1 +1 @@ -import "./imported.js"; +import "./imported.mjs"; From 91a7af743b79192c4689f19ee45e308fc6668491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Feb 2022 00:32:38 +0100 Subject: [PATCH 03/27] example translation with reexports --- imported.js | 4 +++- imported.mjs | 3 ++- reexports.js | 5 +++++ reexports2.js | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 reexports.js create mode 100644 reexports2.js diff --git a/imported.js b/imported.js index 74d862938928cb..95aca690f1761b 100644 --- a/imported.js +++ b/imported.js @@ -4,5 +4,7 @@ exports = { } exports.foo = "foo"; exports.bar = "bar"; +exports.fizz = require("./reexports.js"); -console.log(exports) +console.log(exports); +console.log(exports.fizz); diff --git a/imported.mjs b/imported.mjs index 5172daa5b20862..cf47f584f2399e 100644 --- a/imported.mjs +++ b/imported.mjs @@ -4,4 +4,5 @@ const mod = require("./imported.js"); export default mod; export const foo = mod.foo; -export const bar = mod.bar; \ No newline at end of file +export const bar = mod.bar; +export const fizz = mod.fizz; \ No newline at end of file diff --git a/reexports.js b/reexports.js new file mode 100644 index 00000000000000..8d18a4aa9d1852 --- /dev/null +++ b/reexports.js @@ -0,0 +1,5 @@ +module.exports = { + fizz: "FIZZ", +}; + +module.exports.buzz = require("./reexports2.js"); \ No newline at end of file diff --git a/reexports2.js b/reexports2.js new file mode 100644 index 00000000000000..327a03fec389b7 --- /dev/null +++ b/reexports2.js @@ -0,0 +1 @@ +module.exports = "buzz"; \ No newline at end of file From 8f42232fd24a80df0b8e62a96d7da48ee7ed241a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Feb 2022 10:56:52 +0100 Subject: [PATCH 04/27] working prototype --- Cargo.lock | 29 ++++++++++++++++------------- cli/Cargo.toml | 10 +++++----- cli/proc_state.rs | 21 ++++++++++++++++++++- main.mjs | 2 +- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f180776c530fb..f0e045897e2b99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -805,8 +805,7 @@ dependencies = [ [[package]] name = "deno_ast" version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2780a628fd2fd52c81ef3f9d2bda0609a662b9e4f85623c29051c277363f65" +source = "git+https://github.com/denoland/deno_ast#bb52e7297ada15cdeb39048494e1e45c89282685" dependencies = [ "anyhow", "base64 0.13.0", @@ -900,8 +899,6 @@ dependencies = [ [[package]] name = "deno_doc" version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a37040708cc0c8f1f826073774ea4406bee735d8cb6aacd0b9891e27be7c13bd" dependencies = [ "cfg-if 1.0.0", "deno_ast 0.10.0", @@ -946,8 +943,6 @@ dependencies = [ [[package]] name = "deno_graph" version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86eccec721ae9dceae2e6cbd83b63de3ef810e7c39357c9a867c98b9633d4775" dependencies = [ "anyhow", "cfg-if 1.0.0", @@ -982,8 +977,6 @@ dependencies = [ [[package]] name = "deno_lint" version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c838bbcaffd4143940068bea8eb5faa6de1ed8c3d3d0a63d8ed7319a967f47bd" dependencies = [ "anyhow", "deno_ast 0.10.0", @@ -1231,6 +1224,18 @@ dependencies = [ "serde", ] +[[package]] +name = "dprint-core" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a596556533e5739e71dfe105c8b4496c3eccaf5aa96d14b19db2fdf4157085a" +dependencies = [ + "anyhow", + "bumpalo", + "rustc-hash", + "serde", +] + [[package]] name = "dprint-plugin-json" version = "0.14.0" @@ -1238,7 +1243,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c531c535098419053a6ad7a166c7cfb940a54f97a36f934315394e61a329d85" dependencies = [ "anyhow", - "dprint-core", + "dprint-core 0.49.1", "jsonc-parser", "serde", ] @@ -1250,7 +1255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df207f2c04b4b473e30890efdb425cf5b7c15504d2a932c7a6a31402a2ed25fe" dependencies = [ "anyhow", - "dprint-core", + "dprint-core 0.49.1", "pulldown-cmark", "regex", "serde", @@ -1259,12 +1264,10 @@ dependencies = [ [[package]] name = "dprint-plugin-typescript" version = "0.62.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312bd921ea751a303703123e2e26e28456abd6ae9ff42dce478e16f613154874" dependencies = [ "anyhow", "deno_ast 0.10.0", - "dprint-core", + "dprint-core 0.50.0", "parking_lot_core", "rustc-hash", "serde", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 5b4095d55e1b1d..dee99b9ac35ea3 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -45,11 +45,11 @@ winapi = "=0.3.9" winres = "=0.1.11" [dependencies] -deno_ast = { version = "0.10.0", features = ["bundler", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] } +deno_ast = { git = "https://github.com/denoland/deno_ast", features = ["bundler", "cjs", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] } deno_core = { version = "0.117.0", path = "../core" } -deno_doc = "0.28.0" -deno_graph = "0.21.1" -deno_lint = { version = "0.23.0", features = ["docs"] } +deno_doc = { version = "0.28.0", path = "../../deno_doc" } +deno_graph = { version = "0.21.1", path = "../../deno_graph" } +deno_lint = { version = "0.23.0", features = ["docs"], path = "../../deno_lint" } deno_runtime = { version = "0.43.0", path = "../runtime" } atty = "=0.2.14" @@ -63,7 +63,7 @@ data-url = "=0.1.1" dissimilar = "=1.0.2" dprint-plugin-json = "=0.14.0" dprint-plugin-markdown = "=0.12.1" -dprint-plugin-typescript = "=0.62.1" +dprint-plugin-typescript = { version = "=0.62.1", path = "../../dprint-plugin-typescript" } encoding_rs = "=0.8.29" env_logger = "=0.8.4" fancy-regex = "=0.7.1" diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 49be96f04391fd..f37e16782d1e66 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -519,7 +519,26 @@ impl ProcState { | MediaType::Mjs | MediaType::Json => { if needs_cjs_esm_translation { - panic!("{} needs to be translated from CJS to ESM", specifier); + let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { + specifier: specifier.to_string(), + source: deno_ast::SourceTextInfo::new(code.clone()), + media_type: *media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + })?; + let analysis = parsed_source.analyze_cjs(); + let mut source = vec![ + r#"import { createRequire } from "node:module";"#.to_string(), + r#"const require = createRequire(import.meta.url);"#.to_string(), + ]; + source.push(format!("const mod = require(\"{}\");", specifier.to_file_path().unwrap().to_str().unwrap())); + source.push("export default mod".to_string()); + for export in analysis.exports { + source.push(format!("export const {} = mod.{}", export, export)); + } + + source.join("\n").to_string() } else { code.as_ref().clone() } diff --git a/main.mjs b/main.mjs index 058f209562bf23..6fbed1b7c059cf 100644 --- a/main.mjs +++ b/main.mjs @@ -1 +1 @@ -import "./imported.mjs"; +import "./imported.js"; From 7ce82b2a3eb0809a5ade33ae9e32da1ea949bacc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 1 Feb 2022 11:27:51 +0100 Subject: [PATCH 05/27] try to handle reexports --- cli/proc_state.rs | 48 +++++++++++++++++++++++++++++++---------------- imported.js | 6 +++--- reexports.js | 4 ++-- reexports2.js | 2 +- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/cli/proc_state.rs b/cli/proc_state.rs index f37e16782d1e66..13fe9cd17faf0e 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -41,8 +41,8 @@ use deno_core::SharedArrayBufferStore; use deno_graph::create_graph; use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; -use deno_graph::source::ResolveResponse; use deno_graph::source::Loader; +use deno_graph::source::ResolveResponse; use deno_graph::ModuleKind; use deno_graph::Resolved; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; @@ -502,7 +502,8 @@ impl ProcState { if let Some(resolver) = &self.maybe_resolver { if let Some(referrer) = maybe_referrer { let response = resolver.resolve(specifier.as_str(), &referrer); - needs_cjs_esm_translation = matches!(response, ResolveResponse::CommonJs(_)); + needs_cjs_esm_translation = + matches!(response, ResolveResponse::CommonJs(_)); } } @@ -519,30 +520,45 @@ impl ProcState { | MediaType::Mjs | MediaType::Json => { if needs_cjs_esm_translation { - let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { - specifier: specifier.to_string(), - source: deno_ast::SourceTextInfo::new(code.clone()), - media_type: *media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, - })?; + let parsed_source = + deno_ast::parse_script(deno_ast::ParseParams { + specifier: specifier.to_string(), + source: deno_ast::SourceTextInfo::new(code.clone()), + media_type: *media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + })?; let analysis = parsed_source.analyze_cjs(); let mut source = vec![ r#"import { createRequire } from "node:module";"#.to_string(), - r#"const require = createRequire(import.meta.url);"#.to_string(), + r#"const require = createRequire(import.meta.url);"# + .to_string(), ]; - source.push(format!("const mod = require(\"{}\");", specifier.to_file_path().unwrap().to_str().unwrap())); + source.push(format!( + "const mod = require(\"{}\");", + specifier.to_file_path().unwrap().to_str().unwrap() + )); source.push("export default mod".to_string()); - for export in analysis.exports { - source.push(format!("export const {} = mod.{}", export, export)); + for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { + // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, + // but it might not be necessary here since our analysis is more detailed? + source + .push(format!("export const {} = mod.{}", export, export)); + } + // TODO(bartlomieju): handle reexports + for (idx, reexport) in analysis.reexports.iter().enumerate() { + // source.push(format!( + // "const reexport{} = require(\"{}\");", + // idx, reexport + // )); + source.push(format!("throw new Error('Unhandled reexport {}')", reexport)); } - source.join("\n").to_string() } else { code.as_ref().clone() } - }, + } MediaType::Dts => "".to_string(), _ => { let emit_path = self diff --git a/imported.js b/imported.js index 95aca690f1761b..6ee5eb0837e8b2 100644 --- a/imported.js +++ b/imported.js @@ -1,7 +1,7 @@ exports = { - a: "A", - b: "B", -} + a: "A", + b: "B", +}; exports.foo = "foo"; exports.bar = "bar"; exports.fizz = require("./reexports.js"); diff --git a/reexports.js b/reexports.js index 8d18a4aa9d1852..6d338b4215f10d 100644 --- a/reexports.js +++ b/reexports.js @@ -1,5 +1,5 @@ module.exports = { - fizz: "FIZZ", + fizz: "FIZZ", }; -module.exports.buzz = require("./reexports2.js"); \ No newline at end of file +module.exports.buzz = require("./reexports2.js"); diff --git a/reexports2.js b/reexports2.js index 327a03fec389b7..fe30ee367cd116 100644 --- a/reexports2.js +++ b/reexports2.js @@ -1 +1 @@ -module.exports = "buzz"; \ No newline at end of file +module.exports = "buzz"; From f034640d33f93f5de44e59d537e9d89a89e62f46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 2 Feb 2022 22:39:46 +0100 Subject: [PATCH 06/27] fix after merge --- Cargo.lock | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c9125954c9a7a6..2147b0efac7f46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1214,18 +1214,6 @@ dependencies = [ "serde", ] -[[package]] -name = "dprint-core" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a596556533e5739e71dfe105c8b4496c3eccaf5aa96d14b19db2fdf4157085a" -dependencies = [ - "anyhow", - "bumpalo", - "rustc-hash", - "serde", -] - [[package]] name = "dprint-plugin-json" version = "0.14.1" @@ -1233,7 +1221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d7673b479d64372e60e344722789ed9c61ea1023e9a8a13c7da8e7d63b0f7dc" dependencies = [ "anyhow", - "dprint-core 0.49.1", + "dprint-core", "jsonc-parser", "serde", "text_lines", @@ -1246,7 +1234,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00a8c4e905bf5c95dbcc6eab7d81a1d884f7fbd0e7f2fd99af0154692fabc8a8" dependencies = [ "anyhow", - "dprint-core 0.49.1", + "dprint-core", "pulldown-cmark", "regex", "serde", From c2474f2bbd1dfb8ed478b5d76e1e8e3dedb11714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 3 Feb 2022 00:16:50 +0100 Subject: [PATCH 07/27] wip --- cli/Cargo.toml | 2 +- cli/compat/mod.rs | 19 +++++++++++++ cli/proc_state.rs | 70 +++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b4444597498dbc..fa22ea1c579b4c 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -45,7 +45,7 @@ winapi = "=0.3.9" winres = "=0.1.11" [dependencies] -deno_ast = { version = "0.11.0", features = ["bundler", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] } +deno_ast = { version = "0.11.0", features = ["bundler", "cjs", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] } deno_core = { version = "0.117.0", path = "../core" } deno_doc = "0.29.0" deno_graph = "0.22.0" diff --git a/cli/compat/mod.rs b/cli/compat/mod.rs index abfffb2d273fcb..1f7399b0a8fd35 100644 --- a/cli/compat/mod.rs +++ b/cli/compat/mod.rs @@ -137,6 +137,25 @@ pub(crate) fn add_global_require( Ok(()) } +pub(crate) fn resolve_cjs_module( + js_runtime: &mut JsRuntime, + referrer_mod: &str, + mod_to_resolve: &str, +) -> Result<(), AnyError> { + let source_code = &format!( + r#"const CJSModule = require("module"); + const referrerMod = require("{}"); + const resolvedMod = CJSModule._resolveFilename("{}", referrerMod); + return resolvedMod; + "#, + escape_for_single_quote_string(referrer_mod), + escape_for_single_quote_string(mod_to_resolve), + ); + + js_runtime.execute_script(&located_script_name!(), source_code)?; + Ok(()) +} + fn escape_for_single_quote_string(text: &str) -> String { text.replace(r"\", r"\\").replace("'", r"\'") } diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 13fe9cd17faf0e..300e175dd404e2 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -343,6 +343,15 @@ impl ProcState { None, ) .await; + + for module in graph.modules() { + if module.kind == ModuleKind::CommonJs { + eprintln!("cjs module {}", module.specifier); + eprintln!("deps {:#?}", module.dependencies); + eprintln!("maybe source {:#?}", module.maybe_source); + } + } + // If there was a locker, validate the integrity of all the modules in the // locker. graph_lock_or_exit(&graph); @@ -540,9 +549,11 @@ impl ProcState { specifier.to_file_path().unwrap().to_str().unwrap() )); source.push("export default mod".to_string()); - for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { + for export in + analysis.exports.iter().filter(|e| e.as_str() != "default") + { // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, - // but it might not be necessary here since our analysis is more detailed? + // but it might not be necessary here since our analysis is more detailed? source .push(format!("export const {} = mod.{}", export, export)); } @@ -552,7 +563,10 @@ impl ProcState { // "const reexport{} = require(\"{}\");", // idx, reexport // )); - source.push(format!("throw new Error('Unhandled reexport {}')", reexport)); + source.push(format!( + "throw new Error('Unhandled reexport {}')", + reexport + )); } source.join("\n").to_string() } else { @@ -701,3 +715,53 @@ fn source_map_from_code(code: String) -> Option> { None } } + +fn translate_cjs_to_esm( + specifier: ModuleSpecifier, + // TODO(bartlomieju): could use `maybe_parsed_source` if available + code: String, + media_type: MediaType, +) -> String { + let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { + specifier: specifier.to_string(), + source: deno_ast::SourceTextInfo::new(code), + media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + })?; + let analysis = parsed_source.analyze_cjs(); + + // if there are reexports, handle them first + + let mut source = vec![ + r#"import { createRequire } from "node:module";"#.to_string(), + r#"const require = createRequire(import.meta.url);"#.to_string(), + ]; + + source.push(format!( + "const mod = require(\"{}\");", + specifier.to_file_path().unwrap().to_str().unwrap() + )); + source.push("export default mod".to_string()); + + for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { + // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, + // but it might not be necessary here since our analysis is more detailed? + source.push(format!("export const {} = mod.{}", export, export)); + } + + // TODO(bartlomieju): handle reexports + for (idx, reexport) in analysis.reexports.iter().enumerate() { + // source.push(format!( + // "const reexport{} = require(\"{}\");", + // idx, reexport + // )); + source.push(format!( + "throw new Error('Unhandled reexport {}')", + reexport + )); + } + + source.join("\n").to_string() +} From 53723e463b65492ebc3b4c62f665f2613ae43fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 3 Feb 2022 23:38:40 +0100 Subject: [PATCH 08/27] wip2 --- cli/proc_state.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 300e175dd404e2..4f62aa1e14335c 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -344,11 +344,20 @@ impl ProcState { ) .await; - for module in graph.modules() { - if module.kind == ModuleKind::CommonJs { - eprintln!("cjs module {}", module.specifier); - eprintln!("deps {:#?}", module.dependencies); - eprintln!("maybe source {:#?}", module.maybe_source); + let needs_cjs_esm_translation = graph.modules().iter().find(|m| m.kind == ModuleKind::CommonJs); + + if needs_cjs_esm_translation { + // TODO(bartlomieju): extremely inefficient to create a new worker, just + // to do CJS module resolution, but we currently don't have implementation + // of it in Rust + // TODO: create new MainWorker here + + for module in graph.modules() { + if module.kind == ModuleKind::CommonJs { + eprintln!("cjs module {}", module.specifier); + eprintln!("deps {:#?}", module.dependencies); + eprintln!("maybe source {:#?}", module.maybe_source); + } } } @@ -733,6 +742,10 @@ fn translate_cjs_to_esm( let analysis = parsed_source.analyze_cjs(); // if there are reexports, handle them first + for (idx, reexport) in analysis.reexports.iter().enumerate() { + // first resolve relate reexport specifier + + } let mut source = vec![ r#"import { createRequire } from "node:module";"#.to_string(), From 810ac05b15dd7a3b5623c4a961e5033792c3dd9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 4 Feb 2022 00:28:12 +0100 Subject: [PATCH 09/27] recursive analysis --- cli/compat/mod.rs | 14 ++++- cli/proc_state.rs | 148 +++++++++++++++++++++++++++++----------------- reexports.js | 4 -- reexports2.js | 3 +- 4 files changed, 106 insertions(+), 63 deletions(-) diff --git a/cli/compat/mod.rs b/cli/compat/mod.rs index 1f7399b0a8fd35..1375648af1fd90 100644 --- a/cli/compat/mod.rs +++ b/cli/compat/mod.rs @@ -5,7 +5,9 @@ mod esm_resolver; use deno_core::error::AnyError; use deno_core::located_script_name; +use deno_core::serde_v8; use deno_core::url::Url; +use deno_core::v8; use deno_core::JsRuntime; use once_cell::sync::Lazy; @@ -141,7 +143,7 @@ pub(crate) fn resolve_cjs_module( js_runtime: &mut JsRuntime, referrer_mod: &str, mod_to_resolve: &str, -) -> Result<(), AnyError> { +) -> Result { let source_code = &format!( r#"const CJSModule = require("module"); const referrerMod = require("{}"); @@ -152,8 +154,14 @@ pub(crate) fn resolve_cjs_module( escape_for_single_quote_string(mod_to_resolve), ); - js_runtime.execute_script(&located_script_name!(), source_code)?; - Ok(()) + let value = + js_runtime.execute_script(&located_script_name!(), source_code)?; + let resolved_specifier = { + let scope = &mut js_runtime.handle_scope(); + let local = v8::Local::::new(scope, value); + serde_v8::from_v8(scope, local)? + }; + Ok(resolved_specifier) } fn escape_for_single_quote_string(text: &str) -> String { diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 4f62aa1e14335c..5f7c89f3c4dd47 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -34,6 +34,7 @@ use deno_core::parking_lot::RwLock; use deno_core::resolve_url; use deno_core::url::Url; use deno_core::CompiledWasmModuleStore; +use deno_core::JsRuntime; use deno_core::ModuleSource; use deno_core::ModuleSpecifier; use deno_core::ModuleType; @@ -344,7 +345,11 @@ impl ProcState { ) .await; - let needs_cjs_esm_translation = graph.modules().iter().find(|m| m.kind == ModuleKind::CommonJs); + let needs_cjs_esm_translation = graph + .modules() + .iter() + .find(|m| m.kind == ModuleKind::CommonJs) + .is_some(); if needs_cjs_esm_translation { // TODO(bartlomieju): extremely inefficient to create a new worker, just @@ -635,6 +640,93 @@ impl ProcState { None } } + + /// Translates given CJS module into ESM. This function will perform static + /// analysis on the file to find defined exports and reexports. + /// + /// For all discovered reexports the analysis will be performed recursively. + /// + /// If successful a source code for equivalent ES module is returned. + fn translate_cjs_to_esm( + &self, + js_runtime: &mut JsRuntime, + specifier: ModuleSpecifier, + // TODO(bartlomieju): could use `maybe_parsed_source` if available + code: String, + media_type: MediaType, + ) -> Result { + let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { + specifier: specifier.to_string(), + source: deno_ast::SourceTextInfo::new(Arc::new(code)), + media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + })?; + let analysis = parsed_source.analyze_cjs(); + + // if there are reexports, handle them first + for (idx, reexport) in analysis.reexports.iter().enumerate() { + // Firstly, resolve relate reexport specifier + let resolved_reexport = compat::resolve_cjs_module( + js_runtime, + specifier.as_str(), + reexport.as_str(), + )?; + let reexport_specifier = + ModuleSpecifier::from_file_path(&resolved_reexport).unwrap(); + // Secondly, read the source code from disk + let reexport_file = self.file_fetcher.get_source(&reexport_specifier).unwrap(); + // Now perform analysis again + + { + let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { + specifier: resolved_reexport, + source: deno_ast::SourceTextInfo::new(reexport_file.source), + media_type: reexport_file.media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + })?; + let analysis = parsed_source.analyze_cjs(); + + // And recurse again if there are reexports! + } + + todo!() + } + + let mut source = vec![ + r#"import { createRequire } from "node:module";"#.to_string(), + r#"const require = createRequire(import.meta.url);"#.to_string(), + ]; + + source.push(format!( + "const mod = require(\"{}\");", + specifier.to_file_path().unwrap().to_str().unwrap() + )); + source.push("export default mod".to_string()); + + for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { + // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, + // but it might not be necessary here since our analysis is more detailed? + source.push(format!("export const {} = mod.{}", export, export)); + } + + // TODO(bartlomieju): handle reexports + for (idx, reexport) in analysis.reexports.iter().enumerate() { + // source.push(format!( + // "const reexport{} = require(\"{}\");", + // idx, reexport + // )); + source.push(format!( + "throw new Error('Unhandled reexport {}')", + reexport + )); + } + + Ok(source.join("\n").to_string()) + } } // TODO(@kitsonk) this is only temporary, but should be refactored to somewhere @@ -724,57 +816,3 @@ fn source_map_from_code(code: String) -> Option> { None } } - -fn translate_cjs_to_esm( - specifier: ModuleSpecifier, - // TODO(bartlomieju): could use `maybe_parsed_source` if available - code: String, - media_type: MediaType, -) -> String { - let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { - specifier: specifier.to_string(), - source: deno_ast::SourceTextInfo::new(code), - media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, - })?; - let analysis = parsed_source.analyze_cjs(); - - // if there are reexports, handle them first - for (idx, reexport) in analysis.reexports.iter().enumerate() { - // first resolve relate reexport specifier - - } - - let mut source = vec![ - r#"import { createRequire } from "node:module";"#.to_string(), - r#"const require = createRequire(import.meta.url);"#.to_string(), - ]; - - source.push(format!( - "const mod = require(\"{}\");", - specifier.to_file_path().unwrap().to_str().unwrap() - )); - source.push("export default mod".to_string()); - - for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { - // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, - // but it might not be necessary here since our analysis is more detailed? - source.push(format!("export const {} = mod.{}", export, export)); - } - - // TODO(bartlomieju): handle reexports - for (idx, reexport) in analysis.reexports.iter().enumerate() { - // source.push(format!( - // "const reexport{} = require(\"{}\");", - // idx, reexport - // )); - source.push(format!( - "throw new Error('Unhandled reexport {}')", - reexport - )); - } - - source.join("\n").to_string() -} diff --git a/reexports.js b/reexports.js index 6d338b4215f10d..7a0e8843e22867 100644 --- a/reexports.js +++ b/reexports.js @@ -1,5 +1 @@ -module.exports = { - fizz: "FIZZ", -}; - module.exports.buzz = require("./reexports2.js"); diff --git a/reexports2.js b/reexports2.js index fe30ee367cd116..183d833b097a89 100644 --- a/reexports2.js +++ b/reexports2.js @@ -1 +1,2 @@ -module.exports = "buzz"; +exports.buzz = "buzz"; +exports.fizz = "FIZZ"; From 898cf1d484c8e4263e714c6c8fc58eefa1d1f3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 4 Feb 2022 01:04:55 +0100 Subject: [PATCH 10/27] more progress --- cli/compat/mod.rs | 18 ++++++++--- cli/proc_state.rs | 77 +++++++++++++++++++++++++++++++---------------- 2 files changed, 64 insertions(+), 31 deletions(-) diff --git a/cli/compat/mod.rs b/cli/compat/mod.rs index 1375648af1fd90..a34a16f33c4d3a 100644 --- a/cli/compat/mod.rs +++ b/cli/compat/mod.rs @@ -139,23 +139,31 @@ pub(crate) fn add_global_require( Ok(()) } -pub(crate) fn resolve_cjs_module( +pub(crate) async fn resolve_cjs_module( js_runtime: &mut JsRuntime, + fake_main: &str, referrer_mod: &str, mod_to_resolve: &str, ) -> Result { + eprintln!("resolve_cjs_module {} {}", referrer_mod, mod_to_resolve); let source_code = &format!( - r#"const CJSModule = require("module"); - const referrerMod = require("{}"); - const resolvedMod = CJSModule._resolveFilename("{}", referrerMod); - return resolvedMod; + r#"(async function resolveCjsModule(main) {{ + const CJSModule = await import("{}"); + const require = CJSModule.createRequire(main); + const referrerMod = require("{}"); + const resolvedMod = CJSModule.default._resolveFilename("{}", referrerMod); + return resolvedMod; + }})('{}'); "#, + MODULE_URL_STR.as_str(), escape_for_single_quote_string(referrer_mod), escape_for_single_quote_string(mod_to_resolve), + fake_main, ); let value = js_runtime.execute_script(&located_script_name!(), source_code)?; + let value = js_runtime.resolve_value(value).await?; let resolved_specifier = { let scope = &mut js_runtime.handle_scope(); let local = v8::Local::::new(scope, value); diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 5f7c89f3c4dd47..e2579fb9728a71 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -18,6 +18,7 @@ use crate::graph_util::ModuleEntry; use crate::http_cache; use crate::lockfile::as_maybe_locker; use crate::lockfile::Lockfile; +use crate::create_main_worker; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; use crate::source_maps::SourceMapGetter; @@ -356,12 +357,36 @@ impl ProcState { // to do CJS module resolution, but we currently don't have implementation // of it in Rust // TODO: create new MainWorker here + let mut worker = create_main_worker( + self, + deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts").unwrap(), + if is_dynamic { + dynamic_permissions + } else { + root_permissions + }, + vec![], + ); + // Set up Node globals + worker.execute_side_module(&compat::GLOBAL_URL).await?; + // And `module` module that we'll use for checking which + // loader to use and potentially load CJS module with. + // This allows to skip permission check for `--allow-net` + // which would otherwise be requested by dynamically importing + // this file. + worker.execute_side_module(&compat::MODULE_URL).await?; for module in graph.modules() { if module.kind == ModuleKind::CommonJs { eprintln!("cjs module {}", module.specifier); eprintln!("deps {:#?}", module.dependencies); eprintln!("maybe source {:#?}", module.maybe_source); + self.translate_cjs_to_esm( + &mut worker.js_runtime, + &module.specifier, + module.maybe_source.as_ref().unwrap().to_string(), + module.media_type, + ).await?; } } } @@ -572,7 +597,7 @@ impl ProcState { .push(format!("export const {} = mod.{}", export, export)); } // TODO(bartlomieju): handle reexports - for (idx, reexport) in analysis.reexports.iter().enumerate() { + for (_idx, reexport) in analysis.reexports.iter().enumerate() { // source.push(format!( // "const reexport{} = require(\"{}\");", // idx, reexport @@ -647,10 +672,10 @@ impl ProcState { /// For all discovered reexports the analysis will be performed recursively. /// /// If successful a source code for equivalent ES module is returned. - fn translate_cjs_to_esm( + async fn translate_cjs_to_esm( &self, js_runtime: &mut JsRuntime, - specifier: ModuleSpecifier, + specifier: &ModuleSpecifier, // TODO(bartlomieju): could use `maybe_parsed_source` if available code: String, media_type: MediaType, @@ -665,14 +690,20 @@ impl ProcState { })?; let analysis = parsed_source.analyze_cjs(); + let mut source = vec![ + r#"import { createRequire } from "node:module";"#.to_string(), + r#"const require = createRequire(import.meta.url);"#.to_string(), + ]; + // if there are reexports, handle them first for (idx, reexport) in analysis.reexports.iter().enumerate() { // Firstly, resolve relate reexport specifier let resolved_reexport = compat::resolve_cjs_module( js_runtime, - specifier.as_str(), + deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts").unwrap().as_str(), + specifier.to_file_path().unwrap().to_str().unwrap(), reexport.as_str(), - )?; + ).await?; let reexport_specifier = ModuleSpecifier::from_file_path(&resolved_reexport).unwrap(); // Secondly, read the source code from disk @@ -690,16 +721,20 @@ impl ProcState { })?; let analysis = parsed_source.analyze_cjs(); + // TODO: // And recurse again if there are reexports! - } - todo!() - } + source.push( + format!("const reexport{} = require(\"{}\");", idx, reexport) + ); - let mut source = vec![ - r#"import { createRequire } from "node:module";"#.to_string(), - r#"const require = createRequire(import.meta.url);"#.to_string(), - ]; + for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { + // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, + // but it might not be necessary here since our analysis is more detailed? + source.push(format!("export const {} = reexport{}.{};", export, idx, export)); + } + } + } source.push(format!( "const mod = require(\"{}\");", @@ -710,22 +745,12 @@ impl ProcState { for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, // but it might not be necessary here since our analysis is more detailed? - source.push(format!("export const {} = mod.{}", export, export)); - } - - // TODO(bartlomieju): handle reexports - for (idx, reexport) in analysis.reexports.iter().enumerate() { - // source.push(format!( - // "const reexport{} = require(\"{}\");", - // idx, reexport - // )); - source.push(format!( - "throw new Error('Unhandled reexport {}')", - reexport - )); + source.push(format!("export const {} = mod.{};", export, export)); } - Ok(source.join("\n").to_string()) + let translated_source = source.join("\n").to_string(); + eprintln!("translated source: {}", translated_source); + Ok(translated_source) } } From 703da80b4becd4214ef498ee89fe36839078c156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 4 Feb 2022 16:43:37 +0100 Subject: [PATCH 11/27] almost working --- cli/compat/esm_resolver.rs | 11 +++++-- cli/compat/mod.rs | 4 ++- cli/graph_util.rs | 18 +++++++++++ cli/proc_state.rs | 63 +++++--------------------------------- 4 files changed, 38 insertions(+), 58 deletions(-) diff --git a/cli/compat/esm_resolver.rs b/cli/compat/esm_resolver.rs index 300b76f31a11e0..6fbbbe9206ed84 100644 --- a/cli/compat/esm_resolver.rs +++ b/cli/compat/esm_resolver.rs @@ -219,6 +219,7 @@ fn finalize_resolution( return Ok(resolved); } + eprintln!("resolved {} {}", resolved.as_str(), base.as_str()); let encoded_sep_re = Regex::new(r"%2F|%2C").expect("bad regex"); if encoded_sep_re.is_match(resolved.path()) { @@ -794,9 +795,10 @@ fn package_resolve( base: &ModuleSpecifier, conditions: &[&str], ) -> Result { + eprintln!("specifier {}", specifier); let (package_name, package_subpath, is_scoped) = parse_package_name(specifier, base)?; - + eprintln!("package name {} {} {}", package_name, package_subpath, is_scoped); // ResolveSelf let package_config = get_package_scope_config(base)?; if package_config.exists { @@ -805,6 +807,7 @@ fn package_resolve( if package_config.name.as_ref() == Some(&package_name) { if let Some(exports) = &package_config.exports { if !exports.is_null() { + eprintln!("here1"); return package_exports_resolve( package_json_url, package_subpath, @@ -854,6 +857,7 @@ fn package_resolve( let package_config = get_package_config(package_json_path.clone(), specifier, Some(base))?; if package_config.exports.is_some() { + eprintln!("here2"); return package_exports_resolve( package_json_url, package_subpath, @@ -863,14 +867,17 @@ fn package_resolve( ); } if package_subpath == "." { + eprintln!("here3"); return legacy_main_resolve(&package_json_url, &package_config, base); } + eprintln!("here4 {} {}", package_json_url, package_subpath); return package_json_url .join(&package_subpath) .map_err(AnyError::from); } + eprintln!("erroring!"); Err(errors::err_module_not_found( &package_json_url .join(".") @@ -903,7 +910,7 @@ fn parse_package_name( } let package_name = if let Some(index) = separator_index { - specifier[0..=index].to_string() + specifier[0..index].to_string() } else { specifier.to_string() }; diff --git a/cli/compat/mod.rs b/cli/compat/mod.rs index a34a16f33c4d3a..072dc81ec29249 100644 --- a/cli/compat/mod.rs +++ b/cli/compat/mod.rs @@ -150,13 +150,15 @@ pub(crate) async fn resolve_cjs_module( r#"(async function resolveCjsModule(main) {{ const CJSModule = await import("{}"); const require = CJSModule.createRequire(main); - const referrerMod = require("{}"); + require("{}"); + const referrerMod = CJSModule.default._cache["{}"]; const resolvedMod = CJSModule.default._resolveFilename("{}", referrerMod); return resolvedMod; }})('{}'); "#, MODULE_URL_STR.as_str(), escape_for_single_quote_string(referrer_mod), + escape_for_single_quote_string(referrer_mod), escape_for_single_quote_string(mod_to_resolve), fake_main, ); diff --git a/cli/graph_util.rs b/cli/graph_util.rs index f0a38b7a108bac..ac30871062ff4c 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -53,6 +53,7 @@ pub(crate) struct GraphData { /// error messages. referrer_map: HashMap, configurations: HashSet, + cjs_esm_translations: HashMap, } impl GraphData { @@ -254,6 +255,7 @@ impl GraphData { modules, referrer_map, configurations: self.configurations.clone(), + cjs_esm_translations: Default::default() }) } @@ -412,6 +414,22 @@ impl GraphData { ) -> Option<&'a ModuleEntry> { self.modules.get(specifier) } + + pub(crate) fn add_cjs_esm_translation( + &mut self, + specifier: &ModuleSpecifier, + source: String, + ) { + let prev = self.cjs_esm_translations.insert(specifier.to_owned(), source); + assert!(prev.is_none()); + } + + pub(crate) fn get_cjs_esm_translation<'a>( + &'a self, + specifier: &ModuleSpecifier + ) -> Option<&'a String> { + self.cjs_esm_translations.get(specifier) + } } impl From<&ModuleGraph> for GraphData { diff --git a/cli/proc_state.rs b/cli/proc_state.rs index e2579fb9728a71..a5d7fc59ceb504 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -44,7 +44,6 @@ use deno_graph::create_graph; use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; use deno_graph::source::Loader; -use deno_graph::source::ResolveResponse; use deno_graph::ModuleKind; use deno_graph::Resolved; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; @@ -379,14 +378,17 @@ impl ProcState { for module in graph.modules() { if module.kind == ModuleKind::CommonJs { eprintln!("cjs module {}", module.specifier); - eprintln!("deps {:#?}", module.dependencies); - eprintln!("maybe source {:#?}", module.maybe_source); - self.translate_cjs_to_esm( + // eprintln!("deps {:#?}", module.dependencies); + // eprintln!("maybe source {:#?}", module.maybe_source); + let translated_source = self.translate_cjs_to_esm( &mut worker.js_runtime, &module.specifier, module.maybe_source.as_ref().unwrap().to_string(), module.media_type, ).await?; + let mut graph_data = self.graph_data.write(); + eprintln!("adding translation {}", module.specifier); + graph_data.add_cjs_esm_translation(&module.specifier, translated_source); } } } @@ -546,15 +548,6 @@ impl ProcState { is_dynamic ); - let mut needs_cjs_esm_translation = false; - if let Some(resolver) = &self.maybe_resolver { - if let Some(referrer) = maybe_referrer { - let response = resolver.resolve(specifier.as_str(), &referrer); - needs_cjs_esm_translation = - matches!(response, ResolveResponse::CommonJs(_)); - } - } - let graph_data = self.graph_data.read(); let found = graph_data.follow_redirect(&specifier); match graph_data.get(&found) { @@ -567,47 +560,8 @@ impl ProcState { | MediaType::Cjs | MediaType::Mjs | MediaType::Json => { - if needs_cjs_esm_translation { - let parsed_source = - deno_ast::parse_script(deno_ast::ParseParams { - specifier: specifier.to_string(), - source: deno_ast::SourceTextInfo::new(code.clone()), - media_type: *media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, - })?; - let analysis = parsed_source.analyze_cjs(); - let mut source = vec![ - r#"import { createRequire } from "node:module";"#.to_string(), - r#"const require = createRequire(import.meta.url);"# - .to_string(), - ]; - source.push(format!( - "const mod = require(\"{}\");", - specifier.to_file_path().unwrap().to_str().unwrap() - )); - source.push("export default mod".to_string()); - for export in - analysis.exports.iter().filter(|e| e.as_str() != "default") - { - // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, - // but it might not be necessary here since our analysis is more detailed? - source - .push(format!("export const {} = mod.{}", export, export)); - } - // TODO(bartlomieju): handle reexports - for (_idx, reexport) in analysis.reexports.iter().enumerate() { - // source.push(format!( - // "const reexport{} = require(\"{}\");", - // idx, reexport - // )); - source.push(format!( - "throw new Error('Unhandled reexport {}')", - reexport - )); - } - source.join("\n").to_string() + if let Some(source) = graph_data.get_cjs_esm_translation(&specifier) { + source.to_owned() } else { code.as_ref().clone() } @@ -749,7 +703,6 @@ impl ProcState { } let translated_source = source.join("\n").to_string(); - eprintln!("translated source: {}", translated_source); Ok(translated_source) } } From 284f4f9529c40a5850a7da08c6b28c9f7cba840a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 4 Feb 2022 17:03:03 +0100 Subject: [PATCH 12/27] working! --- cli/compat/esm_resolver.rs | 9 +-------- cli/compat/mod.rs | 1 - cli/proc_state.rs | 4 ++-- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/cli/compat/esm_resolver.rs b/cli/compat/esm_resolver.rs index 6fbbbe9206ed84..5546f18fbf571d 100644 --- a/cli/compat/esm_resolver.rs +++ b/cli/compat/esm_resolver.rs @@ -219,7 +219,6 @@ fn finalize_resolution( return Ok(resolved); } - eprintln!("resolved {} {}", resolved.as_str(), base.as_str()); let encoded_sep_re = Regex::new(r"%2F|%2C").expect("bad regex"); if encoded_sep_re.is_match(resolved.path()) { @@ -795,10 +794,9 @@ fn package_resolve( base: &ModuleSpecifier, conditions: &[&str], ) -> Result { - eprintln!("specifier {}", specifier); let (package_name, package_subpath, is_scoped) = parse_package_name(specifier, base)?; - eprintln!("package name {} {} {}", package_name, package_subpath, is_scoped); + // ResolveSelf let package_config = get_package_scope_config(base)?; if package_config.exists { @@ -807,7 +805,6 @@ fn package_resolve( if package_config.name.as_ref() == Some(&package_name) { if let Some(exports) = &package_config.exports { if !exports.is_null() { - eprintln!("here1"); return package_exports_resolve( package_json_url, package_subpath, @@ -857,7 +854,6 @@ fn package_resolve( let package_config = get_package_config(package_json_path.clone(), specifier, Some(base))?; if package_config.exports.is_some() { - eprintln!("here2"); return package_exports_resolve( package_json_url, package_subpath, @@ -867,17 +863,14 @@ fn package_resolve( ); } if package_subpath == "." { - eprintln!("here3"); return legacy_main_resolve(&package_json_url, &package_config, base); } - eprintln!("here4 {} {}", package_json_url, package_subpath); return package_json_url .join(&package_subpath) .map_err(AnyError::from); } - eprintln!("erroring!"); Err(errors::err_module_not_found( &package_json_url .join(".") diff --git a/cli/compat/mod.rs b/cli/compat/mod.rs index 072dc81ec29249..1bb873858b0880 100644 --- a/cli/compat/mod.rs +++ b/cli/compat/mod.rs @@ -145,7 +145,6 @@ pub(crate) async fn resolve_cjs_module( referrer_mod: &str, mod_to_resolve: &str, ) -> Result { - eprintln!("resolve_cjs_module {} {}", referrer_mod, mod_to_resolve); let source_code = &format!( r#"(async function resolveCjsModule(main) {{ const CJSModule = await import("{}"); diff --git a/cli/proc_state.rs b/cli/proc_state.rs index a5d7fc59ceb504..f0a7eafccec8b6 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -377,7 +377,7 @@ impl ProcState { for module in graph.modules() { if module.kind == ModuleKind::CommonJs { - eprintln!("cjs module {}", module.specifier); + // eprintln!("cjs module {}", module.specifier); // eprintln!("deps {:#?}", module.dependencies); // eprintln!("maybe source {:#?}", module.maybe_source); let translated_source = self.translate_cjs_to_esm( @@ -387,7 +387,7 @@ impl ProcState { module.media_type, ).await?; let mut graph_data = self.graph_data.write(); - eprintln!("adding translation {}", module.specifier); + // eprintln!("adding translation {}", module.specifier); graph_data.add_cjs_esm_translation(&module.specifier, translated_source); } } From 750fca25b433d3d5eb13bceaee9ef54621fc1515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 4 Feb 2022 17:03:23 +0100 Subject: [PATCH 13/27] fmt --- cli/graph_util.rs | 8 +++++--- cli/proc_state.rs | 51 ++++++++++++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/cli/graph_util.rs b/cli/graph_util.rs index ac30871062ff4c..70abfd1f3f726d 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -255,7 +255,7 @@ impl GraphData { modules, referrer_map, configurations: self.configurations.clone(), - cjs_esm_translations: Default::default() + cjs_esm_translations: Default::default(), }) } @@ -420,13 +420,15 @@ impl GraphData { specifier: &ModuleSpecifier, source: String, ) { - let prev = self.cjs_esm_translations.insert(specifier.to_owned(), source); + let prev = self + .cjs_esm_translations + .insert(specifier.to_owned(), source); assert!(prev.is_none()); } pub(crate) fn get_cjs_esm_translation<'a>( &'a self, - specifier: &ModuleSpecifier + specifier: &ModuleSpecifier, ) -> Option<&'a String> { self.cjs_esm_translations.get(specifier) } diff --git a/cli/proc_state.rs b/cli/proc_state.rs index f0a7eafccec8b6..1362f51264c2f1 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -6,6 +6,7 @@ use crate::compat; use crate::compat::NodeEsmResolver; use crate::config_file::ConfigFile; use crate::config_file::MaybeImportsResult; +use crate::create_main_worker; use crate::deno_dir; use crate::emit; use crate::file_fetcher::get_root_cert_store; @@ -18,7 +19,6 @@ use crate::graph_util::ModuleEntry; use crate::http_cache; use crate::lockfile::as_maybe_locker; use crate::lockfile::Lockfile; -use crate::create_main_worker; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; use crate::source_maps::SourceMapGetter; @@ -358,7 +358,8 @@ impl ProcState { // TODO: create new MainWorker here let mut worker = create_main_worker( self, - deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts").unwrap(), + deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts") + .unwrap(), if is_dynamic { dynamic_permissions } else { @@ -380,15 +381,18 @@ impl ProcState { // eprintln!("cjs module {}", module.specifier); // eprintln!("deps {:#?}", module.dependencies); // eprintln!("maybe source {:#?}", module.maybe_source); - let translated_source = self.translate_cjs_to_esm( - &mut worker.js_runtime, - &module.specifier, - module.maybe_source.as_ref().unwrap().to_string(), - module.media_type, - ).await?; + let translated_source = self + .translate_cjs_to_esm( + &mut worker.js_runtime, + &module.specifier, + module.maybe_source.as_ref().unwrap().to_string(), + module.media_type, + ) + .await?; let mut graph_data = self.graph_data.write(); // eprintln!("adding translation {}", module.specifier); - graph_data.add_cjs_esm_translation(&module.specifier, translated_source); + graph_data + .add_cjs_esm_translation(&module.specifier, translated_source); } } } @@ -560,7 +564,8 @@ impl ProcState { | MediaType::Cjs | MediaType::Mjs | MediaType::Json => { - if let Some(source) = graph_data.get_cjs_esm_translation(&specifier) { + if let Some(source) = graph_data.get_cjs_esm_translation(&specifier) + { source.to_owned() } else { code.as_ref().clone() @@ -654,14 +659,18 @@ impl ProcState { // Firstly, resolve relate reexport specifier let resolved_reexport = compat::resolve_cjs_module( js_runtime, - deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts").unwrap().as_str(), + deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts") + .unwrap() + .as_str(), specifier.to_file_path().unwrap().to_str().unwrap(), reexport.as_str(), - ).await?; + ) + .await?; let reexport_specifier = ModuleSpecifier::from_file_path(&resolved_reexport).unwrap(); // Secondly, read the source code from disk - let reexport_file = self.file_fetcher.get_source(&reexport_specifier).unwrap(); + let reexport_file = + self.file_fetcher.get_source(&reexport_specifier).unwrap(); // Now perform analysis again { @@ -678,14 +687,20 @@ impl ProcState { // TODO: // And recurse again if there are reexports! - source.push( - format!("const reexport{} = require(\"{}\");", idx, reexport) - ); + source.push(format!( + "const reexport{} = require(\"{}\");", + idx, reexport + )); - for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { + for export in + analysis.exports.iter().filter(|e| e.as_str() != "default") + { // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, // but it might not be necessary here since our analysis is more detailed? - source.push(format!("export const {} = reexport{}.{};", export, idx, export)); + source.push(format!( + "export const {} = reexport{}.{};", + export, idx, export + )); } } } From 4da573bc7a9816f8aae9f98e100c2678dfad568e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 4 Feb 2022 17:12:20 +0100 Subject: [PATCH 14/27] lint --- cli/proc_state.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 1362f51264c2f1..b22ab849523de4 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -348,14 +348,12 @@ impl ProcState { let needs_cjs_esm_translation = graph .modules() .iter() - .find(|m| m.kind == ModuleKind::CommonJs) - .is_some(); + .any(|m| m.kind == ModuleKind::CommonJs); if needs_cjs_esm_translation { // TODO(bartlomieju): extremely inefficient to create a new worker, just // to do CJS module resolution, but we currently don't have implementation // of it in Rust - // TODO: create new MainWorker here let mut worker = create_main_worker( self, deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts") @@ -378,9 +376,6 @@ impl ProcState { for module in graph.modules() { if module.kind == ModuleKind::CommonJs { - // eprintln!("cjs module {}", module.specifier); - // eprintln!("deps {:#?}", module.dependencies); - // eprintln!("maybe source {:#?}", module.maybe_source); let translated_source = self .translate_cjs_to_esm( &mut worker.js_runtime, @@ -390,7 +385,6 @@ impl ProcState { ) .await?; let mut graph_data = self.graph_data.write(); - // eprintln!("adding translation {}", module.specifier); graph_data .add_cjs_esm_translation(&module.specifier, translated_source); } @@ -717,7 +711,7 @@ impl ProcState { source.push(format!("export const {} = mod.{};", export, export)); } - let translated_source = source.join("\n").to_string(); + let translated_source = source.join("\n"); Ok(translated_source) } } From daaa9ab9e9f177595443be3c11683cd06c7381c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 5 Feb 2022 15:05:02 +0100 Subject: [PATCH 15/27] simplify test --- imported.js | 1 - 1 file changed, 1 deletion(-) diff --git a/imported.js b/imported.js index 6ee5eb0837e8b2..49ab4c7829deaf 100644 --- a/imported.js +++ b/imported.js @@ -7,4 +7,3 @@ exports.bar = "bar"; exports.fizz = require("./reexports.js"); console.log(exports); -console.log(exports.fizz); From cf9142c57d94533676122a369701db4f490117ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 5 Feb 2022 15:25:13 +0100 Subject: [PATCH 16/27] move files to cli/tests/ --- .../tests/testdata/compat/import_cjs_from_esm/imported.js | 0 .../tests/testdata/compat/import_cjs_from_esm/imported.mjs | 0 .../tests/testdata/compat/import_cjs_from_esm/main.mjs | 0 .../tests/testdata/compat/import_cjs_from_esm/reexports.js | 0 .../tests/testdata/compat/import_cjs_from_esm/reexports2.js | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename imported.js => cli/tests/testdata/compat/import_cjs_from_esm/imported.js (100%) rename imported.mjs => cli/tests/testdata/compat/import_cjs_from_esm/imported.mjs (100%) rename main.mjs => cli/tests/testdata/compat/import_cjs_from_esm/main.mjs (100%) rename reexports.js => cli/tests/testdata/compat/import_cjs_from_esm/reexports.js (100%) rename reexports2.js => cli/tests/testdata/compat/import_cjs_from_esm/reexports2.js (100%) diff --git a/imported.js b/cli/tests/testdata/compat/import_cjs_from_esm/imported.js similarity index 100% rename from imported.js rename to cli/tests/testdata/compat/import_cjs_from_esm/imported.js diff --git a/imported.mjs b/cli/tests/testdata/compat/import_cjs_from_esm/imported.mjs similarity index 100% rename from imported.mjs rename to cli/tests/testdata/compat/import_cjs_from_esm/imported.mjs diff --git a/main.mjs b/cli/tests/testdata/compat/import_cjs_from_esm/main.mjs similarity index 100% rename from main.mjs rename to cli/tests/testdata/compat/import_cjs_from_esm/main.mjs diff --git a/reexports.js b/cli/tests/testdata/compat/import_cjs_from_esm/reexports.js similarity index 100% rename from reexports.js rename to cli/tests/testdata/compat/import_cjs_from_esm/reexports.js diff --git a/reexports2.js b/cli/tests/testdata/compat/import_cjs_from_esm/reexports2.js similarity index 100% rename from reexports2.js rename to cli/tests/testdata/compat/import_cjs_from_esm/reexports2.js From f31d9b25936ca7c64a68ad61adb1824467ab5da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 5 Feb 2022 15:28:03 +0100 Subject: [PATCH 17/27] remove not needed file --- .../testdata/compat/import_cjs_from_esm/imported.mjs | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 cli/tests/testdata/compat/import_cjs_from_esm/imported.mjs diff --git a/cli/tests/testdata/compat/import_cjs_from_esm/imported.mjs b/cli/tests/testdata/compat/import_cjs_from_esm/imported.mjs deleted file mode 100644 index cf47f584f2399e..00000000000000 --- a/cli/tests/testdata/compat/import_cjs_from_esm/imported.mjs +++ /dev/null @@ -1,8 +0,0 @@ -import { createRequire } from "node:module"; -const require = createRequire(import.meta.url); -const mod = require("./imported.js"); - -export default mod; -export const foo = mod.foo; -export const bar = mod.bar; -export const fizz = mod.fizz; \ No newline at end of file From 00fc975b618a4fa532e3085a1596a9537bea4061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 15 Feb 2022 03:12:47 +0100 Subject: [PATCH 18/27] use node_resolve crate --- Cargo.lock | 10 ++++++++++ cli/Cargo.toml | 1 + cli/proc_state.rs | 47 ++++++++++------------------------------------- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49d478a00be609..8e55db4d4179a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -753,6 +753,7 @@ dependencies = [ "log", "lspower", "nix", + "node_resolve", "notify", "num_cpus", "once_cell", @@ -2428,6 +2429,15 @@ dependencies = [ "memoffset", ] +[[package]] +name = "node_resolve" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", +] + [[package]] name = "notify" version = "5.0.0-pre.12" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index d4802a40080a1c..058bdfde630689 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -73,6 +73,7 @@ jsonc-parser = { version = "=0.19.0", features = ["serde"] } libc = "=0.2.106" log = { version = "=0.4.14", features = ["serde"] } lspower = "=1.4.0" +node_resolve = { version = "0.1.0", path = "../../node_resolve" } notify = "=5.0.0-pre.12" num_cpus = "=1.13.0" once_cell = "=1.9.0" diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 5c0022d943f348..42b6130888034f 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -6,7 +6,6 @@ use crate::compat; use crate::compat::NodeEsmResolver; use crate::config_file::ConfigFile; use crate::config_file::MaybeImportsResult; -use crate::create_main_worker; use crate::deno_dir; use crate::emit; use crate::file_fetcher::get_root_cert_store; @@ -35,7 +34,6 @@ use deno_core::parking_lot::RwLock; use deno_core::resolve_url; use deno_core::url::Url; use deno_core::CompiledWasmModuleStore; -use deno_core::JsRuntime; use deno_core::ModuleSource; use deno_core::ModuleSpecifier; use deno_core::ModuleType; @@ -352,34 +350,11 @@ impl ProcState { .any(|m| m.kind == ModuleKind::CommonJs); if needs_cjs_esm_translation { - // TODO(bartlomieju): extremely inefficient to create a new worker, just - // to do CJS module resolution, but we currently don't have implementation - // of it in Rust - let mut worker = create_main_worker( - self, - deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts") - .unwrap(), - if is_dynamic { - dynamic_permissions - } else { - root_permissions - }, - vec![], - ); - // Set up Node globals - worker.execute_side_module(&compat::GLOBAL_URL).await?; - // And `module` module that we'll use for checking which - // loader to use and potentially load CJS module with. - // This allows to skip permission check for `--allow-net` - // which would otherwise be requested by dynamically importing - // this file. - worker.execute_side_module(&compat::MODULE_URL).await?; - for module in graph.modules() { if module.kind == ModuleKind::CommonJs { + eprintln!("doing translation {}", module.specifier); let translated_source = self .translate_cjs_to_esm( - &mut worker.js_runtime, &module.specifier, module.maybe_source.as_ref().unwrap().to_string(), module.media_type, @@ -628,7 +603,6 @@ impl ProcState { /// If successful a source code for equivalent ES module is returned. async fn translate_cjs_to_esm( &self, - js_runtime: &mut JsRuntime, specifier: &ModuleSpecifier, // TODO(bartlomieju): could use `maybe_parsed_source` if available code: String, @@ -644,6 +618,8 @@ impl ProcState { })?; let analysis = parsed_source.analyze_cjs(); + eprintln!("parsed {:#?}", analysis); + let mut source = vec![ r#"import { createRequire } from "node:module";"#.to_string(), r#"const require = createRequire(import.meta.url);"#.to_string(), @@ -652,15 +628,12 @@ impl ProcState { // if there are reexports, handle them first for (idx, reexport) in analysis.reexports.iter().enumerate() { // Firstly, resolve relate reexport specifier - let resolved_reexport = compat::resolve_cjs_module( - js_runtime, - deno_core::resolve_url_or_path("./$deno$cjs_esm_translation.ts") - .unwrap() - .as_str(), - specifier.to_file_path().unwrap().to_str().unwrap(), - reexport.as_str(), - ) - .await?; + let resolved_reexport = node_resolve::node_resolve( + reexport, + &specifier.to_file_path().unwrap(), + // FIXME(bartlomieju): check what should be proper conditions + &["deno", "require", "default"], + )?; let reexport_specifier = ModuleSpecifier::from_file_path(&resolved_reexport).unwrap(); // Secondly, read the source code from disk @@ -670,7 +643,7 @@ impl ProcState { { let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { - specifier: resolved_reexport, + specifier: reexport_specifier.to_string(), source: deno_ast::SourceTextInfo::new(reexport_file.source), media_type: reexport_file.media_type, capture_tokens: true, From 4f049e8a505c86284a02c6f4b4e10b4611fe620c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Tue, 15 Feb 2022 04:24:38 +0100 Subject: [PATCH 19/27] remove debug log --- cli/proc_state.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 42b6130888034f..910691e765110e 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -352,7 +352,6 @@ impl ProcState { if needs_cjs_esm_translation { for module in graph.modules() { if module.kind == ModuleKind::CommonJs { - eprintln!("doing translation {}", module.specifier); let translated_source = self .translate_cjs_to_esm( &module.specifier, @@ -618,8 +617,6 @@ impl ProcState { })?; let analysis = parsed_source.analyze_cjs(); - eprintln!("parsed {:#?}", analysis); - let mut source = vec![ r#"import { createRequire } from "node:module";"#.to_string(), r#"const require = createRequire(import.meta.url);"#.to_string(), From bf63bab9f24616c08de5398e4c40918c75cc2fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 24 Feb 2022 14:24:06 +0100 Subject: [PATCH 20/27] use node_resolver --- Cargo.lock | 6 ++++-- cli/Cargo.toml | 2 +- cli/compat/mod.rs | 36 ------------------------------------ cli/proc_state.rs | 2 +- 4 files changed, 6 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05076172c6465b..3ef04ee618c758 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,7 +763,7 @@ dependencies = [ "log", "lspower", "nix", - "node_resolve", + "node_resolver", "notify", "num_cpus", "once_cell", @@ -2460,8 +2460,10 @@ dependencies = [ ] [[package]] -name = "node_resolve" +name = "node_resolver" version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35ed1604f6f4e33b51926d6be3bc09423f35eec776165c460c313dc2970ea5a" dependencies = [ "anyhow", "serde", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index cdad390908ce6a..1076951f748093 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -74,7 +74,7 @@ jsonc-parser = { version = "=0.19.0", features = ["serde"] } libc = "=0.2.106" log = { version = "=0.4.14", features = ["serde"] } lspower = "=1.4.0" -node_resolve = { version = "0.1.0", path = "../../node_resolve" } +node_resolver = "0.1.0" notify = "=5.0.0-pre.12" num_cpus = "=1.13.0" once_cell = "=1.9.0" diff --git a/cli/compat/mod.rs b/cli/compat/mod.rs index 5cda37f8d8d822..375edc9a774155 100644 --- a/cli/compat/mod.rs +++ b/cli/compat/mod.rs @@ -5,9 +5,7 @@ mod esm_resolver; use deno_core::error::AnyError; use deno_core::located_script_name; -use deno_core::serde_v8; use deno_core::url::Url; -use deno_core::v8; use deno_core::JsRuntime; use once_cell::sync::Lazy; @@ -139,40 +137,6 @@ pub(crate) fn add_global_require( Ok(()) } -pub(crate) async fn resolve_cjs_module( - js_runtime: &mut JsRuntime, - fake_main: &str, - referrer_mod: &str, - mod_to_resolve: &str, -) -> Result { - let source_code = &format!( - r#"(async function resolveCjsModule(main) {{ - const CJSModule = await import("{}"); - const require = CJSModule.createRequire(main); - require("{}"); - const referrerMod = CJSModule.default._cache["{}"]; - const resolvedMod = CJSModule.default._resolveFilename("{}", referrerMod); - return resolvedMod; - }})('{}'); - "#, - MODULE_URL_STR.as_str(), - escape_for_single_quote_string(referrer_mod), - escape_for_single_quote_string(referrer_mod), - escape_for_single_quote_string(mod_to_resolve), - fake_main, - ); - - let value = - js_runtime.execute_script(&located_script_name!(), source_code)?; - let value = js_runtime.resolve_value(value).await?; - let resolved_specifier = { - let scope = &mut js_runtime.handle_scope(); - let local = v8::Local::::new(scope, value); - serde_v8::from_v8(scope, local)? - }; - Ok(resolved_specifier) -} - fn escape_for_single_quote_string(text: &str) -> String { text.replace(r"\", r"\\").replace("'", r"\'") } diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 5a87c88a4701ff..168d650bb5ed94 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -625,7 +625,7 @@ impl ProcState { // if there are reexports, handle them first for (idx, reexport) in analysis.reexports.iter().enumerate() { // Firstly, resolve relate reexport specifier - let resolved_reexport = node_resolve::node_resolve( + let resolved_reexport = node_resolver::node_resolve( reexport, &specifier.to_file_path().unwrap(), // FIXME(bartlomieju): check what should be proper conditions From 46b97b427ead9c2572b208de45b99b7d599a9cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 25 Feb 2022 00:31:28 +0100 Subject: [PATCH 21/27] update std submodule --- test_util/std | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_util/std b/test_util/std index efa94f2ccc2df5..97dc90373893b8 160000 --- a/test_util/std +++ b/test_util/std @@ -1 +1 @@ -Subproject commit efa94f2ccc2df51f9437e14bc5e3455b013b99b9 +Subproject commit 97dc90373893b83713b05b1891a109ad48f8abc2 From fc827bb4824235426322115168b31fa5fd069478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 26 Feb 2022 15:05:50 +0100 Subject: [PATCH 22/27] add test --- cli/graph_util.rs | 3 +++ cli/proc_state.rs | 10 +++++++--- cli/tests/integration/compat_tests.rs | 6 ++++++ cli/tests/testdata/compat/import_cjs_from_esm.out | 7 +++++++ 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 cli/tests/testdata/compat/import_cjs_from_esm.out diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 70abfd1f3f726d..4b01f54e0b3b28 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -415,6 +415,9 @@ impl GraphData { self.modules.get(specifier) } + // TODO(bartlomieju): after saving translated source + // it's never removed, potentially leading to excessive + // memory consumption pub(crate) fn add_cjs_esm_translation( &mut self, specifier: &ModuleSpecifier, diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 35ff4c6b5beca6..a4229d1f0e9ac6 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -348,6 +348,11 @@ impl ProcState { if needs_cjs_esm_translation { for module in graph.modules() { + // TODO(bartlomieju): this is overly simplistic heuristic, once we are + // in compat mode, all files ending with plain `.js` extension are + // considered CommonJs modules. Which leads to situation where valid + // ESM modules with `.js` extension might undergo translation (it won't + // work in this situation). if module.kind == ModuleKind::CommonJs { let translated_source = self .translate_cjs_to_esm( @@ -600,7 +605,6 @@ impl ProcState { async fn translate_cjs_to_esm( &self, specifier: &ModuleSpecifier, - // TODO(bartlomieju): could use `maybe_parsed_source` if available code: String, media_type: MediaType, ) -> Result { @@ -625,7 +629,8 @@ impl ProcState { let resolved_reexport = node_resolver::node_resolve( reexport, &specifier.to_file_path().unwrap(), - // FIXME(bartlomieju): check what should be proper conditions + // FIXME(bartlomieju): check if these conditions are okay, probably + // should be `deno-require`, because `deno` is already used in `esm_resolver.rs` &["deno", "require", "default"], )?; let reexport_specifier = @@ -634,7 +639,6 @@ impl ProcState { let reexport_file = self.file_fetcher.get_source(&reexport_specifier).unwrap(); // Now perform analysis again - { let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { specifier: reexport_specifier.to_string(), diff --git a/cli/tests/integration/compat_tests.rs b/cli/tests/integration/compat_tests.rs index 189e1eb41d2d91..b83ce06c22b042 100644 --- a/cli/tests/integration/compat_tests.rs +++ b/cli/tests/integration/compat_tests.rs @@ -95,6 +95,12 @@ itest!(compat_worker { output: "compat/worker/worker_test.out", }); +itest!(cjs_esm_interop { + args: + "run --compat --unstable -A --no-check compat/import_cjs_from_esm/main.mjs", + output: "compat/import_cjs_from_esm.out", +}); + #[test] fn globals_in_repl() { let (out, _err) = util::run_and_collect_output_with_args( diff --git a/cli/tests/testdata/compat/import_cjs_from_esm.out b/cli/tests/testdata/compat/import_cjs_from_esm.out new file mode 100644 index 00000000000000..0906d33650242b --- /dev/null +++ b/cli/tests/testdata/compat/import_cjs_from_esm.out @@ -0,0 +1,7 @@ +{ + a: "A", + b: "B", + foo: "foo", + bar: "bar", + fizz: { buzz: { buzz: "buzz", fizz: "FIZZ" } } +} From 9e75822e8f2ee4a894b0878de4b3762b103b06f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 26 Feb 2022 15:07:46 +0100 Subject: [PATCH 23/27] update test --- cli/proc_state.rs | 3 --- cli/tests/testdata/compat/import_cjs_from_esm.out | 8 +------- .../testdata/compat/import_cjs_from_esm/reexports.js | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/cli/proc_state.rs b/cli/proc_state.rs index a4229d1f0e9ac6..036e9a691343b4 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -650,9 +650,6 @@ impl ProcState { })?; let analysis = parsed_source.analyze_cjs(); - // TODO: - // And recurse again if there are reexports! - source.push(format!( "const reexport{} = require(\"{}\");", idx, reexport diff --git a/cli/tests/testdata/compat/import_cjs_from_esm.out b/cli/tests/testdata/compat/import_cjs_from_esm.out index 0906d33650242b..1e53e564f1d0a8 100644 --- a/cli/tests/testdata/compat/import_cjs_from_esm.out +++ b/cli/tests/testdata/compat/import_cjs_from_esm.out @@ -1,7 +1 @@ -{ - a: "A", - b: "B", - foo: "foo", - bar: "bar", - fizz: { buzz: { buzz: "buzz", fizz: "FIZZ" } } -} +{ a: "A", b: "B", foo: "foo", bar: "bar", fizz: { buzz: "buzz", fizz: "FIZZ" } } \ No newline at end of file diff --git a/cli/tests/testdata/compat/import_cjs_from_esm/reexports.js b/cli/tests/testdata/compat/import_cjs_from_esm/reexports.js index 7a0e8843e22867..62edb770854c99 100644 --- a/cli/tests/testdata/compat/import_cjs_from_esm/reexports.js +++ b/cli/tests/testdata/compat/import_cjs_from_esm/reexports.js @@ -1 +1 @@ -module.exports.buzz = require("./reexports2.js"); +module.exports = require("./reexports2.js"); From 7ea9d726c46ff1b121b7bf5bfcee87cd59f324e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 26 Feb 2022 15:20:42 +0100 Subject: [PATCH 24/27] add missing newline :D --- cli/tests/testdata/compat/import_cjs_from_esm.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/tests/testdata/compat/import_cjs_from_esm.out b/cli/tests/testdata/compat/import_cjs_from_esm.out index 1e53e564f1d0a8..ffaa5e40669c3e 100644 --- a/cli/tests/testdata/compat/import_cjs_from_esm.out +++ b/cli/tests/testdata/compat/import_cjs_from_esm.out @@ -1 +1 @@ -{ a: "A", b: "B", foo: "foo", bar: "bar", fizz: { buzz: "buzz", fizz: "FIZZ" } } \ No newline at end of file +{ a: "A", b: "B", foo: "foo", bar: "bar", fizz: { buzz: "buzz", fizz: "FIZZ" } } From 604289f45ba26c07ed7931ecb876cab9ab72062d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 26 Feb 2022 15:30:27 +0100 Subject: [PATCH 25/27] use --queit --- cli/tests/integration/compat_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/tests/integration/compat_tests.rs b/cli/tests/integration/compat_tests.rs index b83ce06c22b042..c8fc1c0a058341 100644 --- a/cli/tests/integration/compat_tests.rs +++ b/cli/tests/integration/compat_tests.rs @@ -97,7 +97,7 @@ itest!(compat_worker { itest!(cjs_esm_interop { args: - "run --compat --unstable -A --no-check compat/import_cjs_from_esm/main.mjs", + "run --compat --unstable -A --quiet --no-check compat/import_cjs_from_esm/main.mjs", output: "compat/import_cjs_from_esm.out", }); From cbcf804dd9a90edd0324df0df175d87987c123d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 26 Feb 2022 17:20:53 +0100 Subject: [PATCH 26/27] escape file path --- cli/proc_state.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 036e9a691343b4..ce6a8d9c91f42e 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -670,7 +670,14 @@ impl ProcState { source.push(format!( "const mod = require(\"{}\");", - specifier.to_file_path().unwrap().to_str().unwrap() + specifier + .to_file_path() + .unwrap() + .to_str() + .unwrap() + .replace('\\', "\\\\") + .replace('\'', "\\\'") + .replace('\"', "\\\"") )); source.push("export default mod".to_string()); From 8b792b12ba76af5fc05d5791d22e456df5bd669c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sun, 27 Feb 2022 14:13:18 +0100 Subject: [PATCH 27/27] move to compat/mod.rs --- cli/compat/mod.rs | 97 +++++++++++++++++++++++++++++++++++++++++ cli/proc_state.rs | 109 +++------------------------------------------- 2 files changed, 104 insertions(+), 102 deletions(-) diff --git a/cli/compat/mod.rs b/cli/compat/mod.rs index 0c30a58fc8882e..0f1c36084008be 100644 --- a/cli/compat/mod.rs +++ b/cli/compat/mod.rs @@ -3,11 +3,15 @@ mod errors; mod esm_resolver; +use crate::file_fetcher::FileFetcher; +use deno_ast::MediaType; use deno_core::error::AnyError; use deno_core::located_script_name; use deno_core::url::Url; use deno_core::JsRuntime; +use deno_core::ModuleSpecifier; use once_cell::sync::Lazy; +use std::sync::Arc; pub use esm_resolver::check_if_should_use_esm_loader; pub(crate) use esm_resolver::NodeEsmResolver; @@ -155,3 +159,96 @@ pub fn setup_builtin_modules( js_runtime.execute_script("setup_node_builtins.js", &script)?; Ok(()) } + +/// Translates given CJS module into ESM. This function will perform static +/// analysis on the file to find defined exports and reexports. +/// +/// For all discovered reexports the analysis will be performed recursively. +/// +/// If successful a source code for equivalent ES module is returned. +pub async fn translate_cjs_to_esm( + file_fetcher: &FileFetcher, + specifier: &ModuleSpecifier, + code: String, + media_type: MediaType, +) -> Result { + let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { + specifier: specifier.to_string(), + source: deno_ast::SourceTextInfo::new(Arc::new(code)), + media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + })?; + let analysis = parsed_source.analyze_cjs(); + + let mut source = vec![ + r#"import { createRequire } from "node:module";"#.to_string(), + r#"const require = createRequire(import.meta.url);"#.to_string(), + ]; + + // if there are reexports, handle them first + for (idx, reexport) in analysis.reexports.iter().enumerate() { + // Firstly, resolve relate reexport specifier + let resolved_reexport = node_resolver::node_resolve( + reexport, + &specifier.to_file_path().unwrap(), + // FIXME(bartlomieju): check if these conditions are okay, probably + // should be `deno-require`, because `deno` is already used in `esm_resolver.rs` + &["deno", "require", "default"], + )?; + let reexport_specifier = + ModuleSpecifier::from_file_path(&resolved_reexport).unwrap(); + // Secondly, read the source code from disk + let reexport_file = file_fetcher.get_source(&reexport_specifier).unwrap(); + // Now perform analysis again + { + let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { + specifier: reexport_specifier.to_string(), + source: deno_ast::SourceTextInfo::new(reexport_file.source), + media_type: reexport_file.media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + })?; + let analysis = parsed_source.analyze_cjs(); + + source.push(format!( + "const reexport{} = require(\"{}\");", + idx, reexport + )); + + for export in analysis.exports.iter().filter(|e| e.as_str() != "default") + { + // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, + // but it might not be necessary here since our analysis is more detailed? + source.push(format!( + "export const {} = reexport{}.{};", + export, idx, export + )); + } + } + } + + source.push(format!( + "const mod = require(\"{}\");", + specifier + .to_file_path() + .unwrap() + .to_str() + .unwrap() + .replace('\\', "\\\\") + .replace('\'', "\\\'") + .replace('\"', "\\\"") + )); + source.push("export default mod".to_string()); + + for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { + // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, + // but it might not be necessary here since our analysis is more detailed? + source.push(format!("export const {} = mod.{};", export, export)); + } + + let translated_source = source.join("\n"); + Ok(translated_source) +} diff --git a/cli/proc_state.rs b/cli/proc_state.rs index ce6a8d9c91f42e..8933cb9867f3bd 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -354,13 +354,13 @@ impl ProcState { // ESM modules with `.js` extension might undergo translation (it won't // work in this situation). if module.kind == ModuleKind::CommonJs { - let translated_source = self - .translate_cjs_to_esm( - &module.specifier, - module.maybe_source.as_ref().unwrap().to_string(), - module.media_type, - ) - .await?; + let translated_source = compat::translate_cjs_to_esm( + &self.file_fetcher, + &module.specifier, + module.maybe_source.as_ref().unwrap().to_string(), + module.media_type, + ) + .await?; let mut graph_data = self.graph_data.write(); graph_data .add_cjs_esm_translation(&module.specifier, translated_source); @@ -595,101 +595,6 @@ impl ProcState { None } } - - /// Translates given CJS module into ESM. This function will perform static - /// analysis on the file to find defined exports and reexports. - /// - /// For all discovered reexports the analysis will be performed recursively. - /// - /// If successful a source code for equivalent ES module is returned. - async fn translate_cjs_to_esm( - &self, - specifier: &ModuleSpecifier, - code: String, - media_type: MediaType, - ) -> Result { - let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { - specifier: specifier.to_string(), - source: deno_ast::SourceTextInfo::new(Arc::new(code)), - media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, - })?; - let analysis = parsed_source.analyze_cjs(); - - let mut source = vec![ - r#"import { createRequire } from "node:module";"#.to_string(), - r#"const require = createRequire(import.meta.url);"#.to_string(), - ]; - - // if there are reexports, handle them first - for (idx, reexport) in analysis.reexports.iter().enumerate() { - // Firstly, resolve relate reexport specifier - let resolved_reexport = node_resolver::node_resolve( - reexport, - &specifier.to_file_path().unwrap(), - // FIXME(bartlomieju): check if these conditions are okay, probably - // should be `deno-require`, because `deno` is already used in `esm_resolver.rs` - &["deno", "require", "default"], - )?; - let reexport_specifier = - ModuleSpecifier::from_file_path(&resolved_reexport).unwrap(); - // Secondly, read the source code from disk - let reexport_file = - self.file_fetcher.get_source(&reexport_specifier).unwrap(); - // Now perform analysis again - { - let parsed_source = deno_ast::parse_script(deno_ast::ParseParams { - specifier: reexport_specifier.to_string(), - source: deno_ast::SourceTextInfo::new(reexport_file.source), - media_type: reexport_file.media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, - })?; - let analysis = parsed_source.analyze_cjs(); - - source.push(format!( - "const reexport{} = require(\"{}\");", - idx, reexport - )); - - for export in - analysis.exports.iter().filter(|e| e.as_str() != "default") - { - // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, - // but it might not be necessary here since our analysis is more detailed? - source.push(format!( - "export const {} = reexport{}.{};", - export, idx, export - )); - } - } - } - - source.push(format!( - "const mod = require(\"{}\");", - specifier - .to_file_path() - .unwrap() - .to_str() - .unwrap() - .replace('\\', "\\\\") - .replace('\'', "\\\'") - .replace('\"', "\\\"") - )); - source.push("export default mod".to_string()); - - for export in analysis.exports.iter().filter(|e| e.as_str() != "default") { - // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object, - // but it might not be necessary here since our analysis is more detailed? - source.push(format!("export const {} = mod.{};", export, export)); - } - - let translated_source = source.join("\n"); - Ok(translated_source) - } } // TODO(@kitsonk) this is only temporary, but should be refactored to somewhere