feat(compat): CJS/ESM interoperability

cli/Cargo.toml

@@ -45,7 +45,7 @@ winapi = "=0.3.9"
 winres = "=0.1.11"
 
 [dependencies]
-deno_ast = { version = "0.11.0", features = ["bundler", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] }
+deno_ast = { version = "0.11.0", features = ["bundler", "cjs", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "transpiling", "typescript", "view", "visit"] }
 deno_core = { version = "0.118.0", path = "../core" }
 deno_doc = "0.29.0"
 deno_graph = "0.22.0"
@@ -73,6 +73,7 @@ jsonc-parser = { version = "=0.19.0", features = ["serde"] }
 libc = "=0.2.106"
 log = { version = "=0.4.14", features = ["serde"] }
 lspower = "=1.4.0"
+node_resolve = { version = "0.1.0", path = "../../node_resolve" }
 notify = "=5.0.0-pre.12"
 num_cpus = "=1.13.0"
 once_cell = "=1.9.0"

cli/compat/mod.rs

@@ -5,7 +5,9 @@ mod esm_resolver;
 
 use deno_core::error::AnyError;
 use deno_core::located_script_name;
+use deno_core::serde_v8;
 use deno_core::url::Url;
+use deno_core::v8;
 use deno_core::JsRuntime;
 use once_cell::sync::Lazy;
 
@@ -137,6 +139,40 @@ pub(crate) fn add_global_require(
   Ok(())
 }
 
+pub(crate) async fn resolve_cjs_module(
+  js_runtime: &mut JsRuntime,
+  fake_main: &str,
+  referrer_mod: &str,
+  mod_to_resolve: &str,
+) -> Result<String, AnyError> {
+  let source_code = &format!(
+    r#"(async function resolveCjsModule(main) {{
+      const CJSModule = await import("{}");
+      const require = CJSModule.createRequire(main);
+      require("{}");
+      const referrerMod = CJSModule.default._cache["{}"];
+      const resolvedMod = CJSModule.default._resolveFilename("{}", referrerMod);
+      return resolvedMod;
+    }})('{}');
+    "#,
+    MODULE_URL_STR.as_str(),
+    escape_for_single_quote_string(referrer_mod),
+    escape_for_single_quote_string(referrer_mod),
+    escape_for_single_quote_string(mod_to_resolve),
+    fake_main,
+  );
+
+  let value =
+    js_runtime.execute_script(&located_script_name!(), source_code)?;
+  let value = js_runtime.resolve_value(value).await?;
+  let resolved_specifier = {
+    let scope = &mut js_runtime.handle_scope();
+    let local = v8::Local::<v8::Value>::new(scope, value);
+    serde_v8::from_v8(scope, local)?
+  };
+  Ok(resolved_specifier)
+}
+
 fn escape_for_single_quote_string(text: &str) -> String {
   text.replace(r"\", r"\\").replace("'", r"\'")
 }

cli/graph_util.rs

@@ -53,6 +53,7 @@ pub(crate) struct GraphData {
   /// error messages.
   referrer_map: HashMap<ModuleSpecifier, Range>,
   configurations: HashSet<ModuleSpecifier>,
+  cjs_esm_translations: HashMap<ModuleSpecifier, String>,
 }
 
 impl GraphData {
@@ -254,6 +255,7 @@ impl GraphData {
       modules,
       referrer_map,
       configurations: self.configurations.clone(),
+      cjs_esm_translations: Default::default(),
     })
   }
 
@@ -412,6 +414,24 @@ impl GraphData {
   ) -> Option<&'a ModuleEntry> {
     self.modules.get(specifier)
   }
+
+  pub(crate) fn add_cjs_esm_translation(
+    &mut self,
+    specifier: &ModuleSpecifier,
+    source: String,
+  ) {
+    let prev = self
+      .cjs_esm_translations
+      .insert(specifier.to_owned(), source);
+    assert!(prev.is_none());
+  }
+
+  pub(crate) fn get_cjs_esm_translation<'a>(
+    &'a self,
+    specifier: &ModuleSpecifier,
+  ) -> Option<&'a String> {
+    self.cjs_esm_translations.get(specifier)
+  }
 }
 
 impl From<&ModuleGraph> for GraphData {

cli/proc_state.rs

@@ -343,6 +343,30 @@ impl ProcState {
       None,
     )
     .await;
+
+    let needs_cjs_esm_translation = graph
+      .modules()
+      .iter()
+      .any(|m| m.kind == ModuleKind::CommonJs);
+
+    if needs_cjs_esm_translation {
+      for module in graph.modules() {
+        if module.kind == ModuleKind::CommonJs {
+          eprintln!("doing translation {}", module.specifier);
+          let translated_source = self
+            .translate_cjs_to_esm(
+              &module.specifier,
+              module.maybe_source.as_ref().unwrap().to_string(),
+              module.media_type,
+            )
+            .await?;
+          let mut graph_data = self.graph_data.write();
+          graph_data
+            .add_cjs_esm_translation(&module.specifier, translated_source);
+        }
+      }
+    }
+
     // If there was a locker, validate the integrity of all the modules in the
     // locker.
     graph_lock_or_exit(&graph);
@@ -509,7 +533,14 @@ impl ProcState {
           | MediaType::Unknown
           | MediaType::Cjs
           | MediaType::Mjs
-          | MediaType::Json => code.as_ref().clone(),
+          | MediaType::Json => {
+            if let Some(source) = graph_data.get_cjs_esm_translation(&specifier)
+            {
+              source.to_owned()
+            } else {
+              code.as_ref().clone()
+            }
+          }
           MediaType::Dts => "".to_string(),
           _ => {
             let emit_path = self
@@ -563,6 +594,100 @@ impl ProcState {
       None
     }
   }
+
+  /// Translates given CJS module into ESM. This function will perform static
+  /// analysis on the file to find defined exports and reexports.
+  ///
+  /// For all discovered reexports the analysis will be performed recursively.
+  ///
+  /// If successful a source code for equivalent ES module is returned.
+  async fn translate_cjs_to_esm(
+    &self,
+    specifier: &ModuleSpecifier,
+    // TODO(bartlomieju): could use `maybe_parsed_source` if available
+    code: String,
+    media_type: MediaType,
+  ) -> Result<String, AnyError> {
+    let parsed_source = deno_ast::parse_script(deno_ast::ParseParams {
+      specifier: specifier.to_string(),
+      source: deno_ast::SourceTextInfo::new(Arc::new(code)),
+      media_type,
+      capture_tokens: true,
+      scope_analysis: false,
+      maybe_syntax: None,
+    })?;
+    let analysis = parsed_source.analyze_cjs();
+
+    eprintln!("parsed {:#?}", analysis);
+
+    let mut source = vec![
+      r#"import { createRequire } from "node:module";"#.to_string(),
+      r#"const require = createRequire(import.meta.url);"#.to_string(),
+    ];
+
+    // if there are reexports, handle them first
+    for (idx, reexport) in analysis.reexports.iter().enumerate() {
+      // Firstly, resolve relate reexport specifier
+      let resolved_reexport = node_resolve::node_resolve(
+        reexport,
+        &specifier.to_file_path().unwrap(),
+        // FIXME(bartlomieju): check what should be proper conditions
+        &["deno", "require", "default"],
+      )?;
+      let reexport_specifier =
+        ModuleSpecifier::from_file_path(&resolved_reexport).unwrap();
+      // Secondly, read the source code from disk
+      let reexport_file =
+        self.file_fetcher.get_source(&reexport_specifier).unwrap();
+      // Now perform analysis again
+
+      {
+        let parsed_source = deno_ast::parse_script(deno_ast::ParseParams {
+          specifier: reexport_specifier.to_string(),
+          source: deno_ast::SourceTextInfo::new(reexport_file.source),
+          media_type: reexport_file.media_type,
+          capture_tokens: true,
+          scope_analysis: false,
+          maybe_syntax: None,
+        })?;
+        let analysis = parsed_source.analyze_cjs();
+
+        // TODO:
+        // And recurse again if there are reexports!
+
+        source.push(format!(
+          "const reexport{} = require(\"{}\");",
+          idx, reexport
+        ));
+
+        for export in
+          analysis.exports.iter().filter(|e| e.as_str() != "default")
+        {
+          // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object,
+          // but it might not be necessary here since our analysis is more detailed?
+          source.push(format!(
+            "export const {} = reexport{}.{};",
+            export, idx, export
+          ));
+        }
+      }
+    }
+
+    source.push(format!(
+      "const mod = require(\"{}\");",
+      specifier.to_file_path().unwrap().to_str().unwrap()
+    ));
+    source.push("export default mod".to_string());
+
+    for export in analysis.exports.iter().filter(|e| e.as_str() != "default") {
+      // TODO(bartlomieju): Node actually checks if a given export exists in `exports` object,
+      // but it might not be necessary here since our analysis is more detailed?
+      source.push(format!("export const {} = mod.{};", export, export));
+    }
+
+    let translated_source = source.join("\n");
+    Ok(translated_source)
+  }
 }
 
 // TODO(@kitsonk) this is only temporary, but should be refactored to somewhere

cli/tests/testdata/compat/import_cjs_from_esm/imported.js

@@ -0,0 +1,9 @@
+exports = {
+  a: "A",
+  b: "B",
+};
+exports.foo = "foo";
+exports.bar = "bar";
+exports.fizz = require("./reexports.js");
+
+console.log(exports);

cli/tests/testdata/compat/import_cjs_from_esm/main.mjs

@@ -0,0 +1 @@
+import "./imported.js";

cli/tests/testdata/compat/import_cjs_from_esm/reexports.js

@@ -0,0 +1 @@
+module.exports.buzz = require("./reexports2.js");

cli/tests/testdata/compat/import_cjs_from_esm/reexports2.js

@@ -0,0 +1,2 @@
+exports.buzz = "buzz";
+exports.fizz = "FIZZ";