diff --git a/cli/flags.rs b/cli/flags.rs index a8a4ddba2d2eaa..9282decce5a00e 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -22,6 +22,10 @@ pub enum DenoSubcommand { source_file: String, out_file: Option, }, + Compile { + source_file: String, + out_file: Option, + }, Completions { buf: Box<[u8]>, }, @@ -292,6 +296,8 @@ pub fn flags_from_vec_safe(args: Vec) -> clap::Result { doc_parse(&mut flags, m); } else if let Some(m) = matches.subcommand_matches("lint") { lint_parse(&mut flags, m); + } else if let Some(m) = matches.subcommand_matches("compile") { + compile_parse(&mut flags, m); } else { repl_parse(&mut flags, &matches); } @@ -341,6 +347,7 @@ If the flag is set, restrict these messages to errors.", ) .subcommand(bundle_subcommand()) .subcommand(cache_subcommand()) + .subcommand(compile_subcommand()) .subcommand(completions_subcommand()) .subcommand(doc_subcommand()) .subcommand(eval_subcommand()) @@ -408,6 +415,18 @@ fn install_parse(flags: &mut Flags, matches: &clap::ArgMatches) { }; } +fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) { + compile_args_parse(flags, matches); + + let source_file = matches.value_of("source_file").unwrap().to_string(); + let out_file = matches.value_of("out_file").map(|s| s.to_string()); + + flags.subcommand = DenoSubcommand::Compile { + source_file, + out_file, + }; +} + fn bundle_parse(flags: &mut Flags, matches: &clap::ArgMatches) { compile_args_parse(flags, matches); @@ -802,6 +821,32 @@ The installation root is determined, in order of precedence: These must be added to the path manually if required.") } +fn compile_subcommand<'a, 'b>() -> App<'a, 'b> { + compile_args(SubCommand::with_name("compile")) + .arg( + Arg::with_name("source_file") + .takes_value(true) + .required(true), + ) + .arg(Arg::with_name("out_file").takes_value(true)) + .about("Compile the script into a self contained executable") + .long_about( + "Compiles the given script into a self contained executable. + deno compile --unstable https://deno.land/std/http/file_server.ts + deno compile --unstable https://deno.land/std/examples/colors.ts color_util + +The executable name is inferred by default: + - Attempt to take the file stem of the URL path. The above example would + become 'file_server'. + - If the file stem is something generic like 'main', 'mod', 'index' or 'cli', + and the path has no parent, take the file name of the parent path. Otherwise + settle with the generic name. + - If the resulting name has an '@...' suffix, strip it. + +Cross compiling binaries for different platforms is not currently possible.", + ) +} + fn bundle_subcommand<'a, 'b>() -> App<'a, 'b> { compile_args(SubCommand::with_name("bundle")) .arg( @@ -3200,4 +3245,48 @@ mod tests { } ); } + + #[test] + fn compile() { + let r = flags_from_vec_safe(svec![ + "deno", + "compile", + "https://deno.land/std/examples/colors.ts" + ]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Compile { + source_file: "https://deno.land/std/examples/colors.ts".to_string(), + out_file: None + }, + ..Flags::default() + } + ); + } + + #[test] + fn compile_with_flags() { + #[rustfmt::skip] + let r = flags_from_vec_safe(svec!["deno", "compile", "--unstable", "--import-map", "import_map.json", "--no-remote", "--config", "tsconfig.json", "--no-check", "--reload", "--lock", "lock.json", "--lock-write", "--cert", "example.crt", "https://deno.land/std/examples/colors.ts", "colors"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Compile { + source_file: "https://deno.land/std/examples/colors.ts".to_string(), + out_file: Some("colors".to_string()) + }, + unstable: true, + import_map_path: Some("import_map.json".to_string()), + no_remote: true, + config_path: Some("tsconfig.json".to_string()), + no_check: true, + reload: true, + lock: Some(PathBuf::from("lock.json")), + lock_write: true, + ca_file: Some("example.crt".to_string()), + ..Flags::default() + } + ); + } } diff --git a/cli/main.rs b/cli/main.rs index 6c48a75f6b3ec5..b4e1f1d2c16e9b 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -39,6 +39,7 @@ mod resolve_addr; mod signal; mod source_maps; mod specifier_handler; +mod standalone; mod text_encoding; mod tokio_util; mod tools; @@ -51,10 +52,16 @@ mod worker; use crate::file_fetcher::File; use crate::file_fetcher::FileFetcher; use crate::file_watcher::ModuleResolutionResult; +use crate::flags::DenoSubcommand; +use crate::flags::Flags; +use crate::import_map::ImportMap; use crate::media_type::MediaType; use crate::permissions::Permissions; +use crate::program_state::exit_unstable; use crate::program_state::ProgramState; use crate::specifier_handler::FetchHandler; +use crate::standalone::create_standalone_binary; +use crate::tools::installer::infer_name_from_url; use crate::worker::MainWorker; use deno_core::error::generic_error; use deno_core::error::AnyError; @@ -66,12 +73,8 @@ use deno_core::v8_set_flags; use deno_core::ModuleSpecifier; use deno_doc as doc; use deno_doc::parser::DocFileLoader; -use flags::DenoSubcommand; -use flags::Flags; -use import_map::ImportMap; use log::Level; use log::LevelFilter; -use program_state::exit_unstable; use std::cell::RefCell; use std::env; use std::io::Read; @@ -149,6 +152,56 @@ fn get_types(unstable: bool) -> String { types } +async fn compile_command( + flags: Flags, + source_file: String, + out_file: Option, +) -> Result<(), AnyError> { + if !flags.unstable { + exit_unstable("compile"); + } + + let debug = flags.log_level == Some(log::Level::Debug); + + let module_specifier = ModuleSpecifier::resolve_url_or_path(&source_file)?; + let program_state = ProgramState::new(flags.clone())?; + + let out_file = + out_file.or_else(|| infer_name_from_url(module_specifier.as_url())); + let out_file = match out_file { + Some(out_file) => out_file, + None => return Err(generic_error( + "An executable name was not provided. One could not be inferred from the URL. Aborting.", + )), + }; + + let module_graph = create_module_graph_and_maybe_check( + module_specifier.clone(), + program_state.clone(), + debug, + ) + .await?; + + info!( + "{} {}", + colors::green("Bundle"), + module_specifier.to_string() + ); + let bundle_str = bundle_module_graph(module_graph, flags, debug)?; + + info!( + "{} {}", + colors::green("Compile"), + module_specifier.to_string() + ); + create_standalone_binary(bundle_str.as_bytes().to_vec(), out_file.clone()) + .await?; + + info!("{} {}", colors::green("Emit"), out_file); + + Ok(()) +} + async fn info_command( flags: Flags, maybe_specifier: Option, @@ -299,6 +352,73 @@ async fn eval_command( Ok(()) } +async fn create_module_graph_and_maybe_check( + module_specifier: ModuleSpecifier, + program_state: Arc, + debug: bool, +) -> Result { + let handler = Rc::new(RefCell::new(FetchHandler::new( + &program_state, + // when bundling, dynamic imports are only access for their type safety, + // therefore we will allow the graph to access any module. + Permissions::allow_all(), + )?)); + let mut builder = module_graph::GraphBuilder::new( + handler, + program_state.maybe_import_map.clone(), + program_state.lockfile.clone(), + ); + builder.add(&module_specifier, false).await?; + let module_graph = builder.get_graph(); + + if !program_state.flags.no_check { + // TODO(@kitsonk) support bundling for workers + let lib = if program_state.flags.unstable { + module_graph::TypeLib::UnstableDenoWindow + } else { + module_graph::TypeLib::DenoWindow + }; + let result_info = + module_graph.clone().check(module_graph::CheckOptions { + debug, + emit: false, + lib, + maybe_config_path: program_state.flags.config_path.clone(), + reload: program_state.flags.reload, + })?; + + debug!("{}", result_info.stats); + if let Some(ignored_options) = result_info.maybe_ignored_options { + eprintln!("{}", ignored_options); + } + if !result_info.diagnostics.is_empty() { + return Err(generic_error(result_info.diagnostics.to_string())); + } + } + + Ok(module_graph) +} + +fn bundle_module_graph( + module_graph: module_graph::Graph, + flags: Flags, + debug: bool, +) -> Result { + let (bundle, stats, maybe_ignored_options) = + module_graph.bundle(module_graph::BundleOptions { + debug, + maybe_config_path: flags.config_path, + })?; + match maybe_ignored_options { + Some(ignored_options) if flags.no_check => { + eprintln!("{}", ignored_options); + } + _ => {} + } + debug!("{}", stats); + Ok(bundle) +} + async fn bundle_command( flags: Flags, source_file: String, @@ -323,44 +443,12 @@ async fn bundle_command( module_specifier.to_string() ); - let handler = Rc::new(RefCell::new(FetchHandler::new( - &program_state, - // when bundling, dynamic imports are only access for their type safety, - // therefore we will allow the graph to access any module. - Permissions::allow_all(), - )?)); - let mut builder = module_graph::GraphBuilder::new( - handler, - program_state.maybe_import_map.clone(), - program_state.lockfile.clone(), - ); - builder.add(&module_specifier, false).await?; - let module_graph = builder.get_graph(); - - if !flags.no_check { - // TODO(@kitsonk) support bundling for workers - let lib = if flags.unstable { - module_graph::TypeLib::UnstableDenoWindow - } else { - module_graph::TypeLib::DenoWindow - }; - let result_info = - module_graph.clone().check(module_graph::CheckOptions { - debug, - emit: false, - lib, - maybe_config_path: flags.config_path.clone(), - reload: flags.reload, - })?; - - debug!("{}", result_info.stats); - if let Some(ignored_options) = result_info.maybe_ignored_options { - eprintln!("{}", ignored_options); - } - if !result_info.diagnostics.is_empty() { - return Err(generic_error(result_info.diagnostics.to_string())); - } - } + let module_graph = create_module_graph_and_maybe_check( + module_specifier, + program_state.clone(), + debug, + ) + .await?; let mut paths_to_watch: Vec = module_graph .get_modules() @@ -392,19 +480,7 @@ async fn bundle_command( let flags = flags.clone(); let out_file = out_file.clone(); async move { - let (output, stats, maybe_ignored_options) = - module_graph.bundle(module_graph::BundleOptions { - debug, - maybe_config_path: flags.config_path, - })?; - - match maybe_ignored_options { - Some(ignored_options) if flags.no_check => { - eprintln!("{}", ignored_options); - } - _ => {} - } - debug!("{}", stats); + let output = bundle_module_graph(module_graph, flags, debug)?; debug!(">>>>> bundle END"); @@ -898,6 +974,10 @@ fn get_subcommand( DenoSubcommand::Cache { files } => { cache_command(flags, files).boxed_local() } + DenoSubcommand::Compile { + source_file, + out_file, + } => compile_command(flags, source_file, out_file).boxed_local(), DenoSubcommand::Fmt { check, files, @@ -968,8 +1048,12 @@ pub fn main() { colors::enable_ansi(); // For Windows 10 let args: Vec = env::args().collect(); - let flags = flags::flags_from_vec(args); + if let Err(err) = standalone::try_run_standalone_binary(args.clone()) { + eprintln!("{}: {}", colors::red_bold("error"), err.to_string()); + std::process::exit(1); + } + let flags = flags::flags_from_vec(args); if let Some(ref v8_flags) = flags.v8_flags { init_v8_flags(v8_flags); } diff --git a/cli/standalone.rs b/cli/standalone.rs new file mode 100644 index 00000000000000..805849c81f989d --- /dev/null +++ b/cli/standalone.rs @@ -0,0 +1,159 @@ +use crate::colors; +use crate::flags::Flags; +use crate::permissions::Permissions; +use crate::program_state::ProgramState; +use crate::tokio_util; +use crate::worker::MainWorker; +use deno_core::error::type_error; +use deno_core::error::AnyError; +use deno_core::futures::FutureExt; +use deno_core::ModuleLoader; +use deno_core::ModuleSpecifier; +use deno_core::OpState; +use std::cell::RefCell; +use std::convert::TryInto; +use std::env::current_exe; +use std::fs::File; +use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; +use std::io::Write; +use std::pin::Pin; +use std::rc::Rc; + +const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; + +/// This function will try to run this binary as a standalone binary +/// produced by `deno compile`. It determines if this is a stanalone +/// binary by checking for the magic trailer string `D3N0` at EOF-12. +/// After the magic trailer is a u64 pointer to the start of the JS +/// file embedded in the binary. This file is read, and run. If no +/// magic trailer is present, this function exits with Ok(()). +pub fn try_run_standalone_binary(args: Vec) -> Result<(), AnyError> { + let current_exe_path = current_exe()?; + + let mut current_exe = File::open(current_exe_path)?; + let trailer_pos = current_exe.seek(SeekFrom::End(-16))?; + let mut trailer = [0; 16]; + current_exe.read_exact(&mut trailer)?; + let (magic_trailer, bundle_pos_arr) = trailer.split_at(8); + if magic_trailer == MAGIC_TRAILER { + let bundle_pos_arr: &[u8; 8] = bundle_pos_arr.try_into()?; + let bundle_pos = u64::from_be_bytes(*bundle_pos_arr); + current_exe.seek(SeekFrom::Start(bundle_pos))?; + + let bundle_len = trailer_pos - bundle_pos; + let mut bundle = String::new(); + current_exe.take(bundle_len).read_to_string(&mut bundle)?; + // TODO: check amount of bytes read + + if let Err(err) = tokio_util::run_basic(run(bundle, args)) { + eprintln!("{}: {}", colors::red_bold("error"), err.to_string()); + std::process::exit(1); + } + std::process::exit(0); + } else { + Ok(()) + } +} + +const SPECIFIER: &str = "file://$deno$/bundle.js"; + +struct EmbeddedModuleLoader(String); + +impl ModuleLoader for EmbeddedModuleLoader { + fn resolve( + &self, + _op_state: Rc>, + specifier: &str, + _referrer: &str, + _is_main: bool, + ) -> Result { + if specifier != SPECIFIER { + return Err(type_error( + "Self-contained binaries don't support module loading", + )); + } + Ok(ModuleSpecifier::resolve_url(specifier)?) + } + + fn load( + &self, + _op_state: Rc>, + module_specifier: &ModuleSpecifier, + _maybe_referrer: Option, + _is_dynamic: bool, + ) -> Pin> { + let module_specifier = module_specifier.clone(); + let code = self.0.to_string(); + async move { + if module_specifier.to_string() != SPECIFIER { + return Err(type_error( + "Self-contained binaries don't support module loading", + )); + } + Ok(deno_core::ModuleSource { + code, + module_url_specified: module_specifier.to_string(), + module_url_found: module_specifier.to_string(), + }) + } + .boxed_local() + } +} + +async fn run(source_code: String, args: Vec) -> Result<(), AnyError> { + let mut flags = Flags::default(); + flags.argv = args[1..].to_vec(); + // TODO(lucacasonato): remove once you can specify this correctly through embedded metadata + flags.unstable = true; + let main_module = ModuleSpecifier::resolve_url(SPECIFIER)?; + let program_state = ProgramState::new(flags.clone())?; + let permissions = Permissions::allow_all(); + let module_loader = Rc::new(EmbeddedModuleLoader(source_code)); + let mut worker = MainWorker::from_options( + &program_state, + main_module.clone(), + permissions, + module_loader, + ); + worker.execute_module(&main_module).await?; + worker.execute("window.dispatchEvent(new Event('load'))")?; + worker.run_event_loop().await?; + worker.execute("window.dispatchEvent(new Event('unload'))")?; + Ok(()) +} + +/// This functions creates a standalone deno binary by appending a bundle +/// and magic trailer to the currently executing binary. +pub async fn create_standalone_binary( + mut source_code: Vec, + out_file: String, +) -> Result<(), AnyError> { + let original_binary_path = std::env::current_exe()?; + let mut original_bin = tokio::fs::read(original_binary_path).await?; + + let mut trailer = MAGIC_TRAILER.to_vec(); + trailer.write_all(&original_bin.len().to_be_bytes())?; + + let mut final_bin = + Vec::with_capacity(original_bin.len() + source_code.len() + trailer.len()); + final_bin.append(&mut original_bin); + final_bin.append(&mut source_code); + final_bin.append(&mut trailer); + + let out_file = if cfg!(windows) && !out_file.ends_with(".exe") { + format!("{}.exe", out_file) + } else { + out_file + }; + tokio::fs::write(&out_file, final_bin).await?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o777); + tokio::fs::set_permissions(out_file, perms).await?; + } + + Ok(()) +} diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 8ad4ea5ed85dc3..380fa0fda7ab1e 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -98,7 +98,6 @@ fn eval_p() { } #[test] - fn run_from_stdin() { let mut deno = util::deno_cmd() .current_dir(util::root_path()) @@ -4496,3 +4495,100 @@ fn fmt_ignore_unexplicit_files() { assert!(output.status.success()); assert_eq!(output.stderr, b"Checked 0 file\n"); } + +#[test] +fn compile() { + let dir = TempDir::new().expect("tempdir fail"); + let exe = if cfg!(windows) { + dir.path().join("welcome.exe") + } else { + dir.path().join("welcome") + }; + let output = util::deno_cmd() + .current_dir(util::root_path()) + .arg("compile") + .arg("--unstable") + .arg("./std/examples/welcome.ts") + .arg(&exe) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + let output = Command::new(exe) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + assert_eq!(output.stdout, "Welcome to Deno 🦕\n".as_bytes()); +} + +#[test] +fn standalone_args() { + let dir = TempDir::new().expect("tempdir fail"); + let exe = if cfg!(windows) { + dir.path().join("args.exe") + } else { + dir.path().join("args") + }; + let output = util::deno_cmd() + .current_dir(util::root_path()) + .arg("compile") + .arg("--unstable") + .arg("./cli/tests/028_args.ts") + .arg(&exe) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + let output = Command::new(exe) + .arg("foo") + .arg("--bar") + .arg("--unstable") + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + assert_eq!(output.stdout, b"foo\n--bar\n--unstable\n"); +} + +#[test] +fn standalone_no_module_load() { + let dir = TempDir::new().expect("tempdir fail"); + let exe = if cfg!(windows) { + dir.path().join("hello.exe") + } else { + dir.path().join("hello") + }; + let output = util::deno_cmd() + .current_dir(util::root_path()) + .arg("compile") + .arg("--unstable") + .arg("./cli/tests/standalone_import.ts") + .arg(&exe) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(output.status.success()); + let output = Command::new(exe) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap() + .wait_with_output() + .unwrap(); + assert!(!output.status.success()); + assert_eq!(output.stdout, b"start\n"); + let stderr_str = String::from_utf8(output.stderr).unwrap(); + assert!(util::strip_ansi_codes(&stderr_str) + .contains("Self-contained binaries don't support module loading")); +} diff --git a/cli/tests/standalone_import.ts b/cli/tests/standalone_import.ts new file mode 100644 index 00000000000000..804102a535a7f3 --- /dev/null +++ b/cli/tests/standalone_import.ts @@ -0,0 +1,2 @@ +console.log("start"); +await import("./001_hello.js"); diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index e0a99873a12d08..f2f5562c2ceb48 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -108,7 +108,7 @@ fn get_installer_root() -> Result { Ok(home_path) } -fn infer_name_from_url(url: &Url) -> Option { +pub fn infer_name_from_url(url: &Url) -> Option { let path = PathBuf::from(url.path()); let mut stem = match path.file_stem() { Some(stem) => stem.to_string_lossy().to_string(), diff --git a/cli/worker.rs b/cli/worker.rs index f4a919df6aadc6..68f0a2210e570e 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -17,9 +17,11 @@ use deno_core::futures::future::FutureExt; use deno_core::url::Url; use deno_core::JsRuntime; use deno_core::ModuleId; +use deno_core::ModuleLoader; use deno_core::ModuleSpecifier; use deno_core::RuntimeOptions; use std::env; +use std::rc::Rc; use std::sync::Arc; use std::task::Context; use std::task::Poll; @@ -45,6 +47,16 @@ impl MainWorker { ) -> Self { let module_loader = CliModuleLoader::new(program_state.maybe_import_map.clone()); + + Self::from_options(program_state, main_module, permissions, module_loader) + } + + pub fn from_options( + program_state: &Arc, + main_module: ModuleSpecifier, + permissions: Permissions, + module_loader: Rc, + ) -> Self { let global_state_ = program_state.clone(); let js_error_create_fn = Box::new(move |core_js_error| {