core: introduce extensions

cli/tests/unit/metrics_test.ts

@@ -69,3 +69,12 @@ unitTest(
     assert(metrics.opsDispatchedAsync === metrics.opsCompletedAsync);
   },
 );
+
+// Test that ops from op_crates have metrics (via OpMiddleware)
+unitTest(function metricsForOpCrates(): void {
+  const _ = new URL("https://deno.land");
+
+  const m1 = Deno.metrics().ops["op_url_parse"];
+  assert(m1.opsDispatched > 0);
+  assert(m1.opsCompleted > 0);
+});

core/extensions.rs

@@ -0,0 +1,105 @@
+use crate::error::AnyError;
+use crate::{OpFn, OpState};
+
+pub type SourcePair = (&'static str, &'static str);
+pub type OpPair = (&'static str, Box<OpFn>);
+pub type OpMiddlewareFn = dyn Fn(&'static str, Box<OpFn>) -> Box<OpFn>;
+pub type OpStateFn = dyn Fn(&mut OpState) -> Result<(), AnyError>;
+
+#[derive(Default)]
+pub struct Extension {
+  js_files: Option<Vec<SourcePair>>,
+  ops: Option<Vec<OpPair>>,
+  opstate_fn: Option<Box<OpStateFn>>,
+  middleware_fn: Option<Box<OpMiddlewareFn>>,
+  initialized: bool,
+}
+
+impl Extension {
+  pub fn new(
+    js_files: Option<Vec<SourcePair>>,
+    ops: Option<Vec<OpPair>>,
+    opstate_fn: Option<Box<OpStateFn>>,
+    middleware_fn: Option<Box<OpMiddlewareFn>>,
+  ) -> Self {
+    Self {
+      js_files,
+      ops,
+      opstate_fn,
+      middleware_fn,
+      initialized: false,
+    }
+  }
+
+  pub fn pure_js(js_files: Vec<SourcePair>) -> Self {
+    Self::new(Some(js_files), None, None, None)
+  }
+
+  pub fn with_ops(
+    js_files: Vec<SourcePair>,
+    ops: Vec<OpPair>,
+    opstate_fn: Option<Box<OpStateFn>>,
+  ) -> Self {
+    Self::new(Some(js_files), Some(ops), opstate_fn, None)
+  }
+}
+
+// Note: this used to be a trait, but we "downgraded" it to a single concrete type
+// for the initial iteration, it will likely become a trait in the future
+impl Extension {
+  /// returns JS source code to be loaded into the isolate (either at snapshotting,
+  /// or at startup).  as a vector of a tuple of the file name, and the source code.
+  pub(crate) fn init_js(&self) -> Vec<SourcePair> {
+    match &self.js_files {
+      Some(files) => files.clone(),
+      None => vec![],
+    }
+  }
+
+  /// Called at JsRuntime startup to initialize ops in the isolate.
+  pub(crate) fn init_ops(&mut self) -> Option<Vec<OpPair>> {
+    // TODO(@AaronO): maybe make op registration idempotent
+    if self.initialized {
+      panic!("init_ops called twice: not idempotent or correct");
+    }
+    self.initialized = true;
+
+    self.ops.take()
+  }
+
+  /// Allows setting up the initial op-state of an isolate at startup.
+  pub(crate) fn init_state(&self, state: &mut OpState) -> Result<(), AnyError> {
+    match &self.opstate_fn {
+      Some(ofn) => ofn(state),
+      None => Ok(()),
+    }
+  }
+
+  /// init_middleware lets us middleware op registrations, it's called before init_ops
+  pub(crate) fn init_middleware(&mut self) -> Option<Box<OpMiddlewareFn>> {
+    self.middleware_fn.take()
+  }
+}
+
+/// Helps embed JS files in an extension. Returns Vec<(&'static str, &'static str)>
+/// representing the filename and source code.
+///
+/// Example:
+/// ```ignore
+/// include_js_files!(
+///   prefix "deno:op_crates/hello",
+///   "01_hello.js",
+///   "02_goodbye.js",
+/// )
+/// ```
+#[macro_export]
+macro_rules! include_js_files {
+  (prefix $prefix:literal, $($file:literal,)+) => {
+    vec![
+      $((
+        concat!($prefix, "/", $file),
+        include_str!($file),
+      ),)+
+    ]
+  };
+}

core/lib.rs

@@ -3,6 +3,7 @@ mod async_cancel;
 mod async_cell;
 mod bindings;
 pub mod error;
+mod extensions;
 mod flags;
 mod gotham_state;
 mod module_specifier;
@@ -75,6 +76,9 @@ pub use crate::runtime::JsErrorCreateFn;
 pub use crate::runtime::JsRuntime;
 pub use crate::runtime::RuntimeOptions;
 pub use crate::runtime::Snapshot;
+// pub use crate::runtime_modules::include_js_files!;
+pub use crate::extensions::Extension;
+pub use crate::extensions::OpMiddlewareFn;
 pub use crate::zero_copy_buf::ZeroCopyBuf;
 
 pub fn v8_version() -> &'static str {

core/runtime.rs

@@ -20,6 +20,8 @@ use crate::modules::NoopModuleLoader;
 use crate::modules::PrepareLoadFuture;
 use crate::modules::RecursiveModuleLoad;
 use crate::ops::*;
+use crate::Extension;
+use crate::OpMiddlewareFn;
 use crate::OpPayload;
 use crate::OpResponse;
 use crate::OpState;
@@ -83,6 +85,7 @@ pub struct JsRuntime {
   snapshot_creator: Option<v8::SnapshotCreator>,
   has_snapshotted: bool,
   allocations: IsolateAllocations,
+  extensions: Vec<Extension>,
 }
 
 struct DynImportModEvaluate {
@@ -186,6 +189,10 @@ pub struct RuntimeOptions {
   /// executed tries to load modules.
   pub module_loader: Option<Rc<dyn ModuleLoader>>,
 
+  /// JsRuntime extensions, not to be confused with ES modules
+  /// these are sets of ops and other JS code to be initialized.
+  pub extensions: Vec<Extension>,
+
   /// V8 snapshot that should be loaded on startup.
   ///
   /// Currently can't be used with `will_snapshot`.
@@ -300,6 +307,7 @@ impl JsRuntime {
       snapshot_creator: maybe_snapshot_creator,
       has_snapshotted: false,
       allocations: IsolateAllocations::default(),
+      extensions: options.extensions,
     };
 
     if !has_startup_snapshot {
@@ -349,6 +357,57 @@ impl JsRuntime {
       .unwrap();
   }
 
+  /// Initializes JS of provided Extensions
+  // NOTE: this will probably change when streamlining snapshot flow
+  pub fn init_extension_js(&mut self) -> Result<(), AnyError> {
+    // Take extensions to avoid double-borrow
+    let mut extensions: Vec<Extension> = std::mem::take(&mut self.extensions);
+    for m in extensions.iter_mut() {
+      let js_files = m.init_js();
+      for (filename, source) in js_files {
+        // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap
+        self.execute(filename, source)?;
+      }
+    }
+    // Restore extensions
+    self.extensions = extensions;
+
+    Ok(())
+  }
+
+  /// Initializes ops of provided Extensions
+  // NOTE: this will probably change when streamlining snapshot flow
+  pub fn init_extension_ops(&mut self) -> Result<(), AnyError> {
+    let op_state = self.op_state();
+    // Take extensions to avoid double-borrow
+    let mut extensions: Vec<Extension> = std::mem::take(&mut self.extensions);
+
+    // Middleware
+    let middleware: Vec<Box<OpMiddlewareFn>> = extensions
+      .iter_mut()
+      .filter_map(|e| e.init_middleware())
+      .collect();
+    // macroware wraps an opfn in all the middleware
+    let macroware =
+      move |name, opfn| middleware.iter().fold(opfn, |opfn, m| m(name, opfn));
+
+    // Register ops
+    for e in extensions.iter_mut() {
+      e.init_state(&mut op_state.borrow_mut())?;
+      // Register each op after middlewaring it
+      let mut ops = e.init_ops().unwrap_or_default();
+      for (name, opfn) in ops {
+        self.register_op(name, macroware(name, opfn));
+      }
+    }
+    // Sync ops cache
+    self.sync_ops_cache();
+    // Restore extensions
+    self.extensions = extensions;
+
+    Ok(())
+  }
+
   /// Grabs a reference to core.js' handleAsyncMsgFromRust
   fn init_recv_cb(&mut self) {
     let context = self.global_context();

op_crates/console/lib.rs

@@ -1,23 +1,15 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
-use deno_core::JsRuntime;
+use deno_core::include_js_files;
+use deno_core::Extension;
 use std::path::PathBuf;
 
-/// Load and execute the javascript code.
-pub fn init(isolate: &mut JsRuntime) {
-  let files = vec![
-    (
-      "deno:op_crates/console/01_colors.js",
-      include_str!("01_colors.js"),
-    ),
-    (
-      "deno:op_crates/console/02_console.js",
-      include_str!("02_console.js"),
-    ),
-  ];
-  for (url, source_code) in files {
-    isolate.execute(url, source_code).unwrap();
-  }
+pub fn init() -> Extension {
+  Extension::pure_js(include_js_files!(
+    prefix "deno:op_crates/console",
+    "01_colors.js",
+    "02_console.js",
+  ))
 }
 
 pub fn get_declaration() -> PathBuf {

op_crates/crypto/lib.rs

@@ -4,25 +4,36 @@
 
 use deno_core::error::null_opbuf;
 use deno_core::error::AnyError;
-use deno_core::JsRuntime;
+use deno_core::include_js_files;
+use deno_core::op_sync;
+use deno_core::Extension;
 use deno_core::OpState;
 use deno_core::ZeroCopyBuf;
 use rand::rngs::StdRng;
 use rand::thread_rng;
 use rand::Rng;
+use rand::SeedableRng;
 use std::path::PathBuf;
 
 pub use rand; // Re-export rand
 
-/// Execute this crates' JS source files.
-pub fn init(isolate: &mut JsRuntime) {
-  let files = vec![(
-    "deno:op_crates/crypto/01_crypto.js",
-    include_str!("01_crypto.js"),
-  )];
-  for (url, source_code) in files {
-    isolate.execute(url, source_code).unwrap();
-  }
+pub fn init(maybe_seed: Option<u64>) -> Extension {
+  Extension::with_ops(
+    include_js_files!(
+      prefix "deno:op_crates/crypto",
+      "01_crypto.js",
+    ),
+    vec![(
+      "op_crypto_get_random_values",
+      op_sync(op_crypto_get_random_values),
+    )],
+    Some(Box::new(move |state| {
+      if let Some(seed) = maybe_seed {
+        state.put(StdRng::seed_from_u64(seed));
+      }
+      Ok(())
+    })),
+  )
 }
 
 pub fn op_crypto_get_random_values(