core: introduce extensions

cli/tests/unit/metrics_test.ts

@@ -66,3 +66,15 @@ unitTest(
     assert(metrics.opsDispatchedAsync === metrics.opsCompletedAsync);
   },
 );
+
+// Test that ops from op_crates have metrics (via OpMiddleware)
+unitTest(function metricsForOpCrates(): void {
+  const _ = new URL("https://deno.land");
+
+  const m1 = Deno.metrics().ops["op_url_parse"];
+  assert(m1.opsDispatched > 0);
+  assert(m1.opsCompleted > 0);
+  assert(m1.bytesSentControl > 0);
+  assert(m1.bytesSentData >= 0);
+  assert(m1.bytesReceived > 0);
+});

core/extensions.rs

@@ -0,0 +1,136 @@
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use crate::error::AnyError;
+use crate::{OpFn, OpId, OpState};
+
+pub type SourcePair = (&'static str, &'static str);
+pub type OpPair = (&'static str, Box<OpFn>);
+pub type RcOpRegistrar = Rc<RefCell<dyn OpRegistrar>>;
+pub type OpMiddlewareFn = dyn Fn(&'static str, Box<OpFn>) -> Box<OpFn>;
+pub type OpStateFn = dyn Fn(&mut OpState) -> Result<(), AnyError>;
+
+#[derive(Default)]
+pub struct Extension {
+  js_files: Option<Vec<SourcePair>>,
+  ops: Option<Vec<OpPair>>,
+  opstate_fn: Option<Box<OpStateFn>>,
+  middleware_fn: Option<Box<OpMiddlewareFn>>,
+}
+
+impl Extension {
+  pub fn new(
+    js_files: Option<Vec<SourcePair>>,
+    ops: Option<Vec<OpPair>>,
+    opstate_fn: Option<Box<OpStateFn>>,
+    middleware_fn: Option<Box<OpMiddlewareFn>>,
+  ) -> Self {
+    Self {
+      js_files,
+      ops,
+      opstate_fn,
+      middleware_fn,
+    }
+  }
+
+  pub fn pure_js(js_files: Vec<SourcePair>) -> Self {
+    Self::new(Some(js_files), None, None, None)
+  }
+
+  pub fn with_ops(
+    js_files: Vec<SourcePair>,
+    ops: Vec<OpPair>,
+    opstate_fn: Option<Box<OpStateFn>>,
+  ) -> Self {
+    Self::new(Some(js_files), Some(ops), opstate_fn, None)
+  }
+}
+
+// Note: this used to be a trait, but we "downgraded" it to a single concrete type
+// for the initial iteration, it will like become a trait in the future
+impl Extension {
+  /// returns JS source code to be loaded into the isolate (either at snapshotting,
+  /// or at startup).  as a vector of a tuple of the file name, and the source code.
+  pub(crate) fn init_js(&self) -> Vec<SourcePair> {
+    match &self.js_files {
+      Some(files) => files.clone(),
+      None => vec![],
+    }
+  }
+
+  /// Called at JsRuntime startup to initialize ops in the isolate.
+  pub(crate) fn init_ops(&mut self, registrar: RcOpRegistrar) {
+    // NOTE: not idempotent
+    // TODO: fail if called twice ?
+    if let Some(ops) = self.ops.take() {
+      for (name, opfn) in ops {
+        registrar.borrow_mut().register_op(name, opfn);
+      }
+    }
+  }
+
+  /// Allows setting up the initial op-state of an isolate at startup.
+  pub(crate) fn init_state(&self, state: &mut OpState) -> Result<(), AnyError> {
+    match &self.opstate_fn {
+      Some(ofn) => ofn(state),
+      None => Ok(()),
+    }
+  }
+
+  /// init_registrar lets us middleware op registrations, it's called before init_ops
+  pub(crate) fn init_registrar(
+    &mut self,
+    registrar: RcOpRegistrar,
+  ) -> RcOpRegistrar {
+    match self.middleware_fn.take() {
+      Some(middleware_fn) => Rc::new(RefCell::new(OpMiddleware {
+        registrar,
+        middleware_fn,
+      })),
+      None => registrar,
+    }
+  }
+}
+
+/// The OpRegistrar trait allows building op "middleware" such as:
+/// OpMetrics, OpTracing or OpDisabler that wrap OpFns for profiling, debugging, etc...
+/// JsRuntime is itself an OpRegistrar
+pub trait OpRegistrar {
+  fn register_op(&mut self, name: &'static str, op_fn: Box<OpFn>) -> OpId;
+}
+
+/// OpMiddleware wraps an original OpRegistrar with an OpMiddlewareFn
+pub struct OpMiddleware {
+  registrar: RcOpRegistrar,
+  middleware_fn: Box<OpMiddlewareFn>,
+}
+
+impl OpRegistrar for OpMiddleware {
+  fn register_op(&mut self, name: &'static str, op_fn: Box<OpFn>) -> OpId {
+    let new_op = (self.middleware_fn)(name, op_fn);
+    self.registrar.borrow_mut().register_op(name, new_op)
+  }
+}
+
+/// Helps embed JS files in an extension. Returns Vec<(&'static str, &'static str)>
+/// representing the filename and source code.
+///
+/// Example:
+/// ```ignore
+/// include_js_files!(
+///   prefix "deno:op_crates/hello",
+///   "01_hello.js",
+///   "02_goodbye.js",
+/// )
+/// ```
+#[macro_export]
+macro_rules! include_js_files {
+  (prefix $prefix:literal, $($file:literal,)+) => {
+    vec![
+      $((
+        concat!($prefix, "/", $file),
+        include_str!($file),
+      ),)+
+    ]
+  };
+}

core/lib.rs

@@ -9,6 +9,7 @@ mod async_cancel;
 mod async_cell;
 mod bindings;
 pub mod error;
+mod extensions;
 mod flags;
 mod gotham_state;
 mod module_specifier;
@@ -81,6 +82,10 @@ pub use crate::runtime::JsErrorCreateFn;
 pub use crate::runtime::JsRuntime;
 pub use crate::runtime::RuntimeOptions;
 pub use crate::runtime::Snapshot;
+// pub use crate::runtime_modules::include_js_files!;
+pub use crate::extensions::Extension;
+pub use crate::extensions::OpRegistrar;
+pub use crate::extensions::RcOpRegistrar;
 pub use crate::zero_copy_buf::BufVec;
 pub use crate::zero_copy_buf::ZeroCopyBuf;
 

core/runtime.rs

@@ -23,7 +23,10 @@ use crate::ops::*;
 use crate::shared_queue::SharedQueue;
 use crate::shared_queue::RECOMMENDED_SIZE;
 use crate::BufVec;
+use crate::Extension;
+use crate::OpRegistrar;
 use crate::OpState;
+use crate::RcOpRegistrar;
 use futures::channel::mpsc;
 use futures::future::poll_fn;
 use futures::stream::FuturesUnordered;
@@ -81,6 +84,7 @@ pub struct JsRuntime {
   snapshot_creator: Option<v8::SnapshotCreator>,
   has_snapshotted: bool,
   allocations: IsolateAllocations,
+  extensions: Vec<Extension>,
 }
 
 struct DynImportModEvaluate {
@@ -180,6 +184,10 @@ pub struct RuntimeOptions {
   /// executed tries to load modules.
   pub module_loader: Option<Rc<dyn ModuleLoader>>,
 
+  /// JsRuntime extensions, not to be confused with ES modules
+  /// these are sets of ops and other JS code to be initialized.
+  pub extensions: Vec<Extension>,
+
   /// V8 snapshot that should be loaded on startup.
   ///
   /// Currently can't be used with `will_snapshot`.
@@ -297,6 +305,7 @@ impl JsRuntime {
       snapshot_creator: maybe_snapshot_creator,
       has_snapshotted: false,
       allocations: IsolateAllocations::default(),
+      extensions: options.extensions,
     };
 
     if !has_startup_snapshot {
@@ -349,6 +358,41 @@ impl JsRuntime {
       .unwrap();
   }
 
+  /// Initializes JS of provided Extensions
+  // NOTE: this will probably change when streamlining snapshot flow
+  pub fn init_extension_js(&mut self) -> Result<(), AnyError> {
+    // Take extensions to avoid double-borrow
+    let mut extensions: Vec<Extension> = self.extensions.drain(..).collect();
+    for m in extensions.iter_mut() {
+      let js_files = m.init_js();
+      for (filename, source) in js_files {
+        self.execute_static(filename, source)?;
+      }
+    }
+    self.extensions = extensions;
+    Ok(())
+  }
+
+  /// Initializes ops of provided Extensions
+  // NOTE: this will probably change when streamlining snapshot flow
+  pub fn init_extension_ops(&mut self) -> Result<(), AnyError> {
+    let op_state = self.op_state();
+    // Original OpRegistrar
+    let mut registrar: RcOpRegistrar =
+      Rc::new(RefCell::new(JsRuntimeOpRegistrar(op_state.clone())));
+
+    // Wrap registrar
+    for e in self.extensions.iter_mut() {
+      registrar = e.init_registrar(registrar.clone());
+    }
+
+    for e in self.extensions.iter_mut() {
+      e.init_state(&mut op_state.borrow_mut())?;
+      e.init_ops(registrar.clone());
+    }
+    Ok(())
+  }
+
   /// Executes a JavaScript code to initialize shared queue binding
   /// between Rust and JS.
   ///
@@ -411,6 +455,40 @@ impl JsRuntime {
     }
   }
 
+  // same as execute() but for internal JS code
+  fn execute_static(
+    &mut self,
+    js_filename: &'static str,
+    js_source: &'static str,
+  ) -> Result<(), AnyError> {
+    let context = self.global_context();
+
+    let scope = &mut v8::HandleScope::with_context(self.v8_isolate(), context);
+
+    let source = v8::String::new(scope, js_source).unwrap();
+    let name = v8::String::new(scope, js_filename).unwrap();
+    let origin = bindings::script_origin(scope, name);
+
+    let tc_scope = &mut v8::TryCatch::new(scope);
+
+    let script = match v8::Script::compile(tc_scope, source, Some(&origin)) {
+      Some(script) => script,
+      None => {
+        let exception = tc_scope.exception().unwrap();
+        return exception_to_err_result(tc_scope, exception, false);
+      }
+    };
+
+    match script.run(tc_scope) {
+      Some(_) => Ok(()),
+      None => {
+        assert!(tc_scope.has_caught());
+        let exception = tc_scope.exception().unwrap();
+        exception_to_err_result(tc_scope, exception, false)
+      }
+    }
+  }
+
   /// Takes a snapshot. The isolate should have been created with will_snapshot
   /// set to true.
   ///
@@ -1480,6 +1558,14 @@ impl JsRuntime {
   }
 }
 
+struct JsRuntimeOpRegistrar(Rc<RefCell<OpState>>);
+
+impl OpRegistrar for JsRuntimeOpRegistrar {
+  fn register_op(&mut self, name: &str, op_fn: Box<OpFn>) -> OpId {
+    self.0.borrow_mut().op_table.register_op(name, op_fn)
+  }
+}
+
 #[cfg(test)]
 pub mod tests {
   use super::*;

op_crates/console/lib.rs

@@ -1,23 +1,15 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
-use deno_core::JsRuntime;
+use deno_core::include_js_files;
+use deno_core::Extension;
 use std::path::PathBuf;
 
-/// Load and execute the javascript code.
-pub fn init(isolate: &mut JsRuntime) {
-  let files = vec![
-    (
-      "deno:op_crates/console/01_colors.js",
-      include_str!("01_colors.js"),
-    ),
-    (
-      "deno:op_crates/console/02_console.js",
-      include_str!("02_console.js"),
-    ),
-  ];
-  for (url, source_code) in files {
-    isolate.execute(url, source_code).unwrap();
-  }
+pub fn init() -> Extension {
+  Extension::pure_js(include_js_files!(
+    prefix "deno:op_crates/console",
+    "01_colors.js",
+    "02_console.js",
+  ))
 }
 
 pub fn get_declaration() -> PathBuf {

op_crates/crypto/lib.rs

@@ -3,27 +3,38 @@
 #![deny(warnings)]
 
 use deno_core::error::AnyError;
+use deno_core::include_js_files;
+use deno_core::json_op_sync;
 use deno_core::serde_json::json;
 use deno_core::serde_json::Value;
-use deno_core::JsRuntime;
+use deno_core::Extension;
 use deno_core::OpState;
 use deno_core::ZeroCopyBuf;
 use rand::rngs::StdRng;
 use rand::thread_rng;
 use rand::Rng;
+use rand::SeedableRng;
 use std::path::PathBuf;
 
 pub use rand; // Re-export rand
 
-/// Execute this crates' JS source files.
-pub fn init(isolate: &mut JsRuntime) {
-  let files = vec![(
-    "deno:op_crates/crypto/01_crypto.js",
-    include_str!("01_crypto.js"),
-  )];
-  for (url, source_code) in files {
-    isolate.execute(url, source_code).unwrap();
-  }
+pub fn init(maybe_seed: Option<u64>) -> Extension {
+  Extension::with_ops(
+    include_js_files!(
+      prefix "deno:op_crates/crypto",
+      "01_crypto.js",
+    ),
+    vec![(
+      "op_crypto_get_random_values",
+      json_op_sync(op_crypto_get_random_values),
+    )],
+    Some(Box::new(move |state| {
+      if let Some(seed) = maybe_seed {
+        state.put(StdRng::seed_from_u64(seed));
+      }
+      Ok(())
+    })),
+  )
 }
 
 pub fn op_crypto_get_random_values(