feat: lazy loaded esm sources

core/extensions.rs

@@ -270,6 +270,8 @@ macro_rules! or {
 ///  * bounds: a comma-separated list of additional type bounds, eg: `bounds = [ P::MyAssociatedType: MyTrait ]`
 ///  * ops: a comma-separated list of [`OpDecl`]s to provide, eg: `ops = [ op_foo, op_bar ]`
 ///  * esm: a comma-separated list of ESM module filenames (see [`include_js_files`]), eg: `esm = [ dir "dir", "my_file.js" ]`
+///  * lazy_loaded_esm: a comma-separated list of ESM module filenames (see [`include_js_files`]), that will be included in
+///     the produced binary, but not automatically evaluated. Eg: `lazy_loaded_esm = [ dir "dir", "my_file.js" ]`
 ///  * js: a comma-separated list of JS filenames (see [`include_js_files`]), eg: `js = [ dir "dir", "my_file.js" ]`
 ///  * config: a structure-like definition for configuration parameters which will be required when initializing this extension, eg: `config = { my_param: Option<usize> }`
 ///  * middleware: an [`OpDecl`] middleware function with the signature `fn (OpDecl) -> OpDecl`
@@ -289,6 +291,7 @@ macro_rules! extension {
     $(, ops = [ $( $(#[$m:meta])* $( $op:ident )::+ $( < $( $op_param:ident ),* > )?  ),+ $(,)? ] )?
     $(, esm_entry_point = $esm_entry_point:expr )?
     $(, esm = [ $( dir $dir_esm:expr , )? $( $esm:literal $( with_specifier $esm_specifier:expr )? ),* $(,)? ] )?
+    $(, lazy_loaded_esm = [ $( dir $dir_lazy_loaded_esm:expr , )? $( $lazy_loaded_esm:literal $( with_specifier $lazy_loaded_esm_specifier:expr )? ),* $(,)? ] )?
     $(, js = [ $( dir $dir_js:expr , )? $( $js:literal ),* $(,)? ] )?
     $(, options = { $( $options_id:ident : $options_type:ty ),* $(,)? } )?
     $(, middleware = $middleware_fn:expr )?
@@ -338,6 +341,10 @@ macro_rules! extension {
             const V: std::borrow::Cow<'static, [$crate::ExtensionFileSource]> = std::borrow::Cow::Borrowed(&$crate::or!($($crate::include_js_files!( $name $( dir $dir_esm , )? $( $esm $( with_specifier $esm_specifier )? , )* ))?, []));
             V
           },
+          lazy_loaded_esm_files: {
+            const V: std::borrow::Cow<'static, [$crate::ExtensionFileSource]> = std::borrow::Cow::Borrowed(&$crate::or!($($crate::include_lazy_loaded_js_files!( $name $( dir $dir_lazy_loaded_esm , )? $( $lazy_loaded_esm $( with_specifier $lazy_loaded_esm_specifier )? , )* ))?, []));
+            V
+          },
           esm_entry_point: {
             const V: Option<&'static str> = $crate::or!($(Some($esm_entry_point))?, None);
             V
@@ -497,6 +504,7 @@ pub struct Extension {
   pub deps: &'static [&'static str],
   pub js_files: Cow<'static, [ExtensionFileSource]>,
   pub esm_files: Cow<'static, [ExtensionFileSource]>,
+  pub lazy_loaded_esm_files: Cow<'static, [ExtensionFileSource]>,
   pub esm_entry_point: Option<&'static str>,
   pub ops: Cow<'static, [OpDecl]>,
   pub external_references: Cow<'static, [v8::ExternalReference<'static>]>,
@@ -515,6 +523,7 @@ impl Default for Extension {
       deps: &[],
       js_files: Cow::Borrowed(&[]),
       esm_files: Cow::Borrowed(&[]),
+      lazy_loaded_esm_files: Cow::Borrowed(&[]),
       esm_entry_point: None,
       ops: Cow::Borrowed(&[]),
       external_references: Cow::Borrowed(&[]),
@@ -582,6 +591,10 @@ impl Extension {
     self.esm_files.as_ref()
   }
 
+  pub fn get_lazy_loaded_esm_sources(&self) -> &[ExtensionFileSource] {
+    self.lazy_loaded_esm_files.as_ref()
+  }
+
   pub fn get_esm_entry_point(&self) -> Option<&'static str> {
     self.esm_entry_point
   }
@@ -644,6 +657,7 @@ impl Extension {
 pub struct ExtensionBuilder {
   js: Vec<ExtensionFileSource>,
   esm: Vec<ExtensionFileSource>,
+  lazy_loaded_esm: Vec<ExtensionFileSource>,
   esm_entry_point: Option<&'static str>,
   ops: Vec<OpDecl>,
   state: Option<Box<OpStateFn>>,
@@ -667,6 +681,14 @@ impl ExtensionBuilder {
     self
   }
 
+  pub fn lazy_loaded_esm(
+    &mut self,
+    lazy_loaded_esm_files: Vec<ExtensionFileSource>,
+  ) -> &mut Self {
+    self.lazy_loaded_esm.extend(lazy_loaded_esm_files);
+    self
+  }
+
   pub fn esm_entry_point(&mut self, entry_point: &'static str) -> &mut Self {
     self.esm_entry_point = Some(entry_point);
     self
@@ -732,6 +754,7 @@ impl ExtensionBuilder {
       deps: self.deps,
       js_files: Cow::Owned(self.js),
       esm_files: Cow::Owned(self.esm),
+      lazy_loaded_esm_files: Cow::Owned(self.lazy_loaded_esm),
       esm_entry_point: self.esm_entry_point,
       ops: Cow::Owned(self.ops),
       external_references: Cow::Owned(self.external_references),
@@ -750,6 +773,9 @@ impl ExtensionBuilder {
       deps: std::mem::take(&mut self.deps),
       js_files: Cow::Owned(std::mem::take(&mut self.js)),
       esm_files: Cow::Owned(std::mem::take(&mut self.esm)),
+      lazy_loaded_esm_files: Cow::Owned(std::mem::take(
+        &mut self.lazy_loaded_esm,
+      )),
       esm_entry_point: self.esm_entry_point.take(),
       ops: Cow::Owned(std::mem::take(&mut self.ops)),
       external_references: Cow::Owned(std::mem::take(
@@ -843,3 +869,28 @@ macro_rules! include_js_files {
     ]
   };
 }
+
+#[macro_export]
+macro_rules! include_lazy_loaded_js_files {
+  ($name:ident dir $dir:expr, $($file:literal $(with_specifier $esm_specifier:expr)?,)+) => {
+    [
+      $($crate::ExtensionFileSource {
+        specifier: $crate::or!($($esm_specifier)?, concat!("ext:", stringify!($name), "/", $file)),
+        code: $crate::ExtensionFileSourceCode::IncludedInBinary(
+          include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/", $dir, "/", $file))
+        ),
+      },)+
+    ]
+  };
+
+  ($name:ident $($file:literal $(with_specifier $esm_specifier:expr)?,)+) => {
+    [
+      $($crate::ExtensionFileSource {
+        specifier: $crate::or!($($esm_specifier)?, concat!("ext:", stringify!($name), "/", $file)),
+        code: $crate::ExtensionFileSourceCode::IncludedInBinary(
+          include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/", $file))
+        ),
+      },)+
+    ]
+  };
+}

core/modules/loaders.rs

@@ -20,6 +20,7 @@ use std::collections::HashMap;
 use std::collections::HashSet;
 use std::future::Future;
 use std::pin::Pin;
+use std::rc::Rc;
 
 pub trait ModuleLoader {
   /// Returns an absolute URL.
@@ -171,6 +172,62 @@ impl ModuleLoader for ExtModuleLoader {
   }
 }
 
+/// A loader that is used in `op_lazy_load_esm` to load and execute
+/// ES modules that were embedded in the binary using `lazy_loaded_esm`
+/// option in `extension!` macro.
+pub(crate) struct LazyEsmModuleLoader {
+  sources: Rc<RefCell<HashMap<&'static str, ExtensionFileSource>>>,
+}
+
+impl LazyEsmModuleLoader {
+  pub fn new(
+    sources: Rc<RefCell<HashMap<&'static str, ExtensionFileSource>>>,
+  ) -> Self {
+    LazyEsmModuleLoader { sources }
+  }
+}
+
+impl ModuleLoader for LazyEsmModuleLoader {
+  fn resolve(
+    &self,
+    specifier: &str,
+    referrer: &str,
+    _kind: ResolutionKind,
+  ) -> Result<ModuleSpecifier, Error> {
+    Ok(resolve_import(specifier, referrer)?)
+  }
+
+  fn load(
+    &self,
+    specifier: &ModuleSpecifier,
+    _maybe_referrer: Option<&ModuleSpecifier>,
+    _is_dyn_import: bool,
+  ) -> Pin<Box<ModuleSourceFuture>> {
+    let sources = self.sources.borrow();
+    let source = match sources.get(specifier.as_str()) {
+      Some(source) => source,
+      None => return futures::future::err(anyhow!("Specifier \"{}\" cannot be lazy-loaded as it was not included in the binary.", specifier)).boxed_local(),
+    };
+    let result = source.load();
+    match result {
+      Ok(code) => {
+        let res = ModuleSource::new(ModuleType::JavaScript, code, specifier);
+        return futures::future::ok(res).boxed_local();
+      }
+      Err(err) => return futures::future::err(err).boxed_local(),
+    }
+  }
+
+  fn prepare_load(
+    &self,
+    _specifier: &ModuleSpecifier,
+    _maybe_referrer: Option<String>,
+    _is_dyn_import: bool,
+  ) -> Pin<Box<dyn Future<Output = Result<(), Error>>>> {
+    async { Ok(()) }.boxed_local()
+  }
+}
+
 impl Drop for ExtModuleLoader {
   fn drop(&mut self) {
     let sources = self.sources.get_mut();

core/modules/map.rs

@@ -21,6 +21,7 @@ use crate::modules::ResolutionKind;
 use crate::runtime::exception_state::ExceptionState;
 use crate::runtime::JsRealm;
 use crate::runtime::SnapshottedData;
+use crate::ExtensionFileSource;
 use crate::JsRuntime;
 use crate::ModuleSource;
 use crate::ModuleSpecifier;
@@ -48,6 +49,7 @@ use tokio::sync::oneshot;
 
 use super::module_map_data::ModuleMapData;
 use super::AssertedModuleType;
+use super::LazyEsmModuleLoader;
 
 type PrepareLoadFuture =
   dyn Future<Output = (ModuleLoadId, Result<RecursiveModuleLoad, Error>)>;
@@ -1331,6 +1333,56 @@ impl ModuleMap {
 
     Ok(v8::Global::new(scope, mod_ns))
   }
+
+  pub(crate) fn add_lazy_loaded_esm_sources(
+    &self,
+    sources: &[ExtensionFileSource],
+  ) {
+    if sources.is_empty() {
+      return;
+    }
+
+    let data = self.data.borrow_mut();
+    data.lazy_esm_sources.borrow_mut().extend(
+      sources
+        .iter()
+        .cloned()
+        .map(|source| (source.specifier, source)),
+    );
+  }
+
+  /// Lazy load and evaluate an ES module. Only modules that have been added
+  /// during build time can be executed (the ones stored in
+  /// `ModuleMapData::lazy_esm_sources`), not _any, random_ module.
+  pub(crate) fn lazy_load_esm_module(
+    &self,
+    scope: &mut v8::HandleScope,
+    module_specifier: &str,
+  ) -> Result<v8::Global<v8::Value>, Error> {
+    let lazy_esm_sources = self.data.borrow().lazy_esm_sources.clone();
+    let loader = LazyEsmModuleLoader::new(lazy_esm_sources);
+
+    // Check if this module has already been loaded.
+    {
+      let module_map_data = self.data.borrow();
+      if let Some(id) = module_map_data
+        .get_id(module_specifier, AssertedModuleType::JavaScriptOrWasm)
+      {
+        let handle = module_map_data.get_handle(id).unwrap();
+        let handle_local = v8::Local::new(scope, handle);
+        let module =
+          v8::Global::new(scope, handle_local.get_module_namespace());
+        return Ok(module);
+      }
+    }
+
+    let specifier = ModuleSpecifier::parse(module_specifier)?;
+    let source = futures::executor::block_on(async {
+      loader.load(&specifier, None, false).await
+    })?;
+
+    self.lazy_load_es_module_from_code(scope, module_specifier, source.code)
+  }
 }
 
 // Clippy thinks the return value doesn't need to be an Option, it's unaware

core/modules/mod.rs

@@ -25,6 +25,7 @@ pub use loaders::TestingModuleLoader;
 pub(crate) use loaders::ExtModuleLoader;
 pub use loaders::ExtModuleLoaderCb;
 pub use loaders::FsModuleLoader;
+pub(crate) use loaders::LazyEsmModuleLoader;
 pub use loaders::ModuleLoader;
 pub use loaders::NoopModuleLoader;
 pub use loaders::StaticModuleLoader;

core/modules/module_map_data.rs

@@ -8,7 +8,10 @@ use crate::modules::ModuleName;
 use crate::modules::ModuleRequest;
 use crate::modules::ModuleType;
 use crate::runtime::SnapshottedData;
+use crate::ExtensionFileSource;
+use std::cell::RefCell;
 use std::collections::HashMap;
+use std::rc::Rc;
 
 /// A symbolic module entity.
 #[derive(Debug, PartialEq)]
@@ -128,6 +131,8 @@ pub(crate) struct ModuleMapData {
   /// to evaluate a "synthetic module".
   pub(crate) synthetic_module_value_store:
     HashMap<v8::Global<v8::Module>, v8::Global<v8::Value>>,
+  pub(crate) lazy_esm_sources:
+    Rc<RefCell<HashMap<&'static str, ExtensionFileSource>>>,
 }
 
 impl ModuleMapData {

core/modules/testdata/lazy_loaded.js

@@ -0,0 +1,6 @@
+export const foo = "foo";
+export const bar = 123;
+export function blah(a) {
+  Deno.core.print(a);
+}
+export default { foo, bar, blah };

core/modules/tests.rs

@@ -410,6 +410,30 @@ fn test_mods() {
   assert_eq!(DISPATCH_COUNT.load(Ordering::Relaxed), 1);
 }
 
+#[test]
+fn test_lazy_loaded_esm() {
+  deno_core::extension!(test_ext, lazy_loaded_esm = [dir "modules/testdata", "lazy_loaded.js"]);
+
+  let mut runtime = JsRuntime::new(RuntimeOptions {
+    extensions: vec![test_ext::init_ops_and_esm()],
+    ..Default::default()
+  });
+
+  runtime
+    .execute_script_static(
+      "setup.js",
+      r#"
+      Deno.core.print("1\n");
+      const module = Deno.core.ops.op_lazy_load_esm("ext:test_ext/lazy_loaded.js");
+      module.blah("hello\n");
+      Deno.core.print(`${JSON.stringify(module)}\n`);
+      const module1 = Deno.core.ops.op_lazy_load_esm("ext:test_ext/lazy_loaded.js");
+      if (module !== module1) throw new Error("should return the same error");
+      "#,
+    )
+    .unwrap();
+}
+
 #[test]
 fn test_json_module() {
   let loader = Rc::new(TestingModuleLoader::new(StaticModuleLoader::new([])));

core/ops_builtin.rs

@@ -47,6 +47,7 @@ deno_core::extension!(
     op_str_byte_length,
     ops_builtin_v8::op_ref_op,
     ops_builtin_v8::op_unref_op,
+    ops_builtin_v8::op_lazy_load_esm,
     ops_builtin_v8::op_set_promise_reject_callback,
     ops_builtin_v8::op_run_microtasks,
     ops_builtin_v8::op_has_tick_scheduled,

core/ops_builtin_v8.rs

@@ -35,6 +35,16 @@ pub fn op_unref_op(scope: &mut v8::HandleScope, promise_id: i32) {
   context_state.borrow_mut().unrefed_ops.insert(promise_id);
 }
 
+#[op2(reentrant)]
+#[global]
+pub fn op_lazy_load_esm(
+  scope: &mut v8::HandleScope,
+  #[string] module_specifier: String,
+) -> Result<v8::Global<v8::Value>, Error> {
+  let module_map_rc = JsRealm::module_map_from(scope);
+  module_map_rc.lazy_load_esm_module(scope, &module_specifier)
+}
+
 #[op2]
 pub fn op_set_promise_reject_callback<'a>(
   scope: &mut v8::HandleScope<'a>,

core/runtime/jsruntime.rs

@@ -934,6 +934,11 @@ impl JsRuntime {
       self.init_cbs(&realm);
 
       for extension in &extensions {
+        // If the extension provides "lazy loaded ES modules" then store them
+        // on the ModuleMap.
+        module_map
+          .add_lazy_loaded_esm_sources(extension.get_lazy_loaded_esm_sources());
+
         let maybe_esm_entry_point = extension.get_esm_entry_point();
 
         for file_source in extension.get_esm_sources() {