broken

src/pyodide/BUILD.bazel

@@ -295,3 +295,39 @@ genrule(
     tools = ["@capnp-cpp//src/capnp:capnp_tool"],
     visibility = ["//visibility:public"],
 )
+
+capnp_embed(
+    name = "emscripten_setup_capnp_file_embed",
+    src = "generated/emscripten_setup.capnp",
+    deps = ["emscripten_setup_capnp_file"],
+)
+
+capnp_embed(
+    name = "emscripten_setup_js_file_embed",
+    src = "generated/emscriptenSetup.js",
+    deps = ["generated/emscriptenSetup"],
+)
+
+capnp_embed(
+    name = "python_stdlib_zip_file_embed",
+    src = "generated/python_stdlib.zip",
+    deps = ["python_stdlib.zip@rule"],
+)
+
+capnp_embed(
+    name = "pyodide_asm_wasm_file_embed",
+    src = "generated/pyodide.asm.wasm",
+    deps = ["pyodide.asm.wasm@rule"],
+)
+
+cc_capnp_library(
+    name = "emscripten_setup_capnp",
+    srcs = ["generated/emscripten_setup.capnp"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":emscripten_setup_capnp_file_embed",
+        ":emscripten_setup_js_file_embed",
+        ":pyodide_asm_wasm_file_embed",
+        ":python_stdlib_zip_file_embed",
+    ],
+)

src/pyodide/internal/pool/builtin_wrappers.ts

@@ -1,11 +1,21 @@
 import type { getRandomValues as getRandomValuesType } from 'pyodide-internal:topLevelEntropy/lib';
+import type { default as UnsafeEvalType } from 'internal:unsafe-eval';
 
+let getRandomValuesInner: typeof getRandomValuesType;
+export function setGetRandomValues(func: typeof getRandomValuesType) {
+  getRandomValuesInner = func;
+}
 
 export function getRandomValues(Module: Module, arr: Uint8Array): Uint8Array {
   return getRandomValuesInner(Module, arr);
 }
 
-
+// We can't import UnsafeEval directly here because it isn't available when setting up Python pool.
+// Thus, we inject it from outside via this function.
+let UnsafeEval: typeof UnsafeEvalType;
+export function setUnsafeEval(mod: typeof UnsafeEvalType) {
+  UnsafeEval = mod;
+}
 let lastTime: number;
 let lastDelta = 0;
 /**

src/pyodide/internal/pool/emscriptenSetup.ts

@@ -7,17 +7,17 @@
 
 import { reportError } from 'pyodide-internal:util';
 
+import {
+  setUnsafeEval,
+  setGetRandomValues,
+} from 'pyodide-internal:pool/builtin_wrappers';
+
 /**
  * _createPyodideModule and pyodideWasmModule together are produced by the
  * Emscripten linker
  */
 import { _createPyodideModule } from 'pyodide-internal:generated/pyodide.asm';
 
-export {
-  setUnsafeEval,
-  setGetRandomValues,
-} from 'pyodide-internal:pool/builtin_wrappers';
-
 /**
  * A preRun hook. Make sure environment variables are visible at runtime.
  */
@@ -56,7 +56,7 @@ function getWaitForDynlibs(resolveReadyPromise: PreRunHook): PreRunHook {
  * This is a simplified version of the `prepareFileSystem` function here:
  * https://github.com/pyodide/pyodide/blob/main/src/js/module.ts
  */
-function getPrepareFileSystem(pythonStdlib: Uint8Array): PreRunHook {
+function getPrepareFileSystem(pythonStdlib: ArrayBuffer): PreRunHook {
   return function prepareFileSystem(Module: Module): void {
     try {
       const pymajor = Module._py_version_major();
@@ -99,11 +99,13 @@ function getInstantiateWasm(
     ) => void
   ): WebAssembly.Exports {
     (async function () {
+      console.log('instantiateWasm1');
       // Instantiate pyodideWasmModule with wasmImports
       const instance = await WebAssembly.instantiate(
         pyodideWasmModule,
         wasmImports
       );
+      console.log('instantiateWasm1');
       successCallback(instance, pyodideWasmModule);
     })();
 
@@ -118,7 +120,7 @@ function getInstantiateWasm(
  */
 function getEmscriptenSettings(
   isWorkerd: boolean,
-  pythonStdlib: Uint8Array,
+  pythonStdlib: ArrayBuffer,
   pyodideWasmModule: WebAssembly.Module
 ): EmscriptenSettings {
   const config: PyodideConfig = {
@@ -193,24 +195,30 @@ function* featureDetectionMonkeyPatchesContextManager() {
  */
 export async function instantiateEmscriptenModule(
   isWorkerd: boolean,
-  pythonStdlib: Uint8Array,
+  pythonStdlib: ArrayBuffer,
   wasmModule: WebAssembly.Module
 ): Promise<Module> {
+  console.log('instantiateEmscriptenModule js 1');
   const emscriptenSettings = getEmscriptenSettings(
     isWorkerd,
     pythonStdlib,
     wasmModule
   );
   try {
     for (const _ of featureDetectionMonkeyPatchesContextManager()) {
+      console.log('instantiateEmscriptenModule js 4');
       // Ignore the returned promise, it won't resolve until we're done preloading dynamic
       // libraries.
       const _promise = _createPyodideModule(emscriptenSettings);
     }
+    console.log('instantiateEmscriptenModule js 5');
 
     // Wait until we've executed all the preRun hooks before proceeding
     const emscriptenModule = await emscriptenSettings.readyPromise;
-    ems
+    console.log('instantiateEmscriptenModule js 6');
+    emscriptenModule.setUnsafeEval = setUnsafeEval;
+    emscriptenModule.setGetRandomValues = setGetRandomValues;
+    console.log('instantiateEmscriptenModule js 7');
     return emscriptenModule;
   } catch (e) {
     console.warn('Error in instantiateEmscriptenModule');

src/pyodide/internal/python.ts

@@ -20,49 +20,8 @@ import {
 } from 'pyodide-internal:topLevelEntropy/lib';
 import { default as SetupEmscripten } from 'internal:setup-emscripten';
 
-// import { default as UnsafeEval } from 'internal:unsafe-eval';
+import { default as UnsafeEval } from 'internal:unsafe-eval';
 
-// /**
-//  * This file is a simplified version of the Pyodide loader:
-//  * https://github.com/pyodide/pyodide/blob/main/src/js/pyodide.ts
-//  *
-//  * In particular, it drops the package lock, which disables
-//  * `pyodide.loadPackage`. In trade we add memory snapshots here.
-//  */
-
-// /**
-//  * _createPyodideModule and pyodideWasmModule together are produced by the
-//  * Emscripten linker
-//  */
-// import pyodideWasmModule from 'pyodide-internal:generated/pyodide.asm.wasm';
-
-// /**
-//  * The Python and Pyodide stdlib zipped together. The zip format is convenient
-//  * because Python has a "ziploader" that allows one to import directly from a
-//  * zip file.
-//  *
-//  * The ziploader solves bootstrapping problems around unpacking: Python comes
-//  * with a bunch of C libs to unpack various archive formats, but they need stuff
-//  * in this zip file to initialize their runtime state.
-//  */
-// import pythonStdlib from 'pyodide-internal:generated/python_stdlib.zip';
-import {
-  instantiateEmscriptenModule,
-  setUnsafeEval,
-  setGetRandomValues,
-} from 'pyodide-internal:generated/emscriptenSetup';
-
-// We can't import UnsafeEval directly here because it isn't available when setting up Python pool.
-// Thus, we inject it from outside via this function.
-let UnsafeEval: typeof UnsafeEvalType;
-function setUnsafeEval(mod: typeof UnsafeEvalType) {
-  UnsafeEval = mod;
-}
-
-let getRandomValuesInner: typeof getRandomValuesType;
-function setGetRandomValues(func: typeof getRandomValuesType) {
-  getRandomValuesInner = func;
-}
 /**
  * After running `instantiateEmscriptenModule` but before calling into any C
  * APIs, we call this function. If `MEMORY` is defined, then we will have passed
@@ -95,13 +54,18 @@ export async function loadPyodide(
   lockfile: PackageLock,
   indexURL: string
 ): Promise<Pyodide> {
+  console.log('getting module');
   const Module = await SetupEmscripten.getModule();
+  console.log('got module');
   if (isWorkerd) {
+    console.log('im workerd using module');
     Module.API.config.indexURL = indexURL;
     Module.API.config.resolveLockFilePromise!(lockfile);
   }
-  setUnsafeEval(UnsafeEval);
-  setGetRandomValues(getRandomValues);
+  console.log('im setting stuff module');
+  Module.setUnsafeEval(UnsafeEval);
+  Module.setGetRandomValues(getRandomValues);
+  console.log('im jaeger');
   await enterJaegerSpan('prepare_wasm_linear_memory', () =>
     prepareWasmLinearMemory(Module)
   );

src/pyodide/pyodide_extra.capnp

@@ -1,5 +1,5 @@
 @0x96c290dbf479ac0c;
 
-const pythonEntrypoint :Text = embed "generated/python-entrypoint.js";
-const pyodidePackagesTar :Data = embed "generated/pyodide_packages.tar";
-const pyodideLock :Text = embed "generated/pyodide-lock.json";
+const pythonEntrypoint :Text = embed "python-entrypoint.js";
+const pyodidePackagesTar :Data = embed "pyodide_packages.tar";
+const pyodideLock :Text = embed "pyodide-lock.json";

src/pyodide/python-entrypoint-helper.ts

@@ -14,13 +14,14 @@ import {
 import {
   IS_TRACING,
   IS_WORKERD,
+  LOCKFILE,
   MAIN_MODULE_NAME,
+  WORKERD_INDEX_URL,
 } from 'pyodide-internal:metadata';
 import { reportError } from 'pyodide-internal:util';
 import { default as Limiter } from 'pyodide-internal:limiter';
 import { entropyBeforeRequest } from 'pyodide-internal:topLevelEntropy/lib';
 import { loadPackages } from 'pyodide-internal:loadPackage';
-import type { default as UnsafeEvalType } from 'internal:unsafe-eval';
 
 function pyimportMainModule(pyodide: Pyodide): PyModule {
   if (!MAIN_MODULE_NAME.endsWith('.py')) {

src/pyodide/types/emscripten.d.ts

@@ -68,4 +68,8 @@ interface Module {
   addRunDependency(x: string): void;
   removeRunDependency(x: string): void;
   noInitialRun: boolean;
+  setUnsafeEval(mod: typeof import('internal:unsafe-eval').default): void;
+  setGetRandomValues(
+    func: typeof import('pyodide-internal:topLevelEntropy/lib').getRandomValues
+  ): void;
 }

src/workerd/api/BUILD.bazel

@@ -16,6 +16,7 @@ filegroup(
             "hyperdrive.c++",
             "pyodide.c++",
             "pyodide/pyodide.c++",
+            "pyodide/setup-emscripten.c++",
             "rtti.c++",
             "url.c++",
             "util.c++",
@@ -50,6 +51,7 @@ wd_cc_library(
     hdrs = [
         "modules.h",
         "rtti.h",
+        "//src/pyodide:generated/emscripten_setup.capnp.h",
         "//src/pyodide:generated/pyodide_extra.capnp.h",
         "//src/workerd/server:workerd.capnp.h",
     ],
@@ -58,6 +60,7 @@ wd_cc_library(
         ":html-rewriter",
         ":hyperdrive",
         "//src/pyodide",
+        "//src/pyodide:emscripten_setup_capnp",
         "//src/pyodide:pyodide_extra_capnp",
         "//src/workerd/api/node",
         "//src/workerd/io",
@@ -93,15 +96,19 @@ wd_cc_library(
     name = "pyodide",
     srcs = [
         "pyodide/pyodide.c++",
+        "pyodide/setup-emscripten.c++",
     ],
     hdrs = [
         "pyodide/pyodide.h",
+        "pyodide/setup-emscripten.h",
+        "//src/pyodide:generated/emscripten_setup.capnp.h",
         "//src/pyodide:generated/pyodide_extra.capnp.h",
     ],
     implementation_deps = ["//src/workerd/util:string-buffer"],
     visibility = ["//visibility:public"],
     deps = [
         "//src/pyodide",
+        "//src/pyodide:emscripten_setup_capnp",
         "//src/pyodide:pyodide_extra_capnp",
         "//src/workerd/io",
         "//src/workerd/jsg",

src/workerd/api/modules.h

@@ -6,6 +6,7 @@
 
 #include <workerd/api/node/node.h>
 #include <workerd/api/pyodide/pyodide.h>
+#include <workerd/api/pyodide/setup-emscripten.h>
 #include <workerd/api/rtti.h>
 #include <workerd/api/sockets.h>
 #include <workerd/api/unsafe.h>
@@ -21,6 +22,7 @@ template <class Registry>
 void registerModules(Registry& registry, auto featureFlags) {
   node::registerNodeJsCompatModules(registry, featureFlags);
   if (featureFlags.getPythonWorkers()) {
+    pyodide::registerSetupEmscriptenModule(registry, featureFlags);
     pyodide::registerPyodideModules(registry, featureFlags);
   }
   registerUnsafeModules(registry, featureFlags);
@@ -48,6 +50,7 @@ void registerBuiltinModules(jsg::modules::ModuleRegistry::Builder& builder, auto
   }
 
   if (featureFlags.getPythonWorkers()) {
+    builder.add(pyodide::getInternalSetupEmscriptenModuleBundle<TypeWrapper>(featureFlags));
     builder.add(pyodide::getExternalPyodideModuleBundle(featureFlags));
     builder.add(pyodide::getInternalPyodideModuleBundle(featureFlags));
   }