Skip to content

Commit

Permalink
broken
Browse files Browse the repository at this point in the history
  • Loading branch information
danlapid committed Oct 17, 2024
1 parent 3349615 commit b3d8a00
Show file tree
Hide file tree
Showing 13 changed files with 213 additions and 55 deletions.
36 changes: 36 additions & 0 deletions src/pyodide/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -295,3 +295,39 @@ genrule(
tools = ["@capnp-cpp//src/capnp:capnp_tool"],
visibility = ["//visibility:public"],
)

capnp_embed(
name = "emscripten_setup_capnp_file_embed",
src = "generated/emscripten_setup.capnp",
deps = ["emscripten_setup_capnp_file"],
)

capnp_embed(
name = "emscripten_setup_js_file_embed",
src = "generated/emscriptenSetup.js",
deps = ["generated/emscriptenSetup"],
)

capnp_embed(
name = "python_stdlib_zip_file_embed",
src = "generated/python_stdlib.zip",
deps = ["python_stdlib.zip@rule"],
)

capnp_embed(
name = "pyodide_asm_wasm_file_embed",
src = "generated/pyodide.asm.wasm",
deps = ["pyodide.asm.wasm@rule"],
)

cc_capnp_library(
name = "emscripten_setup_capnp",
srcs = ["generated/emscripten_setup.capnp"],
visibility = ["//visibility:public"],
deps = [
":emscripten_setup_capnp_file_embed",
":emscripten_setup_js_file_embed",
":pyodide_asm_wasm_file_embed",
":python_stdlib_zip_file_embed",
],
)
12 changes: 11 additions & 1 deletion src/pyodide/internal/pool/builtin_wrappers.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
import type { getRandomValues as getRandomValuesType } from 'pyodide-internal:topLevelEntropy/lib';
import type { default as UnsafeEvalType } from 'internal:unsafe-eval';

let getRandomValuesInner: typeof getRandomValuesType;
export function setGetRandomValues(func: typeof getRandomValuesType) {
getRandomValuesInner = func;
}

export function getRandomValues(Module: Module, arr: Uint8Array): Uint8Array {
return getRandomValuesInner(Module, arr);
}


// We can't import UnsafeEval directly here because it isn't available when setting up Python pool.
// Thus, we inject it from outside via this function.
let UnsafeEval: typeof UnsafeEvalType;
export function setUnsafeEval(mod: typeof UnsafeEvalType) {
UnsafeEval = mod;
}
let lastTime: number;
let lastDelta = 0;
/**
Expand Down
11 changes: 6 additions & 5 deletions src/pyodide/internal/pool/emscriptenSetup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { reportError } from 'pyodide-internal:util';
*/
import { _createPyodideModule } from 'pyodide-internal:generated/pyodide.asm';

export {
import {
setUnsafeEval,
setGetRandomValues,
} from 'pyodide-internal:pool/builtin_wrappers';
Expand Down Expand Up @@ -56,7 +56,7 @@ function getWaitForDynlibs(resolveReadyPromise: PreRunHook): PreRunHook {
* This is a simplified version of the `prepareFileSystem` function here:
* https://github.com/pyodide/pyodide/blob/main/src/js/module.ts
*/
function getPrepareFileSystem(pythonStdlib: Uint8Array): PreRunHook {
function getPrepareFileSystem(pythonStdlib: ArrayBuffer): PreRunHook {
return function prepareFileSystem(Module: Module): void {
try {
const pymajor = Module._py_version_major();
Expand Down Expand Up @@ -118,7 +118,7 @@ function getInstantiateWasm(
*/
function getEmscriptenSettings(
isWorkerd: boolean,
pythonStdlib: Uint8Array,
pythonStdlib: ArrayBuffer,
pyodideWasmModule: WebAssembly.Module
): EmscriptenSettings {
const config: PyodideConfig = {
Expand Down Expand Up @@ -193,7 +193,7 @@ function* featureDetectionMonkeyPatchesContextManager() {
*/
export async function instantiateEmscriptenModule(
isWorkerd: boolean,
pythonStdlib: Uint8Array,
pythonStdlib: ArrayBuffer,
wasmModule: WebAssembly.Module
): Promise<Module> {
const emscriptenSettings = getEmscriptenSettings(
Expand All @@ -210,7 +210,8 @@ export async function instantiateEmscriptenModule(

// Wait until we've executed all the preRun hooks before proceeding
const emscriptenModule = await emscriptenSettings.readyPromise;
ems
emscriptenModule.setUnsafeEval = setUnsafeEval;
emscriptenModule.setGetRandomValues = setGetRandomValues;
return emscriptenModule;
} catch (e) {
console.warn('Error in instantiateEmscriptenModule');
Expand Down
47 changes: 3 additions & 44 deletions src/pyodide/internal/python.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,49 +20,8 @@ import {
} from 'pyodide-internal:topLevelEntropy/lib';
import { default as SetupEmscripten } from 'internal:setup-emscripten';

// import { default as UnsafeEval } from 'internal:unsafe-eval';
import { default as UnsafeEval } from 'internal:unsafe-eval';

// /**
// * This file is a simplified version of the Pyodide loader:
// * https://github.com/pyodide/pyodide/blob/main/src/js/pyodide.ts
// *
// * In particular, it drops the package lock, which disables
// * `pyodide.loadPackage`. In trade we add memory snapshots here.
// */

// /**
// * _createPyodideModule and pyodideWasmModule together are produced by the
// * Emscripten linker
// */
// import pyodideWasmModule from 'pyodide-internal:generated/pyodide.asm.wasm';

// /**
// * The Python and Pyodide stdlib zipped together. The zip format is convenient
// * because Python has a "ziploader" that allows one to import directly from a
// * zip file.
// *
// * The ziploader solves bootstrapping problems around unpacking: Python comes
// * with a bunch of C libs to unpack various archive formats, but they need stuff
// * in this zip file to initialize their runtime state.
// */
// import pythonStdlib from 'pyodide-internal:generated/python_stdlib.zip';
import {
instantiateEmscriptenModule,
setUnsafeEval,
setGetRandomValues,
} from 'pyodide-internal:generated/emscriptenSetup';

// We can't import UnsafeEval directly here because it isn't available when setting up Python pool.
// Thus, we inject it from outside via this function.
let UnsafeEval: typeof UnsafeEvalType;
function setUnsafeEval(mod: typeof UnsafeEvalType) {
UnsafeEval = mod;
}

let getRandomValuesInner: typeof getRandomValuesType;
function setGetRandomValues(func: typeof getRandomValuesType) {
getRandomValuesInner = func;
}
/**
* After running `instantiateEmscriptenModule` but before calling into any C
* APIs, we call this function. If `MEMORY` is defined, then we will have passed
Expand Down Expand Up @@ -100,8 +59,8 @@ export async function loadPyodide(
Module.API.config.indexURL = indexURL;
Module.API.config.resolveLockFilePromise!(lockfile);
}
setUnsafeEval(UnsafeEval);
setGetRandomValues(getRandomValues);
Module.setUnsafeEval(UnsafeEval);
Module.setGetRandomValues(getRandomValues);
await enterJaegerSpan('prepare_wasm_linear_memory', () =>
prepareWasmLinearMemory(Module)
);
Expand Down
6 changes: 3 additions & 3 deletions src/pyodide/pyodide_extra.capnp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
@0x96c290dbf479ac0c;

const pythonEntrypoint :Text = embed "generated/python-entrypoint.js";
const pyodidePackagesTar :Data = embed "generated/pyodide_packages.tar";
const pyodideLock :Text = embed "generated/pyodide-lock.json";
const pythonEntrypoint :Text = embed "python-entrypoint.js";
const pyodidePackagesTar :Data = embed "pyodide_packages.tar";
const pyodideLock :Text = embed "pyodide-lock.json";
3 changes: 2 additions & 1 deletion src/pyodide/python-entrypoint-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,14 @@ import {
import {
IS_TRACING,
IS_WORKERD,
LOCKFILE,
MAIN_MODULE_NAME,
WORKERD_INDEX_URL,
} from 'pyodide-internal:metadata';
import { reportError } from 'pyodide-internal:util';
import { default as Limiter } from 'pyodide-internal:limiter';
import { entropyBeforeRequest } from 'pyodide-internal:topLevelEntropy/lib';
import { loadPackages } from 'pyodide-internal:loadPackage';
import type { default as UnsafeEvalType } from 'internal:unsafe-eval';

function pyimportMainModule(pyodide: Pyodide): PyModule {
if (!MAIN_MODULE_NAME.endsWith('.py')) {
Expand Down
4 changes: 4 additions & 0 deletions src/pyodide/types/emscripten.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,4 +68,8 @@ interface Module {
addRunDependency(x: string): void;
removeRunDependency(x: string): void;
noInitialRun: boolean;
setUnsafeEval(mod: typeof import('internal:unsafe-eval').default): void;
setGetRandomValues(
func: typeof import('pyodide-internal:topLevelEntropy/lib').getRandomValues
): void;
}
7 changes: 7 additions & 0 deletions src/workerd/api/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ filegroup(
"hyperdrive.c++",
"pyodide.c++",
"pyodide/pyodide.c++",
"pyodide/setup-emscripten.c++",
"rtti.c++",
"url.c++",
"util.c++",
Expand Down Expand Up @@ -50,6 +51,7 @@ wd_cc_library(
hdrs = [
"modules.h",
"rtti.h",
"//src/pyodide:generated/emscripten_setup.capnp.h",
"//src/pyodide:generated/pyodide_extra.capnp.h",
"//src/workerd/server:workerd.capnp.h",
],
Expand All @@ -58,6 +60,7 @@ wd_cc_library(
":html-rewriter",
":hyperdrive",
"//src/pyodide",
"//src/pyodide:emscripten_setup_capnp",
"//src/pyodide:pyodide_extra_capnp",
"//src/workerd/api/node",
"//src/workerd/io",
Expand Down Expand Up @@ -93,15 +96,19 @@ wd_cc_library(
name = "pyodide",
srcs = [
"pyodide/pyodide.c++",
"pyodide/setup-emscripten.c++",
],
hdrs = [
"pyodide/pyodide.h",
"pyodide/setup-emscripten.h",
"//src/pyodide:generated/emscripten_setup.capnp.h",
"//src/pyodide:generated/pyodide_extra.capnp.h",
],
implementation_deps = ["//src/workerd/util:string-buffer"],
visibility = ["//visibility:public"],
deps = [
"//src/pyodide",
"//src/pyodide:emscripten_setup_capnp",
"//src/pyodide:pyodide_extra_capnp",
"//src/workerd/io",
"//src/workerd/jsg",
Expand Down
3 changes: 3 additions & 0 deletions src/workerd/api/modules.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#include <workerd/api/node/node.h>
#include <workerd/api/pyodide/pyodide.h>
#include <workerd/api/pyodide/setup-emscripten.h>
#include <workerd/api/rtti.h>
#include <workerd/api/sockets.h>
#include <workerd/api/unsafe.h>
Expand All @@ -21,6 +22,7 @@ template <class Registry>
void registerModules(Registry& registry, auto featureFlags) {
node::registerNodeJsCompatModules(registry, featureFlags);
if (featureFlags.getPythonWorkers()) {
pyodide::registerSetupEmscriptenModule(registry, featureFlags);
pyodide::registerPyodideModules(registry, featureFlags);
}
registerUnsafeModules(registry, featureFlags);
Expand Down Expand Up @@ -48,6 +50,7 @@ void registerBuiltinModules(jsg::modules::ModuleRegistry::Builder& builder, auto
}

if (featureFlags.getPythonWorkers()) {
builder.add(pyodide::getInternalSetupEmscriptenModuleBundle<TypeWrapper>(featureFlags));
builder.add(pyodide::getExternalPyodideModuleBundle(featureFlags));
builder.add(pyodide::getInternalPyodideModuleBundle(featureFlags));
}
Expand Down
63 changes: 63 additions & 0 deletions src/workerd/api/pyodide/setup-emscripten.c++
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#include "setup-emscripten.h"

#include <pyodide/generated/emscripten_setup.capnp.h>

namespace workerd::api::pyodide {

jsg::JsValue SetupEmscripten::getModule(jsg::Lock& js) {
jsg::JsValue jsval(pyodide::initializeEmscriptenRuntime(js));
module_ = jsval.addRef(js);
return KJ_ASSERT_NONNULL(module_).getHandle(js);
}

v8::Local<v8::Value> initializeEmscriptenRuntime(jsg::Lock& js) {
js.setAllowEval(true);
KJ_DEFER(js.setAllowEval(false));

v8::Local<v8::String> contentStr =
jsg::v8Str(js.v8Isolate, EMSCRIPTEN_SETUP->getCode().asArray());
v8::ScriptOrigin origin(
jsg::v8StrIntern(js.v8Isolate, "pyodide-internal:generated/emscriptenSetup"), 0, 0, false, -1,
v8::Local<v8::Value>(), false, false, true);
v8::ScriptCompiler::Source source(contentStr, origin);

v8::Local<v8::Module> module;
if (!v8::ScriptCompiler::CompileModule(js.v8Isolate, &source).ToLocal(&module)) {
KJ_FAIL_ASSERT("Emscripten Setup code didn't parse");
}
// instantiateEmscriptenModule2(js, module);
jsg::instantiateModule(js, module);

auto handle = jsg::check(module->Evaluate(js.v8Context()));
KJ_ASSERT(handle->IsPromise());
auto prom = handle.As<v8::Promise>();
KJ_ASSERT(prom->State() != v8::Promise::PromiseState::kPending);
KJ_ASSERT(module->GetStatus() != v8::Module::kErrored);
auto result =
js.v8Get(module->GetModuleNamespace().As<v8::Object>(), "instantiateEmscriptenModule"_kj);

auto src = EMSCRIPTEN_SETUP->getPyodideAsmWasm();
auto wasmModule = jsg::check(v8::WasmModuleObject::Compile(
js.v8Isolate, v8::MemorySpan<const uint8_t>(src.begin(), src.size())));
auto arg2Buffer =
v8::ArrayBuffer::New(js.v8Isolate, EMSCRIPTEN_SETUP->getPythonStdlibZip().size(),
v8::BackingStoreInitializationMode::kUninitialized);
memcpy(arg2Buffer->Data(), EMSCRIPTEN_SETUP->getPythonStdlibZip().begin(),
EMSCRIPTEN_SETUP->getPythonStdlibZip().size());

v8::LocalVector<v8::Value> argv(js.v8Isolate, 3);
argv[0] = v8::Boolean::New(js.v8Isolate, true);
argv[1] = kj::mv(arg2Buffer);
argv[2] = kj::mv(wasmModule);
auto funcres = jsg::check(
result.As<v8::Function>()->Call(js.v8Context(), js.v8Null(), argv.size(), argv.data()));
KJ_ASSERT(funcres->IsPromise());
auto promise = funcres.As<v8::Promise>();
if (promise->State() == v8::Promise::PromiseState::kPending) {
js.runMicrotasks();
}
KJ_ASSERT(promise->State() == v8::Promise::PromiseState::kFulfilled);
auto promresult = promise->Result();
return promresult;
}
} // namespace workerd::api::pyodide
55 changes: 55 additions & 0 deletions src/workerd/api/pyodide/setup-emscripten.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#pragma once

#include <workerd/io/io-context.h>
#include <workerd/jsg/jsg.h>
#include <workerd/jsg/modules-new.h>
#include <workerd/jsg/url.h>

namespace workerd::api::pyodide {

class EmscriptenModule: public jsg::Object {
public:
explicit EmscriptenModule(jsg::Lock& js) {};
JSG_STRUCT(EmscriptenModule);

private:
};

using instantiateEmscriptenModuleFunction = jsg::Function<jsg::Promise<jsg::JsRef<jsg::JsValue>>(
jsg::JsBoolean, jsg::JsString, jsg::JsString)>;

class SetupEmscripten: public jsg::Object {
public:
SetupEmscripten() {};
SetupEmscripten(jsg::Lock& js, const jsg::Url&) {}

jsg::JsValue getModule(jsg::Lock& js);

JSG_RESOURCE_TYPE(SetupEmscripten) {
JSG_METHOD(getModule);
}

private:
kj::Maybe<jsg::JsRef<jsg::JsValue>> module_;
};

#define EW_SETUP_EMSCRIPTEN_ISOLATE_TYPES api::pyodide::SetupEmscripten

template <class Registry>
void registerSetupEmscriptenModule(Registry& registry, auto featureFlags) {
registry.template addBuiltinModule<SetupEmscripten>(
"internal:setup-emscripten", workerd::jsg::ModuleRegistry::Type::INTERNAL);
}

template <typename TypeWrapper>
kj::Own<jsg::modules::ModuleBundle> getInternalSetupEmscriptenModuleBundle(auto featureFlags) {
jsg::modules::ModuleBundle::BuiltinBuilder builder(
jsg::modules::ModuleBundle::BuiltinBuilder::Type::BUILTIN_ONLY);
static const auto kSpecifier = "internal:setup-emscripten"_url;
builder.addObject<SetupEmscripten, TypeWrapper>(kSpecifier);
return builder.finish();
}

v8::Local<v8::Value> initializeEmscriptenRuntime(jsg::Lock&);

} // namespace workerd::api::pyodide
Loading

0 comments on commit b3d8a00

Please sign in to comment.