Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Removes usage of embedded PYODIDE_LOCK in C++ code. #3328

Merged
merged 1 commit into from
Jan 14, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion src/workerd/api/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,6 @@ wd_cc_library(
hdrs = [
"pyodide/pyodide.h",
"pyodide/setup-emscripten.h",
"//src/pyodide:generated/pyodide_extra.capnp.h",
],
implementation_deps = ["//src/workerd/util:string-buffer"],
visibility = ["//visibility:public"],
Expand Down
2 changes: 0 additions & 2 deletions src/workerd/api/pyodide/pyodide.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
#include <workerd/server/workerd.capnp.h>
#include <workerd/util/autogate.h>

#include <pyodide/generated/pyodide_extra.capnp.h>

#include <capnp/serialize.h>
#include <kj/array.h>
#include <kj/common.h>
Expand Down
1 change: 1 addition & 0 deletions src/workerd/server/tests/python/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ py_wd_test("subdirectory")

py_wd_test(
"sdk",
# TODO(EW-8976): Disabled in dev due to test getting stuck
python_flags = ["0.26.0a2"],
tags = [
# TODO(someday): Fix asan failure for this, see https://github.com/cloudflare/workerd/pull/3140#discussion_r1858273318
Expand Down
4 changes: 3 additions & 1 deletion src/workerd/server/workerd-api.c++
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@
#include <workerd/util/thread-scopes.h>
#include <workerd/util/use-perfetto-categories.h>

#include <pyodide/generated/pyodide_extra.capnp.h>

#include <kj/compat/http.h>
#include <kj/compat/tls.h>
#include <kj/compat/url.h>
Expand Down Expand Up @@ -157,6 +159,7 @@ void writePyodideBundleFileToDisk(const kj::Maybe<kj::Own<const kj::Directory>>&
replacer->commit();
}
}
} // namespace

kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version) {
Expand Down Expand Up @@ -213,7 +216,6 @@ kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
KJ_LOG(INFO, "Loaded Pyodide package from internet");
return pyConfig.pyodideBundleManager.getPyodideBundle(version);
}
} // namespace

struct WorkerdApi::Impl final {
kj::Own<CompatibilityFlags::Reader> features;
Expand Down
3 changes: 3 additions & 0 deletions src/workerd/server/workerd-api.h
Original file line number Diff line number Diff line change
Expand Up @@ -265,4 +265,7 @@ class WorkerdApi final: public Worker::Api {
capnp::List<config::Extension>::Reader extensions) const;
};

kj::Maybe<jsg::Bundle::Reader> fetchPyodideBundle(
const api::pyodide::PythonConfig& pyConfig, kj::StringPtr version);

} // namespace workerd::server
22 changes: 20 additions & 2 deletions src/workerd/server/workerd.c++
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
// https://opensource.org/licenses/Apache-2.0

#include "server.h"
#include "workerd-api.h"

#include <workerd/io/compatibility-date.capnp.h>
#include <workerd/io/compatibility-date.h>
#include <workerd/io/supported-compatibility-date.capnp.h>
#include <workerd/jsg/setup.h>
#include <workerd/rust/cxx-integration/lib.rs.h>
Expand All @@ -15,7 +17,6 @@

#include <fcntl.h>
#include <openssl/rand.h>
#include <pyodide/generated/pyodide_extra.capnp.h>
#include <sys/stat.h>

#include <capnp/dynamic.h>
Expand Down Expand Up @@ -807,7 +808,24 @@ class CliMain final: public SchemaFileImpl::ErrorReporter {
context, getVersionString(), "Outputs the package lock file used by Pyodide.");
return builder
.callAfterParsing([]() -> kj::MainBuilder::Validity {
printf("%s\n", PYODIDE_LOCK->cStr());
static const PythonConfig config{
.packageDiskCacheRoot = kj::none,
.pyodideDiskCacheRoot = kj::none,
dom96 marked this conversation as resolved.
Show resolved Hide resolved
.createSnapshot = false,
.createBaselineSnapshot = false,
};

capnp::MallocMessageBuilder message;
// TODO(EW-8977): Implement option to specify python worker flags.
dom96 marked this conversation as resolved.
Show resolved Hide resolved
auto features = message.getRoot<CompatibilityFlags>();
features.setPythonWorkers(true);
auto pythonRelease = KJ_ASSERT_NONNULL(getPythonSnapshotRelease(features));
auto version = getPythonBundleName(pythonRelease);
KJ_ASSERT_NONNULL(fetchPyodideBundle(config, version), "Failed to get Pyodide bundle");

auto lock = KJ_ASSERT_NONNULL(config.pyodideBundleManager.getPyodideLock(pythonRelease));

printf("%s\n", lock.cStr());
fflush(stdout);
return true;
}).build();
Expand Down
Loading