diff --git a/.github/workflows/release-python-runtime.yml b/.github/workflows/release-python-runtime.yml new file mode 100644 index 00000000000..59e3724797d --- /dev/null +++ b/.github/workflows/release-python-runtime.yml @@ -0,0 +1,74 @@ +name: Build Python Runtime + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +on: + workflow_dispatch: + inputs: + tag: + description: The release tag/version string + dry-run: + description: Actually upload or just test build step? + default: false + type: boolean +jobs: + version: + outputs: + version: ${{ steps.echo.outputs.version }} + # version job uses ubuntu 24.04, this way we don't have to install the updated clang while + # the build job uses 20.04 for libc compatibility. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - id: echo + run: | + echo "::set-output name=version::1.$(cat src/workerd/io/supported-compatibility-date.txt | tr -d '-').0" + + build: + runs-on: ubuntu-20.04 + name: build Python runtime + steps: + - uses: actions/checkout@v4 + with: + show-progress: false + - name: Setup Linux + if: runner.os == 'Linux' + run: | + export DEBIAN_FRONTEND=noninteractive + wget https://apt.llvm.org/llvm.sh + sed -i '/apt-get install/d' llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 15 + sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev + echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc + echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc + - name: Configure download mirrors + shell: bash + run: | + if [ ! -z "${{ secrets.WORKERS_MIRROR_URL }}" ] ; then + # Strip comment in front of WORKERS_MIRROR_URL, then substitute secret to use it. + sed -e '/WORKERS_MIRROR_URL/ { s@# *@@; s@WORKERS_MIRROR_URL@${{ secrets.WORKERS_MIRROR_URL }}@; }' -i.bak WORKSPACE + fi + - name: Bazel build + # Strip debug info here – we don't generate debug info but some is pulled in from external + # static libraries, for example the Rust STL. This is equivalent to the -Wl,-S linker + # option, symbols will not be removed. + run: | + bazelisk build --config=release_linux //src/pyodide:pyodide.capnp.bin@rule + cp bazel-bin/src/pyodide/pyodide.capnp.bin pyodide-${{ inputs.tag }}.capnp.bin + + - id: "GCS-auth" + if: ${{ !inputs.dry-run }} + uses: 'google-github-actions/auth@v2' + with: + service_account: 'python-packages-writer@edgeworkers.iam.gserviceaccount.com' + workload_identity_provider: "projects/672150504942/locations/global/workloadIdentityPools/github-workerd/providers/github-workerd" + - id: 'upload-file' + if: ${{ !inputs.dry-run }} + uses: 'google-github-actions/upload-cloud-storage@v2' + with: + path: 'pyodide-${{ inputs.tag }}.capnp.bin' + destination: 'cloudflare-edgeworker-python-packages/python-runtime-capnp-bin' + parent: false diff --git a/build/wd_js_bundle.bzl b/build/wd_js_bundle.bzl index e76855301f6..30f369a5073 100644 --- a/build/wd_js_bundle.bzl +++ b/build/wd_js_bundle.bzl @@ -114,7 +114,7 @@ def _copy_modules(modules, declarations): result[new_filename] = modules[m] return result, declarations_result -def wd_js_bundle( +def wd_js_bundle_capnp( name, import_name, schema_id, @@ -146,6 +146,7 @@ def wd_js_bundle( internal_json_modules: list of json source files declarations: d.ts label set deps: dependency list + Returns: The set of data dependencies """ builtin_modules_dict = { m: "{}:{}".format(import_name, _to_name(m)) @@ -201,7 +202,7 @@ def wd_js_bundle( gen_api_bundle_capnpn( name = name + "@gen", - out = name + ".capnp", + out = name, schema_id = schema_id, const_name = import_name + "Bundle", builtin_modules = builtin_modules_dict, @@ -213,7 +214,11 @@ def wd_js_bundle( data = data, deps = deps, ) + return data + +def wd_js_bundle(name, import_name, *args, **kwargs): + data = wd_js_bundle_capnp(name + ".capnp", import_name, *args, **kwargs) cc_capnp_library( name = name, srcs = [name + ".capnp"], diff --git a/build/wd_ts_bundle.bzl b/build/wd_ts_bundle.bzl index 1c25e88a751..aaed27a3e79 100644 --- a/build/wd_ts_bundle.bzl +++ b/build/wd_ts_bundle.bzl @@ -1,6 +1,7 @@ load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project") load("@npm//:eslint/package_json.bzl", eslint_bin = "bin") -load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle") +load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle_capnp") +load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library") def _to_js(file_name): if file_name.endswith(".ts"): @@ -10,7 +11,7 @@ def _to_js(file_name): def _to_d_ts(file_name): return file_name.removesuffix(".ts") + ".d.ts" -def wd_ts_bundle( +def wd_ts_bundle_capnp( name, import_name, schema_id, @@ -59,7 +60,7 @@ def wd_ts_bundle( deps = deps, ) - wd_js_bundle( + data = wd_js_bundle_capnp( name = name, import_name = import_name, # builtin modules are accessible under ":" name @@ -96,3 +97,17 @@ def wd_ts_bundle( "//conditions:default": [], }), ) + return data + + +def wd_ts_bundle(name, import_name, *args, **kwargs): + data = wd_ts_bundle_capnp(name + ".capnp", import_name, *args, **kwargs) + cc_capnp_library( + name = name, + srcs = [name + ".capnp"], + strip_include_prefix = "", + visibility = ["//visibility:public"], + data = data, + deps = ["@workerd//src/workerd/jsg:modules_capnp"], + include_prefix = import_name, + ) diff --git a/src/pyodide/BUILD.bazel b/src/pyodide/BUILD.bazel index 738fdbeace7..6075473075b 100644 --- a/src/pyodide/BUILD.bazel +++ b/src/pyodide/BUILD.bazel @@ -3,7 +3,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library") load("//:build/capnp_embed.bzl", "capnp_embed") -load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle") +load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle_capnp") copy_file( name = "pyodide_packages_archive", @@ -146,8 +146,8 @@ expand_template( template = "@pyodide//:pyodide/pyodide.asm.js", ) -wd_ts_bundle( - name = "pyodide", +data = wd_ts_bundle_capnp( + name = "pyodide.capnp", modules = ["python-entrypoint-helper.ts"], import_name = "pyodide", internal_data_modules = ["generated/python_stdlib.zip"] + glob([ @@ -179,3 +179,31 @@ wd_ts_bundle( "pyodide-bucket.json@rule", ], ) + +cc_capnp_library( + name = "pyodide", + srcs = ["pyodide.capnp"], + strip_include_prefix = "", + visibility = ["//visibility:public"], + data = data, + deps = ["@workerd//src/workerd/jsg:modules_capnp"], + include_prefix = "pyodide", +) + + +genrule( + name = "pyodide.capnp.bin@rule", + tools = ["@capnp-cpp//src/capnp:capnp_tool"], + srcs = ["pyodide.capnp", "//src/workerd/jsg:modules.capnp"] + data, + outs = ["pyodide.capnp.bin"], + visibility = ["//visibility:public"], + cmd = " ".join([ + "$(execpath @capnp-cpp//src/capnp:capnp_tool)", + "eval", + "$(location :pyodide.capnp)", + "pyodideBundle", + "-I src/", + "-o binary", + "> $@", + ]) +) diff --git a/src/workerd/jsg/BUILD.bazel b/src/workerd/jsg/BUILD.bazel index 9e8980ffb57..212c79ae2dc 100644 --- a/src/workerd/jsg/BUILD.bazel +++ b/src/workerd/jsg/BUILD.bazel @@ -4,6 +4,8 @@ load("//:build/kj_test.bzl", "kj_test") load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library") load("//:build/wd_cc_library.bzl", "wd_cc_library") +exports_files(["modules.capnp"]) + wd_cc_library( name = "jsg", srcs = [ diff --git a/src/workerd/server/BUILD.bazel b/src/workerd/server/BUILD.bazel index 5f1822024fd..2f2ec766697 100644 --- a/src/workerd/server/BUILD.bazel +++ b/src/workerd/server/BUILD.bazel @@ -1,5 +1,6 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("//:build/kj_test.bzl", "kj_test") load("//:build/wd_cc_binary.bzl", "wd_cc_binary") load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library") @@ -179,3 +180,11 @@ kj_test( "@ssl", ], ) + + +copy_file( + name = "pyodide.capnp.bin@rule", + src = "//src/pyodide:pyodide.capnp.bin@rule", + out = "pyodide.capnp.bin", +) +