Add gha to upload Pyodide javascript bundle to GCS

This adds a github action that makes a binary file out of the pyodide capnproto
bundle and uploads it to GCS.

TODO:
1. Use the uploaded file (see #2430)
2. Move the pyodide js code and this script to pyodide-build-scripts repo.

.github/workflows/release-python-runtime.yml

@@ -0,0 +1,74 @@
+name: Build Python Runtime
+
+permissions:
+    id-token: write # This is required for requesting the JWT
+    contents: read  # This is required for actions/checkout
+
+on:
+    workflow_dispatch:
+        inputs:
+            tag:
+                description: The release tag/version string
+            dry-run:
+                description: Actually upload or just test build step?
+                default: false
+                type: boolean
+jobs:
+  version:
+    outputs:
+      version: ${{ steps.echo.outputs.version }}
+    # version job uses ubuntu 24.04, this way we don't have to install the updated clang while
+    # the build job uses 20.04 for libc compatibility.
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - id: echo
+        run: |
+          echo "::set-output name=version::1.$(cat src/workerd/io/supported-compatibility-date.txt | tr -d '-').0"
+
+  build:
+    runs-on: ubuntu-20.04
+    name: build Python runtime
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: Setup Linux
+        if: runner.os == 'Linux'
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          wget https://apt.llvm.org/llvm.sh
+          sed -i '/apt-get install/d' llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 15
+          sudo apt-get install -y --no-install-recommends clang-15 lld-15 libunwind-15 libc++abi1-15 libc++1-15 libc++-15-dev
+          echo "build:linux --action_env=CC=/usr/lib/llvm-15/bin/clang --action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+          echo "build:linux --host_action_env=CC=/usr/lib/llvm-15/bin/clang --host_action_env=CXX=/usr/lib/llvm-15/bin/clang++" >> .bazelrc
+      - name: Configure download mirrors
+        shell: bash
+        run: |
+          if [ ! -z "${{ secrets.WORKERS_MIRROR_URL }}" ] ; then
+            # Strip comment in front of WORKERS_MIRROR_URL, then substitute secret to use it.
+            sed -e '/WORKERS_MIRROR_URL/ { s@# *@@; s@WORKERS_MIRROR_URL@${{ secrets.WORKERS_MIRROR_URL }}@; }' -i.bak WORKSPACE
+          fi
+      - name: Bazel build
+        # Strip debug info here – we don't generate debug info but some is pulled in from external
+        # static libraries, for example the Rust STL. This is equivalent to the -Wl,-S linker
+        # option, symbols will not be removed.
+        run: |
+          bazelisk build --config=release_linux //src/pyodide:pyodide.capnp.bin@rule
+          cp bazel-bin/src/pyodide/pyodide.capnp.bin pyodide-${{ inputs.tag }}.capnp.bin
+
+      - id: "GCS-auth"
+        if: ${{ !inputs.dry-run }}
+        uses: 'google-github-actions/auth@v2'
+        with:
+            service_account: 'python-packages-writer@edgeworkers.iam.gserviceaccount.com'
+            workload_identity_provider: "projects/672150504942/locations/global/workloadIdentityPools/github-workerd/providers/github-workerd"
+      - id: 'upload-file'
+        if: ${{ !inputs.dry-run }}
+        uses: 'google-github-actions/upload-cloud-storage@v2'
+        with:
+            path: 'pyodide-${{ inputs.tag }}.capnp.bin'
+            destination: 'cloudflare-edgeworker-python-packages/python-runtime-capnp-bin'
+            parent: false

build/wd_js_bundle.bzl

@@ -114,7 +114,7 @@ def _copy_modules(modules, declarations):
         result[new_filename] = modules[m]
     return result, declarations_result
 
-def wd_js_bundle(
+def wd_js_bundle_capnp(
         name,
         import_name,
         schema_id,
@@ -146,6 +146,7 @@ def wd_js_bundle(
      internal_json_modules: list of json source files
      declarations: d.ts label set
      deps: dependency list
+    Returns: The set of data dependencies
     """
     builtin_modules_dict = {
         m: "{}:{}".format(import_name, _to_name(m))
@@ -201,7 +202,7 @@ def wd_js_bundle(
 
     gen_api_bundle_capnpn(
         name = name + "@gen",
-        out = name + ".capnp",
+        out = name,
         schema_id = schema_id,
         const_name = import_name + "Bundle",
         builtin_modules = builtin_modules_dict,
@@ -213,7 +214,11 @@ def wd_js_bundle(
         data = data,
         deps = deps,
     )
+    return data
 
+
+def wd_js_bundle(name, import_name, *args, **kwargs):
+    data = wd_js_bundle_capnp(name + ".capnp", import_name, *args, **kwargs)
     cc_capnp_library(
         name = name,
         srcs = [name + ".capnp"],

build/wd_ts_bundle.bzl

@@ -1,6 +1,7 @@
 load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project")
 load("@npm//:eslint/package_json.bzl", eslint_bin = "bin")
-load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle")
+load("@workerd//:build/wd_js_bundle.bzl", "wd_js_bundle_capnp")
+load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
 
 def _to_js(file_name):
     if file_name.endswith(".ts"):
@@ -10,7 +11,7 @@ def _to_js(file_name):
 def _to_d_ts(file_name):
     return file_name.removesuffix(".ts") + ".d.ts"
 
-def wd_ts_bundle(
+def wd_ts_bundle_capnp(
         name,
         import_name,
         schema_id,
@@ -59,7 +60,7 @@ def wd_ts_bundle(
         deps = deps,
     )
 
-    wd_js_bundle(
+    data = wd_js_bundle_capnp(
         name = name,
         import_name = import_name,
         # builtin modules are accessible under "<import_name>:<module_name>" name
@@ -96,3 +97,17 @@ def wd_ts_bundle(
                 "//conditions:default": [],
             }),
         )
+    return data
+
+
+def wd_ts_bundle(name, import_name, *args, **kwargs):
+    data = wd_ts_bundle_capnp(name + ".capnp", import_name, *args, **kwargs)
+    cc_capnp_library(
+        name = name,
+        srcs = [name + ".capnp"],
+        strip_include_prefix = "",
+        visibility = ["//visibility:public"],
+        data = data,
+        deps = ["@workerd//src/workerd/jsg:modules_capnp"],
+        include_prefix = import_name,
+    )

src/pyodide/BUILD.bazel

@@ -3,7 +3,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
 load("//:build/capnp_embed.bzl", "capnp_embed")
-load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle")
+load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle_capnp")
 
 copy_file(
     name = "pyodide_packages_archive",
@@ -146,8 +146,8 @@ expand_template(
     template = "@pyodide//:pyodide/pyodide.asm.js",
 )
 
-wd_ts_bundle(
-    name = "pyodide",
+data = wd_ts_bundle_capnp(
+    name = "pyodide.capnp",
     modules = ["python-entrypoint-helper.ts"],
     import_name = "pyodide",
     internal_data_modules = ["generated/python_stdlib.zip"] + glob([
@@ -179,3 +179,31 @@ wd_ts_bundle(
         "pyodide-bucket.json@rule",
     ],
 )
+
+cc_capnp_library(
+    name = "pyodide",
+    srcs = ["pyodide.capnp"],
+    strip_include_prefix = "",
+    visibility = ["//visibility:public"],
+    data = data,
+    deps = ["@workerd//src/workerd/jsg:modules_capnp"],
+    include_prefix = "pyodide",
+)
+
+
+genrule(
+    name = "pyodide.capnp.bin@rule",
+    tools = ["@capnp-cpp//src/capnp:capnp_tool"],
+    srcs = ["pyodide.capnp", "//src/workerd/jsg:modules.capnp"] + data,
+    outs = ["pyodide.capnp.bin"],
+    visibility = ["//visibility:public"],
+    cmd = " ".join([
+        "$(execpath @capnp-cpp//src/capnp:capnp_tool)",
+        "eval",
+        "$(location :pyodide.capnp)",
+        "pyodideBundle",
+        "-I src/",
+        "-o binary",
+        "> $@",
+    ])
+)

src/workerd/jsg/BUILD.bazel

@@ -4,6 +4,8 @@ load("//:build/kj_test.bzl", "kj_test")
 load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library")
 load("//:build/wd_cc_library.bzl", "wd_cc_library")
 
+exports_files(["modules.capnp"])
+
 wd_cc_library(
     name = "jsg",
     srcs = [

src/workerd/server/BUILD.bazel

@@ -1,5 +1,6 @@
 load("@bazel_skylib//lib:selects.bzl", "selects")
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("//:build/kj_test.bzl", "kj_test")
 load("//:build/wd_cc_binary.bzl", "wd_cc_binary")
 load("//:build/wd_cc_capnp_library.bzl", "wd_cc_capnp_library")
@@ -179,3 +180,11 @@ kj_test(
         "@ssl",
     ],
 )
+
+
+copy_file(
+    name = "pyodide.capnp.bin@rule",
+    src = "//src/pyodide:pyodide.capnp.bin@rule",
+    out = "pyodide.capnp.bin",
+)
+