Implement build overlay mounting with mkosi-sandbox

Now that we have Context.rootoptions(), we can switch out how we set
up the root mount without having to modify code all over the place.

Let's use this to get rid of mount_build_overlay() and instead replace
it with setup_build_overlay(), which simply configures a bunch of
fields on Context that make rootoptions() set up the root mount as an
overlay instead of a bind mount.

mkosi/__init__.py

@@ -298,7 +298,7 @@ def install_build_packages(context: Context) -> None:
 
     with (
         complete_step(f"Installing build packages for {context.config.distribution.pretty_name()}"),
-        mount_build_overlay(context),
+        setup_build_overlay(context),
     ):
         context.config.distribution.install_packages(context, context.config.build_packages)
 
@@ -474,24 +474,40 @@ def configure_autologin(context: Context) -> None:
 
 
 @contextlib.contextmanager
-def mount_build_overlay(context: Context, volatile: bool = False) -> Iterator[Path]:
+def setup_build_overlay(context: Context, volatile: bool = False) -> Iterator[None]:
     d = context.workspace / "build-overlay"
     if not d.is_symlink():
         with umask(~0o755):
             d.mkdir(exist_ok=True)
 
-    with contextlib.ExitStack() as stack:
-        lower = [context.root]
+    # We don't support multiple levels of root overlay.
+    assert not context.lowerdirs
+    assert not context.upperdir
+    assert not context.workdir
 
+    with contextlib.ExitStack() as stack:
         if volatile:
-            lower += [d]
-            upper = None
+            context.lowerdirs = [d]
+            context.upperdir = Path(
+                stack.enter_context(tempfile.TemporaryDirectory(prefix="volatile-overlay"))
+            )
+            os.chmod(context.upperdir, d.stat().st_mode)
         else:
-            upper = d
+            context.upperdir = d
 
-        stack.enter_context(mount_overlay(lower, context.root, upperdir=upper))
+        context.workdir = stack.enter_context(
+            tempfile.TemporaryDirectory(
+                dir=Path(context.upperdir).parent,
+                prefix=f"{Path(context.upperdir).name}-workdir",
+            )
+        )
 
-        yield context.root
+        try:
+            yield
+        finally:
+            context.lowerdirs = []
+            context.upperdir = None
+            context.workdir = None
 
 
 @contextlib.contextmanager
@@ -751,7 +767,7 @@ def run_prepare_scripts(context: Context, build: bool) -> None:
     env |= context.config.finalize_environment()
 
     with (
-        mount_build_overlay(context) if build else contextlib.nullcontext(),
+        setup_build_overlay(context) if build else contextlib.nullcontext(),
         finalize_source_mounts(
             context.config,
             ephemeral=bool(context.config.build_sources_ephemeral),
@@ -827,7 +843,7 @@ def run_build_scripts(context: Context) -> None:
     env |= context.config.finalize_environment()
 
     with (
-        mount_build_overlay(context, volatile=True),
+        setup_build_overlay(context, volatile=True),
         finalize_source_mounts(context.config, ephemeral=context.config.build_sources_ephemeral) as sources,
         finalize_config_json(context.config) as json,
     ):

mkosi/context.py

@@ -7,7 +7,7 @@
 from typing import Optional
 
 from mkosi.config import Args, Config
-from mkosi.util import PathString
+from mkosi.util import PathString, flatten
 
 
 class Context:
@@ -31,6 +31,9 @@ def __init__(
         self.keyring_dir = keyring_dir
         self.metadata_dir = metadata_dir
         self.package_dir = package_dir or (self.workspace / "packages")
+        self.lowerdirs: list[PathString] = []
+        self.upperdir: Optional[PathString] = None
+        self.workdir: Optional[PathString] = None
 
         self.package_dir.mkdir(exist_ok=True)
         self.staging.mkdir()
@@ -44,7 +47,20 @@ def root(self) -> Path:
         return self.workspace / "root"
 
     def rootoptions(self, dst: PathString = "/buildroot", *, readonly: bool = False) -> list[str]:
-        return ["--ro-bind" if readonly else "--bind", os.fspath(self.root), os.fspath(dst)]
+        if self.lowerdirs or self.upperdir:
+            return [
+                "--overlay-lowerdir", os.fspath(self.root),
+                *flatten(["--overlay-lowerdir", os.fspath(lowerdir)] for lowerdir in self.lowerdirs),
+                *(
+                    ["--overlay-lowerdir" if readonly else "--overlay-upperdir", os.fspath(self.upperdir)]
+                    if self.upperdir
+                    else []
+                ),
+                *(["--overlay-workdir", os.fspath(self.workdir)] if self.workdir and not readonly else []),
+                "--overlay", os.fspath(dst),
+            ]  # fmt: skip
+        else:
+            return ["--ro-bind" if readonly else "--bind", os.fspath(self.root), os.fspath(dst)]
 
     @property
     def staging(self) -> Path: