Add virtfs/9p mounts, instead of sshocker/sshfs

This PR allows selecting mount type, as "9p"

The default mount type is still as before.

Signed-off-by: Anders F Björklund <anders.f.bjorklund@gmail.com>

.github/workflows/test.yml

@@ -115,6 +115,13 @@ jobs:
         retry_on: error
         max_attempts: 3
         command: ./hack/test-example.sh examples/alpine.yaml
+    - name: "Test experimental-9p.yaml"
+      uses: nick-invision/retry@v2
+      with:
+        timeout_minutes: 30
+        retry_on: error
+        max_attempts: 3
+        command: ./hack/test-example.sh examples/experimental-9p.yaml
     - name: "Test vmnet.yaml"
       uses: nick-invision/retry@v2
       with:

README.md

@@ -266,7 +266,6 @@ The current default spec:
 - Performance optimization
 - More guest distros
 - Windows hosts
-- [VirtFS to replace the current reverse sshfs (work has to be done on QEMU repo)](https://github.com/NixOS/nixpkgs/pull/122420)
 - [vsock](https://github.com/apple/darwin-xnu/blob/xnu-7195.81.3/bsd/man/man4/vsock.4) to replace SSH (work has to be done on QEMU repo)
 
 ## FAQs & Troubleshooting

docs/internal.md

@@ -109,6 +109,7 @@ The volume label is "cidata", as defined by [cloud-init NoCloud](https://cloudin
 - `LIMA_CIDATA_UID`: the numeric UID
 - `LIMA_CIDATA_MOUNTS`: the number of the Lima mounts
 - `LIMA_CIDATA_MOUNTS_%d_MOUNTPOINT`: the N-th mount point of Lima mounts (N=0, 1, ...)
+- `LIMA_CIDATA_MOUNTTYPE`: the type of the Lima mounts ("reverse-sshfs", "9p", ...)
 - `LIMA_CIDATA_CONTAINERD_USER`: set to "1" if rootless containerd to be set up
 - `LIMA_CIDATA_CONTAINERD_SYSTEM`: set to "1" if system-wide containerd to be set up
 - `LIMA_CIDATA_SLIRP_GATEWAY`: set to the IP address of the host on the SLIRP network. `192.168.5.2`.

examples/README.md

@@ -34,6 +34,7 @@ Container orchestration:
 Others:
 - [`vmnet.yaml`](./vmnet.yaml): ⭐enable [`vmnet.framework`](../docs/network.md)
 - [`deprecated/centos-7.yaml`](./deprecated/centos-7.yaml): [deprecated] CentOS 7
+- [`experimental-9p.yaml`](experimental-9p.yaml): use 9p mount type
 
 ## Tier 1
 

examples/default.yaml

@@ -61,11 +61,28 @@ mounts:
     # system will look and feel like regular files directories in the Guest OS.
     # 🟢 Builtin default: false
     followSymlinks: null
+  9p:
+    # Supported security models are "passthrough", "mapped-xattr", "mapped-file" and "none".
+    # 🟢 Builtin default: "mapped-xattr"
+    securityModel: null
+    # Select 9P protocol version. Valid options are: "9p2000" (legacy), "9p2000.u", "9p2000.L".
+    # 🟢 Builtin default: "9p2000.L"
+    protocolVersion: null
+    # The number of bytes to use for 9p packet payload, where 4 KiB is the absolute minimum.
+    # 🟢 Builtin default: "128KiB"
+    msize: null
+    # Specifies a caching policy. Valid options are: "none", "loose", "fscache" and "mmap".
+    # 🟢 Builtin default: "mmap"
+    cache: null
 - location: "/tmp/lima"
   # 🟢 Builtin default: false
   # 🔵 This file: true (only for "/tmp/lima")
   writable: true
 
+# Mount type for above mounts, such as "reverse-sshfs" (from sshocker) or "9p" (EXPERIMENTAL, from QEMU’s virtio-9p-pci, aka virtfs)
+# 🟢 Builtin default: "reverse-sshfs"
+mountType: null
+
 ssh:
   # A localhost port of the host. Forwarded to port 22 of the guest.
   # 🟢 Builtin default: 0 (automatically assigned to a free port)

examples/experimental-9p.yaml

@@ -0,0 +1,23 @@
+# This example requires Lima v0.10.0 or later.
+# On macOS hosts, QEMU needs to be v7.0.0 or later. Homebrew's QEMU v6.2.0_1 can be used too.
+# This example is planned to be merged to default.yaml in Lima v1.0 (ETA: 2022 Q2).
+images:
+# Try to use release-yyyyMMdd image if available. Note that release-yyyyMMdd will be removed after several months.
+- location: "https://cloud-images.ubuntu.com/releases/21.10/release-20220201/ubuntu-21.10-server-cloudimg-amd64.img"
+  arch: "x86_64"
+  digest: "sha256:73fe1785c60edeb506f191affff0440abcc2de02420bb70865d51d0ff9b28223"
+- location: "https://cloud-images.ubuntu.com/releases/21.10/release-20220201/ubuntu-21.10-server-cloudimg-arm64.img"
+  arch: "aarch64"
+  digest: "sha256:1b5b3fe616e1eea4176049d434a360344a7d471f799e151190f21b0a27f0b424"
+# Fallback to the latest release image.
+# Hint: run `limactl prune` to invalidate the cache
+- location: "https://cloud-images.ubuntu.com/releases/21.10/release/ubuntu-21.10-server-cloudimg-amd64.img"
+  arch: "x86_64"
+- location: "https://cloud-images.ubuntu.com/releases/21.10/release/ubuntu-21.10-server-cloudimg-arm64.img"
+  arch: "aarch64"
+
+mounts:
+- location: "~"
+- location: "/tmp/lima"
+  writable: true
+mountType: "9p"

pkg/cidata/cidata.TEMPLATE.d/boot/25-guestagent-base.sh

@@ -2,15 +2,17 @@
 
 set -eux
 
-# Create mount points
-# NOTE: Busybox sh does not support `for ((i=0;i<$N;i++))` form
-for f in $(seq 0 $((LIMA_CIDATA_MOUNTS - 1))); do
-	mountpointvar="LIMA_CIDATA_MOUNTS_${f}_MOUNTPOINT"
-	mountpoint="$(eval echo \$"$mountpointvar")"
-	mkdir -p "${mountpoint}"
-	gid=$(id -g "${LIMA_CIDATA_USER}")
-	chown "${LIMA_CIDATA_UID}:${gid}" "${mountpoint}"
-done
+if [ "${LIMA_CIDATA_MOUNTTYPE}" != "9p" ]; then
+	# Create mount points
+	# NOTE: Busybox sh does not support `for ((i=0;i<$N;i++))` form
+	for f in $(seq 0 $((LIMA_CIDATA_MOUNTS - 1))); do
+		mountpointvar="LIMA_CIDATA_MOUNTS_${f}_MOUNTPOINT"
+		mountpoint="$(eval echo \$"$mountpointvar")"
+		mkdir -p "${mountpoint}"
+		gid=$(id -g "${LIMA_CIDATA_USER}")
+		chown "${LIMA_CIDATA_UID}:${gid}" "${mountpoint}"
+	done
+fi
 
 # Install or update the guestagent binary
 install -m 755 "${LIMA_CIDATA_MNT}"/lima-guestagent /usr/local/bin/lima-guestagent

pkg/cidata/cidata.TEMPLATE.d/boot/30-install-packages.sh

@@ -21,8 +21,10 @@ fi
 # Install minimum dependencies
 if command -v apt-get >/dev/null 2>&1; then
 	pkgs=""
-	if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
-		pkgs="${pkgs} sshfs"
+	if [ "${LIMA_CIDATA_MOUNTTYPE}" = "reverse-sshfs" ]; then
+		if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
+			pkgs="${pkgs} sshfs"
+		fi
 	fi
 	if [ "${INSTALL_IPTABLES}" = 1 ] && [ ! -e /usr/sbin/iptables ]; then
 		pkgs="${pkgs} iptables"
@@ -42,8 +44,10 @@ elif command -v dnf >/dev/null 2>&1; then
 	if ! command -v tar >/dev/null 2>&1; then
 		pkgs="${pkgs} tar"
 	fi
-	if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
-		pkgs="${pkgs} fuse-sshfs"
+	if [ "${LIMA_CIDATA_MOUNTTYPE}" = "reverse-sshfs" ]; then
+		if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
+			pkgs="${pkgs} fuse-sshfs"
+		fi
 	fi
 	if [ "${INSTALL_IPTABLES}" = 1 ] && [ ! -e /usr/sbin/iptables ]; then
 		pkgs="${pkgs} iptables"
@@ -102,8 +106,10 @@ elif command -v yum >/dev/null 2>&1; then
 	fi
 elif command -v pacman >/dev/null 2>&1; then
 	pkgs=""
-	if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
-		pkgs="${pkgs} sshfs"
+	if [ "${LIMA_CIDATA_MOUNTTYPE}" = "reverse-sshfs" ]; then
+		if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
+			pkgs="${pkgs} sshfs"
+		fi
 	fi
 	# other dependencies are preinstalled on Arch Linux
 	if [ -n "${pkgs}" ]; then
@@ -112,8 +118,10 @@ elif command -v pacman >/dev/null 2>&1; then
 	fi
 elif command -v zypper >/dev/null 2>&1; then
 	pkgs=""
-	if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
-		pkgs="${pkgs} sshfs"
+	if [ "${LIMA_CIDATA_MOUNTTYPE}" = "reverse-sshfs" ]; then
+		if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
+			pkgs="${pkgs} sshfs"
+		fi
 	fi
 	if [ "${INSTALL_IPTABLES}" = 1 ] && [ ! -e /usr/sbin/iptables ]; then
 		pkgs="${pkgs} iptables"
@@ -127,8 +135,10 @@ elif command -v zypper >/dev/null 2>&1; then
 	fi
 elif command -v apk >/dev/null 2>&1; then
 	pkgs=""
-	if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
-		pkgs="${pkgs} sshfs"
+	if [ "${LIMA_CIDATA_MOUNTTYPE}" = "reverse-sshfs" ]; then
+		if [ "${LIMA_CIDATA_MOUNTS}" -gt 0 ] && ! command -v sshfs >/dev/null 2>&1; then
+			pkgs="${pkgs} sshfs"
+		fi
 	fi
 	if [ "${INSTALL_IPTABLES}" = 1 ] && ! command -v iptables >/dev/null 2>&1; then
 		pkgs="${pkgs} iptables"
@@ -154,4 +164,6 @@ fi
 
 # update_fuse_conf has to be called after installing all the packages,
 # otherwise apt-get fails with conflict
-update_fuse_conf
+if [ "${LIMA_CIDATA_MOUNTTYPE}" = "reverse-sshfs" ]; then
+	update_fuse_conf
+fi

pkg/cidata/cidata.TEMPLATE.d/lima.env

@@ -3,8 +3,9 @@ LIMA_CIDATA_USER={{ .User }}
 LIMA_CIDATA_UID={{ .UID }}
 LIMA_CIDATA_MOUNTS={{ len .Mounts }}
 {{- range $i, $val := .Mounts}}
-LIMA_CIDATA_MOUNTS_{{$i}}_MOUNTPOINT={{$val}}
+LIMA_CIDATA_MOUNTS_{{$i}}_MOUNTPOINT={{$val.Target}}
 {{- end}}
+LIMA_CIDATA_MOUNTTYPE={{ .MountType }}
 {{- if .Containerd.User}}
 LIMA_CIDATA_CONTAINERD_USER=1
 {{- else}}

pkg/cidata/cidata.TEMPLATE.d/user-data

@@ -5,6 +5,15 @@ growpart:
   mode: auto
   devices: ['/']
 
+{{- if eq .MountType "9p" }}
+{{- if .Mounts }}
+mounts:
+  {{- range $m := $.Mounts}}
+- [{{$m.Tag}}, {{$m.Target}}, {{$m.Type}}, "{{$m.Options}}", "0", "0"]
+  {{- end }}
+{{- end }}
+{{- end }}
+
 users:
   - name: "{{.User}}"
     uid: "{{.UID}}"

pkg/cidata/cidata.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/docker/go-units"
 	"github.com/lima-vm/lima/pkg/iso9660util"
 	"github.com/lima-vm/lima/pkg/limayaml"
 	"github.com/lima-vm/lima/pkg/localpathutil"
@@ -120,12 +121,44 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort
 		args.SSHPubKeys = append(args.SSHPubKeys, f.Content)
 	}
 
-	for _, f := range y.Mounts {
+	var fstype string
+	switch *y.MountType {
+	case limayaml.REVSSHFS:
+		fstype = "sshfs"
+	case limayaml.NINEP:
+		fstype = "9p"
+	}
+	for i, f := range y.Mounts {
+		tag := fmt.Sprintf("mount%d", i)
 		expanded, err := localpathutil.Expand(f.Location)
 		if err != nil {
 			return err
 		}
-		args.Mounts = append(args.Mounts, expanded)
+		options := "defaults"
+		if fstype == "9p" {
+			options = "ro"
+			if *f.Writable {
+				options = "rw"
+			}
+			options += ",trans=virtio"
+			options += fmt.Sprintf(",version=%s", *f.NineP.ProtocolVersion)
+			msize, err := units.RAMInBytes(*f.NineP.Msize)
+			if err != nil {
+				return fmt.Errorf("failed to parse msize for %q: %w", expanded, err)
+			}
+			options += fmt.Sprintf(",msize=%d", msize)
+			options += fmt.Sprintf(",cache=%s", *f.NineP.Cache)
+			// don't fail the boot, if virtfs is not available
+			options += ",nofail"
+		}
+		args.Mounts = append(args.Mounts, Mount{Tag: tag, Target: expanded, Type: fstype, Options: options})
+	}
+
+	switch *y.MountType {
+	case limayaml.REVSSHFS:
+		args.MountType = "reverse-sshfs"
+	case limayaml.NINEP:
+		args.MountType = "9p"
 	}
 
 	slirpMACAddress := limayaml.MACAddress(instDir)

pkg/cidata/template.go

@@ -27,13 +27,20 @@ type Network struct {
 	MACAddress string
 	Interface  string
 }
+type Mount struct {
+	Tag     string
+	Target  string // abs path, accessible by the User
+	Type    string
+	Options string
+}
 type TemplateArgs struct {
 	Name            string // instance name
 	IID             string // instance id
 	User            string // user name
 	UID             int
 	SSHPubKeys      []string
-	Mounts          []string // abs path, accessible by the User
+	Mounts          []Mount
+	MountType       string
 	Containerd      Containerd
 	Networks        []Network
 	SlirpNICName    string
@@ -62,7 +69,8 @@ func ValidateTemplateArgs(args TemplateArgs) error {
 	if len(args.SSHPubKeys) == 0 {
 		return errors.New("field SSHPubKeys must be set")
 	}
-	for i, f := range args.Mounts {
+	for i, m := range args.Mounts {
+		f := m.Target
 		if !filepath.IsAbs(f) {
 			return fmt.Errorf("field mounts[%d] must be absolute, got %q", i, f)
 		}

pkg/cidata/template_test.go

@@ -2,6 +2,7 @@ package cidata
 
 import (
 	"io"
+	"strings"
 	"testing"
 
 	"gotest.tools/v3/assert"
@@ -15,10 +16,11 @@ func TestTemplate(t *testing.T) {
 		SSHPubKeys: []string{
 			"ssh-rsa dummy foo@example.com",
 		},
-		Mounts: []string{
-			"/Users/dummy",
-			"/Users/dummy/lima",
+		Mounts: []Mount{
+			{Target: "/Users/dummy"},
+			{Target: "/Users/dummy/lima"},
 		},
+		MountType: "reverse-sshfs",
 	}
 	layout, err := ExecuteTemplate(args)
 	assert.NilError(t, err)
@@ -27,5 +29,37 @@ func TestTemplate(t *testing.T) {
 		b, err := io.ReadAll(f.Reader)
 		assert.NilError(t, err)
 		t.Log(string(b))
+		if f.Path == "user-data" {
+			// mounted later
+			assert.Assert(t, !strings.Contains(string(b), "mounts:"))
+		}
+	}
+}
+
+func TestTemplate9p(t *testing.T) {
+	args := TemplateArgs{
+		Name: "default",
+		User: "foo",
+		UID:  501,
+		SSHPubKeys: []string{
+			"ssh-rsa dummy foo@example.com",
+		},
+		Mounts: []Mount{
+			{Tag: "mount0", Target: "/Users/dummy", Type: "9p", Options: "ro,trans=virtio"},
+			{Tag: "mount1", Target: "/Users/dummy/lima", Type: "9p", Options: "rw,trans=virtio"},
+		},
+		MountType: "9p",
+	}
+	layout, err := ExecuteTemplate(args)
+	assert.NilError(t, err)
+	for _, f := range layout {
+		t.Logf("=== %q ===", f.Path)
+		b, err := io.ReadAll(f.Reader)
+		assert.NilError(t, err)
+		t.Log(string(b))
+		if f.Path == "user-data" {
+			// mounted at boot
+			assert.Assert(t, strings.Contains(string(b), "mounts:"))
+		}
 	}
 }