Use Block CIM layers for container RootFS

This commit adds the ability to parse block CIM layer mounts and to mount the merged block
CIMs to be used as a rootfs for a container.

Signed-off-by: Amit Barve <ambarve@microsoft.com>

internal/layers/helpers.go

@@ -75,8 +75,11 @@ const (
 	// parent layer CIMs
 	parentLayerCimPathsFlag = "parentCimPaths="
 
-	LegacyMountType string = "windows-layer"
-	CimFSMountType  string = "CimFS"
+	LegacyMountType    string = "windows-layer"
+	ForkedCIMMountType string = "CimFS"
+	BlockCIMMountType  string = "BlockCIM"
+	BlockCIMTypeFlag   string = "blockCIMType="
+	mergedCIMPathFlag  string = "mergedCIMPath="
 )
 
 // getOptionAsArray finds if there is an option which has the given prefix and if such an

internal/layers/wcow_mount.go

@@ -12,12 +12,14 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
 	"golang.org/x/sys/windows"
 
 	"github.com/Microsoft/hcsshim/computestorage"
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 	"github.com/Microsoft/hcsshim/internal/hcserror"
 	"github.com/Microsoft/hcsshim/internal/log"
+	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/resources"
 	"github.com/Microsoft/hcsshim/internal/uvm"
 	"github.com/Microsoft/hcsshim/internal/uvm/scsi"
@@ -37,6 +39,11 @@ func MountWCOWLayers(ctx context.Context, containerID string, vm *uvm.UtilityVM,
 			return mountProcessIsolatedForkedCimLayers(ctx, containerID, l)
 		}
 		return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with forked cim layers")
+	case *wcowBlockCIMLayers:
+		if vm == nil {
+			return mountProcessIsolatedBlockCIMLayers(ctx, containerID, l)
+		}
+		return nil, nil, fmt.Errorf("hyperv isolated containers aren't supported with block cim layers")
 	default:
 		return nil, nil, fmt.Errorf("invalid layer type %T", wl)
 	}
@@ -171,53 +178,43 @@ func mountProcessIsolatedWCIFSLayers(ctx context.Context, l *wcowWCIFSLayers) (_
 		}, nil
 }
 
-// wcowHostForkedCIMLayerCloser is used to cleanup forked CIM layers mounted on the host for process isolated
-// containers
-type wcowHostForkedCIMLayerCloser struct {
-	scratchLayerData
-	containerID string
-}
-
-func (l *wcowHostForkedCIMLayerCloser) Release(ctx context.Context) error {
-	mountPath, err := wclayer.GetLayerMountPath(ctx, l.scratchLayerPath)
-	if err != nil {
-		return err
-	}
-
-	if err = computestorage.DetachOverlayFilter(ctx, mountPath, hcsschema.UnionFS); err != nil {
-		return err
-	}
-
-	if err = cimlayer.CleanupContainerMounts(l.containerID); err != nil {
-		return err
-	}
-	return wclayer.DeactivateLayer(ctx, l.scratchLayerPath)
-}
+// Handles the common processing for mounting all 3 types of cimfs layers. This involves
+// mounting the scratch, attaching the filter and preparing the return values.
+// `volume` is the path to the volume at which read only layer CIMs are mounted.
+func mountProcessIsolatedCimLayersCommon(ctx context.Context, containerID string, volume string, s *scratchLayerData) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
+	ctx, span := oc.StartSpan(ctx, "mountProcessIsolatedCimLayersCommon")
+	defer func() {
+		oc.SetSpanStatus(span, err)
+		span.End()
+	}()
+	span.AddAttributes(
+		trace.StringAttribute("scratch path", s.scratchLayerPath),
+		trace.StringAttribute("mounted CIM volume", volume))
 
-func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string, l *wcowForkedCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
-	if err = wclayer.ActivateLayer(ctx, l.scratchLayerPath); err != nil {
-		return nil, nil, err
-	}
+	rcl := &resources.ResourceCloserList{}
 	defer func() {
 		if err != nil {
-			_ = wclayer.DeactivateLayer(ctx, l.scratchLayerPath)
+			if rErr := rcl.Release(ctx); rErr != nil {
+				log.G(ctx).WithError(err).Warnf("mount process isolated cim layers common, undo failed with: %s", rErr)
+			}
 		}
 	}()
 
-	mountPath, err := wclayer.GetLayerMountPath(ctx, l.scratchLayerPath)
-	if err != nil {
+	if err = wclayer.ActivateLayer(ctx, s.scratchLayerPath); err != nil {
 		return nil, nil, err
 	}
+	rcl.AddFunc(func(uCtx context.Context) error {
+		return wclayer.DeactivateLayer(uCtx, s.scratchLayerPath)
+	})
 
-	volume, err := cimlayer.MountForkedCimLayer(ctx, l.layers[0].cimPath, containerID)
+	mountPath, err := wclayer.GetLayerMountPath(ctx, s.scratchLayerPath)
 	if err != nil {
-		return nil, nil, fmt.Errorf("mount layer cim: %w", err)
+		return nil, nil, err
 	}
-	defer func() {
-		if err != nil {
-			_ = cimlayer.UnmountCimLayer(ctx, volume)
-		}
-	}()
+	log.G(ctx).WithFields(logrus.Fields{
+		"scratch":      s.scratchLayerPath,
+		"mounted path": mountPath,
+	}).Debug("scratch activated")
 
 	layerID, err := cimlayer.LayerID(volume)
 	if err != nil {
@@ -239,22 +236,97 @@ func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string
 	if err = computestorage.AttachOverlayFilter(ctx, mountPath, layerData); err != nil {
 		return nil, nil, err
 	}
+	rcl.AddFunc(func(uCtx context.Context) error {
+		return computestorage.DetachOverlayFilter(uCtx, mountPath, hcsschema.UnionFS)
+	})
+
+	log.G(ctx).WithField("layer data", layerData).Debug("unionFS filter attached")
+
+	return &MountedWCOWLayers{
+		RootFS: mountPath,
+		MountedLayerPaths: []MountedWCOWLayer{{
+			LayerID:     layerID,
+			MountedPath: volume,
+		}},
+	}, rcl, nil
+}
+
+func mountProcessIsolatedForkedCimLayers(ctx context.Context, containerID string, l *wcowForkedCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
+	ctx, span := oc.StartSpan(ctx, "mountProcessIsolatedForkedCimLayers")
+	defer func() {
+		oc.SetSpanStatus(span, err)
+		span.End()
+	}()
+
+	rcl := &resources.ResourceCloserList{}
+	defer func() {
+		if err != nil {
+			if rErr := rcl.Release(ctx); rErr != nil {
+				log.G(ctx).WithError(err).Warnf("mount process isolated forked CIM layers, undo failed with: %s", rErr)
+			}
+		}
+	}()
+
+	volume, err := cimlayer.MountForkedCimLayer(ctx, l.layers[0].cimPath, containerID)
+	if err != nil {
+		return nil, nil, fmt.Errorf("mount forked layer cim: %w", err)
+	}
+	rcl.AddFunc(func(uCtx context.Context) error {
+		return cimlayer.UnmountCimLayer(uCtx, volume)
+	})
+
+	mountedLayers, closer, err := mountProcessIsolatedCimLayersCommon(ctx, containerID, volume, &l.scratchLayerData)
+	if err != nil {
+		return nil, nil, err
+	}
+	return mountedLayers, rcl.Add(closer), nil
+}
+
+func mountProcessIsolatedBlockCIMLayers(ctx context.Context, containerID string, l *wcowBlockCIMLayers) (_ *MountedWCOWLayers, _ resources.ResourceCloser, err error) {
+	ctx, span := oc.StartSpan(ctx, "mountProcessIsolatedBlockCIMLayers")
+	defer func() {
+		oc.SetSpanStatus(span, err)
+		span.End()
+	}()
+
+	var volume string
+
+	rcl := &resources.ResourceCloserList{}
 	defer func() {
 		if err != nil {
-			_ = computestorage.DetachOverlayFilter(ctx, mountPath, hcsschema.UnionFS)
+			if rErr := rcl.Release(ctx); rErr != nil {
+				log.G(ctx).WithError(err).Warnf("mount process isolated forked CIM layers, undo failed with: %s", rErr)
+			}
 		}
 	}()
 
-	return &MountedWCOWLayers{
-			RootFS: mountPath,
-			MountedLayerPaths: []MountedWCOWLayer{{
-				LayerID:     layerID,
-				MountedPath: volume,
-			}},
-		}, &wcowHostForkedCIMLayerCloser{
-			containerID:      containerID,
-			scratchLayerData: l.scratchLayerData,
-		}, nil
+	log.G(ctx).WithFields(logrus.Fields{
+		"scratch":       l.scratchLayerPath,
+		"merged layer":  l.mergedLayer,
+		"parent layers": l.parentLayers,
+	}).Debug("mounting process isolated block CIM layers")
+
+	if len(l.parentLayers) > 1 {
+		volume, err = cimlayer.MergeMountBlockCIMLayer(ctx, l.mergedLayer, l.parentLayers, containerID)
+	} else {
+		volume, err = cimlayer.MountBlockCIMLayer(ctx, l.parentLayers[0], containerID)
+	}
+	if err != nil {
+		return nil, nil, fmt.Errorf("mount block CIM layers: %w", err)
+	}
+	rcl.AddFunc(func(uCtx context.Context) error {
+		return cimlayer.UnmountCimLayer(uCtx, volume)
+	})
+
+	log.G(ctx).WithField("volume", volume).Debug("mounted blockCIM layers for process isolated container")
+
+	mountedLayers, layerCloser, err := mountProcessIsolatedCimLayersCommon(ctx, containerID, volume, &l.scratchLayerData)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed mount CIM layers common: %w", err)
+	}
+	rcl.Add(layerCloser)
+
+	return mountedLayers, rcl, nil
 }
 
 type wcowIsolatedWCIFSLayerCloser struct {

internal/layers/wcow_parse.go

@@ -5,15 +5,18 @@ package layers
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/containerd/containerd/api/types"
 
 	"github.com/Microsoft/hcsshim/internal/copyfile"
 	"github.com/Microsoft/hcsshim/internal/uvm"
 	"github.com/Microsoft/hcsshim/internal/uvmfolder"
+	"github.com/Microsoft/hcsshim/pkg/cimfs"
 )
 
 // WCOW image layers is a tagging interface that all WCOW layers MUST implement. This is
@@ -67,6 +70,17 @@ type wcowForkedCIMLayers struct {
 	layers []forkedCIMLayer
 }
 
+// Represents CIM layers where each layer is stored in a block device or in a single file
+// and multiple such layer CIMs are merged before mounting them. Currently can only be
+// used for process isolated containers.
+type wcowBlockCIMLayers struct {
+	scratchLayerData
+	// parent layers in order [layerN (top-most), layerN-1,..layer0 (base)]
+	parentLayers []*cimfs.BlockCIM
+	// a merged layer is prepared by combining all parent layers
+	mergedLayer *cimfs.BlockCIM
+}
+
 func parseForkedCimMount(m *types.Mount) (*wcowForkedCIMLayers, error) {
 	parentLayerPaths, err := getOptionAsArray(m, parentLayerPathsFlag)
 	if err != nil {
@@ -94,8 +108,77 @@ func parseForkedCimMount(m *types.Mount) (*wcowForkedCIMLayers, error) {
 	}, nil
 }
 
-// ParseWCOWLayers parses the layers provided by containerd into the format understood by hcsshim and prepares
-// them for mounting.
+// TODO(ambarve): The code to parse a mount type should be in a separate package/module
+// somewhere and then should be consumed by both hcsshim & containerd from there.
+func parseBlockCIMMount(m *types.Mount) (*wcowBlockCIMLayers, error) {
+	var (
+		parentPaths   []string
+		layerType     cimfs.BlockCIMType
+		mergedCIMPath string
+	)
+
+	for _, option := range m.Options {
+		if val, ok := strings.CutPrefix(option, parentLayerCimPathsFlag); ok {
+			err := json.Unmarshal([]byte(val), &parentPaths)
+			if err != nil {
+				return nil, err
+			}
+		} else if val, ok = strings.CutPrefix(option, BlockCIMTypeFlag); ok {
+			if val == "device" {
+				layerType = cimfs.BlockCIMTypeDevice
+			} else if val == "file" {
+				layerType = cimfs.BlockCIMTypeSingleFile
+			} else {
+				return nil, fmt.Errorf("invalid block CIM type `%s`", val)
+			}
+		} else if val, ok = strings.CutPrefix(option, mergedCIMPathFlag); ok {
+			mergedCIMPath = val
+		}
+	}
+
+	if len(parentPaths) == 0 {
+		return nil, fmt.Errorf("need at least 1 parent layer")
+	}
+	if layerType == cimfs.BlockCIMTypeNone {
+		return nil, fmt.Errorf("BlockCIM type not provided")
+	}
+	if mergedCIMPath == "" && len(parentPaths) > 1 {
+		return nil, fmt.Errorf("merged CIM path not provided")
+	}
+
+	var (
+		parentLayers []*cimfs.BlockCIM
+		mergedLayer  *cimfs.BlockCIM
+	)
+
+	if len(parentPaths) > 1 {
+		// for single parent layers merge won't be done
+		mergedLayer = &cimfs.BlockCIM{
+			Type:      layerType,
+			BlockPath: filepath.Dir(mergedCIMPath),
+			CimName:   filepath.Base(mergedCIMPath),
+		}
+	}
+
+	for _, p := range parentPaths {
+		parentLayers = append(parentLayers, &cimfs.BlockCIM{
+			Type:      layerType,
+			BlockPath: filepath.Dir(p),
+			CimName:   filepath.Base(p),
+		})
+	}
+
+	return &wcowBlockCIMLayers{
+		scratchLayerData: scratchLayerData{
+			scratchLayerPath: m.Source,
+		},
+		parentLayers: parentLayers,
+		mergedLayer:  mergedLayer,
+	}, nil
+}
+
+// ParseWCOWLayers parses the layers provided by containerd into the format understood by
+// hcsshim and prepares them for mounting.
 func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers, error) {
 	if err := validateRootfsAndLayers(rootfs, layerFolders); err != nil {
 		return nil, err
@@ -123,8 +206,10 @@ func ParseWCOWLayers(rootfs []*types.Mount, layerFolders []string) (WCOWLayers,
 			},
 			layerPaths: parentLayers,
 		}, nil
-	case CimFSMountType:
+	case ForkedCIMMountType:
 		return parseForkedCimMount(m)
+	case BlockCIMMountType:
+		return parseBlockCIMMount(m)
 	default:
 		return nil, fmt.Errorf("invalid windows mount type: '%s'", m.Type)
 	}

internal/resources/resources.go

@@ -168,3 +168,27 @@ func ReleaseResources(ctx context.Context, r *Resources, vm *uvm.UtilityVM, all
 	}
 	return nil
 }
+
+type ResourceCloserList struct {
+	closers []ResourceCloser
+}
+
+func (l *ResourceCloserList) Add(rOp ResourceCloser) *ResourceCloserList {
+	l.closers = append(l.closers, rOp)
+	return l
+}
+
+func (l *ResourceCloserList) AddFunc(rOp ResourceCloserFunc) *ResourceCloserList {
+	l.closers = append(l.closers, rOp)
+	return l
+}
+
+func (l *ResourceCloserList) Release(ctx context.Context) error {
+	// MUST release in the reverse order
+	for i := len(l.closers) - 1; i >= 0; i-- {
+		if oErr := l.closers[i].Release(ctx); oErr != nil {
+			return oErr
+		}
+	}
+	return nil
+}