[Do Not Merge][CPPS][gcs][PoC] Create UVM without networking setup, set network during container creation

internal/guest/network/network.go

@@ -34,6 +34,7 @@ const maxDNSSearches = 6
 
 // GenerateEtcHostsContent generates a /etc/hosts file based on `hostname`.
 func GenerateEtcHostsContent(ctx context.Context, hostname string) string {
+	log.G(ctx).Debug("Inside guest-network-GenerateEtcHostsContent")
 	_, span := oc.StartSpan(ctx, "network::GenerateEtcHostsContent")
 	defer span.End()
 	span.AddAttributes(trace.StringAttribute("hostname", hostname))

internal/guest/runtime/hcsv2/sandbox_container.go

@@ -7,13 +7,11 @@ import (
 	"context"
 	"os"
 	"path/filepath"
-	"strings"
 
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"go.opencensus.io/trace"
 
-	"github.com/Microsoft/hcsshim/internal/guest/network"
 	specInternal "github.com/Microsoft/hcsshim/internal/guest/spec"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
@@ -64,20 +62,20 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
 	}
 
 	// Write the hosts
-	sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname)
-	sandboxHostsPath := getSandboxHostsPath(id)
-	if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil {
-		return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath)
-	}
+	// sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname)
+	// sandboxHostsPath := getSandboxHostsPath(id)
+	// if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil {
+	// 	return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath)
+	// }
 
 	// Write resolv.conf
-	ns, err := getNetworkNamespace(getNetworkNamespaceID(spec))
-	if err != nil {
-		return err
-	}
-	var searches, servers []string
-	for _, n := range ns.Adapters() {
-		if len(n.DNSSuffix) > 0 {
+	// ns, err := getNetworkNamespace(getNetworkNamespaceID(spec))
+	// if err != nil {
+	// 	return err
+	// }
+	// var searches, servers []string
+	// for _, n := range ns.Adapters() {
+	/*	if len(n.DNSSuffix) > 0 {
 			searches = network.MergeValues(searches, strings.Split(n.DNSSuffix, ","))
 		}
 		if len(n.DNSServerList) > 0 {
@@ -91,7 +89,7 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) (
 	sandboxResolvPath := getSandboxResolvPath(id)
 	if err := os.WriteFile(sandboxResolvPath, []byte(resolvContent), 0644); err != nil {
 		return errors.Wrap(err, "failed to write sandbox resolv.conf")
-	}
+	}*/
 
 	// User.Username is generally only used on Windows, but as there's no (easy/fast at least) way to grab
 	// a uid:gid pairing for a username string on the host, we need to defer this work until we're here in the

internal/guest/runtime/hcsv2/uvm.go

@@ -357,12 +357,14 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 				return nil, err
 			}
 		case "container":
+			namespaceID = getNetworkNamespaceID(settings.OCISpecification)
 			sid, ok := settings.OCISpecification.Annotations[annotations.KubernetesSandboxID]
 			sandboxID = sid
 			if !ok || sid == "" {
 				return nil, errors.Errorf("unsupported 'io.kubernetes.cri.sandbox-id': '%s'", sid)
 			}
 			if err := setupWorkloadContainerSpec(ctx, sid, id, settings.OCISpecification); err != nil {
+				log.G(ctx).Debug("UVM:CreateContainer: Failed in setupWorkloadContainerSpec")
 				return nil, err
 			}
 
@@ -381,8 +383,10 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 				}
 			}()
 			if err := policy.ExtendPolicyWithNetworkingMounts(sandboxID, h.securityPolicyEnforcer, settings.OCISpecification); err != nil {
+				log.G(ctx).Debug("UVM:CreateContainer: Failed in ExtendPolicyWithNetworkingMounts")
 				return nil, err
 			}
+			log.G(ctx).Debug("UVM:CreateContainer: Completed set up workload container spec")
 		default:
 			return nil, errors.Errorf("unsupported 'io.kubernetes.cri.container-type': '%s'", criType)
 		}
@@ -505,7 +509,9 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 		return nil, errors.Wrapf(err, "failed to flush writer for config.json at: '%s'", configFile)
 	}
 
+	log.G(ctx).Debug("UVM:CreateContainer: Calling h.rtime.CreateContainer")
 	con, err := h.rtime.CreateContainer(id, settings.OCIBundlePath, nil)
+	log.G(ctx).Debug("UVM:CreateContainer: Completed h.rtime.CreateContainer")
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to create container")
 	}
@@ -518,7 +524,13 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 	c.initProcess = newProcess(c, settings.OCISpecification.Process, init, uint32(c.container.Pid()), true)
 
 	// Sandbox or standalone, move the networks to the container namespace
-	if criType == "sandbox" || !isCRI {
+	// However in case of preprovisioned VMs, we create the network in case
+	// of workload containers, so we need to move the networks to the container
+	// namespace.
+	log.G(ctx).Debug("Logging namespace ID:", namespaceID)
+	log.G(ctx).Debug("criType =", criType)
+	if criType != "sandbox" || !isCRI {
+		log.G(ctx).Debug("Moving networks to namespace")
 		ns, err := getNetworkNamespace(namespaceID)
 		if isCRI && err != nil {
 			return nil, err
@@ -532,6 +544,8 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 				return nil, err
 			}
 		}
+	} else {
+		log.G(ctx).Debug("Skip moving networks to namespace")
 	}
 
 	c.setStatus(containerCreated)
@@ -540,6 +554,8 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 
 func (h *Host) modifyHostSettings(ctx context.Context, containerID string, req *guestrequest.ModificationRequest) (err error) {
 	switch req.ResourceType {
+	case guestresource.ResourceTypeSCSIDevice:
+		return modifySCSIDevice(ctx, req.RequestType, req.Settings.(*guestresource.SCSIDevice))
 	case guestresource.ResourceTypeMappedVirtualDisk:
 		mvd := req.Settings.(*guestresource.LCOWMappedVirtualDisk)
 		// find the actual controller number on the bus and update the incoming request.
@@ -939,6 +955,23 @@ func newInvalidRequestTypeError(rt guestrequest.RequestType) error {
 	return errors.Errorf("the RequestType %q is not supported", rt)
 }
 
+func modifySCSIDevice(
+	ctx context.Context,
+	rt guestrequest.RequestType,
+	msd *guestresource.SCSIDevice,
+) error {
+	switch rt {
+	case guestrequest.RequestTypeRemove:
+		cNum, err := scsi.ActualControllerNumber(ctx, msd.Controller)
+		if err != nil {
+			return err
+		}
+		return scsi.UnplugDevice(ctx, cNum, msd.Lun)
+	default:
+		return newInvalidRequestTypeError(rt)
+	}
+}
+
 func modifyMappedVirtualDisk(
 	ctx context.Context,
 	rt guestrequest.RequestType,
@@ -979,7 +1012,7 @@ func modifyMappedVirtualDisk(
 				return err
 			}
 		}
-		return scsi.UnplugDevice(ctx, mvd.Controller, mvd.Lun)
+		return nil
 	default:
 		return newInvalidRequestTypeError(rt)
 	}

internal/guest/runtime/hcsv2/workload_container.go

@@ -17,6 +17,8 @@ import (
 	specInternal "github.com/Microsoft/hcsshim/internal/guest/spec"
 	"github.com/Microsoft/hcsshim/internal/guestpath"
 	"github.com/Microsoft/hcsshim/internal/hooks"
+	"github.com/Microsoft/hcsshim/internal/log"
+	"github.com/Microsoft/hcsshim/internal/guest/network"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
 )
@@ -94,7 +96,48 @@ func specHasGPUDevice(spec *oci.Spec) bool {
 	return false
 }
 
+func setupNetworking(ctx context.Context, hostname, id string, spec *oci.Spec) (err error) {
+	// Write the hosts
+	sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname)
+	sandboxHostsPath := getSandboxHostsPath(id)
+	log.G(ctx).WithField("sandboxHostsPath", sandboxHostsPath).Debug("Printing sandboxHosts Path")
+	log.G(ctx).Debug("SandboxHostsContent ", sandboxHostsContent)
+	if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil {
+	 	return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath)
+	}
+
+	// Write resolv.conf
+	ns, err := getNetworkNamespace(getNetworkNamespaceID(spec))
+	log.G(ctx).Debug("Network namespace", ns)
+	if err != nil {
+		log.G(ctx).Debug("Error getting network namespace")
+		return err
+	}
+	var searches, servers []string
+	for _, n := range ns.Adapters() {
+		if len(n.DNSSuffix) > 0 {
+			searches = network.MergeValues(searches, strings.Split(n.DNSSuffix, ","))
+		}
+		if len(n.DNSServerList) > 0 {
+			servers = network.MergeValues(servers, strings.Split(n.DNSServerList, ","))
+		}
+	}
+	resolvContent, err := network.GenerateResolvConfContent(ctx, searches, servers, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to generate sandbox resolv.conf content")
+	}
+	sandboxResolvPath := getSandboxResolvPath(id)
+	log.G(ctx).Debug("sandboxResolvPath", sandboxResolvPath)
+	log.G(ctx).Debug("sandboxResolvContent", resolvContent)
+	if err := os.WriteFile(sandboxResolvPath, []byte(resolvContent), 0644); err != nil {
+		return errors.Wrap(err, "failed to write sandbox resolv.conf")
+	}
+
+	return nil
+}
+
 func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci.Spec) (err error) {
+	log.G(ctx).Debug("Inside setupWorkloadContainerSpec")
 	ctx, span := oc.StartSpan(ctx, "hcsv2::setupWorkloadContainerSpec")
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
@@ -107,6 +150,18 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci.
 		return errors.Errorf("workload container must not change hostname: %s", spec.Hostname)
 	}
 
+	hostname, err := os.Hostname()
+	if hostname == "" {
+		return errors.Wrap(err, "failed to get hostname")
+	}
+	log.G(ctx).WithField("hostname", hostname).Debug("Printing hostname");
+
+	// Set up networking
+	log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> Setting up networking");
+	err = setupNetworking(ctx, hostname, sbid, spec);
+	log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> Completed setting up networking");
+
+	log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> UpdateSandboxMounts")
 	// update any sandbox mounts with the sandboxMounts directory path and create files
 	if err = updateSandboxMounts(sbid, spec); err != nil {
 		return errors.Wrapf(err, "failed to update sandbox mounts for container %v in sandbox %v", id, sbid)
@@ -116,6 +171,7 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci.
 		return errors.Wrapf(err, "failed to update hugepages mounts for container %v in sandbox %v", id, sbid)
 	}
 
+	log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> GenerateWorkloadContainerNetworkMounts")
 	// Add default mounts for container networking (e.g. /etc/hostname, /etc/hosts),
 	// if spec didn't override them explicitly.
 	networkingMounts := specInternal.GenerateWorkloadContainerNetworkMounts(sbid, spec)
@@ -171,5 +227,6 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci.
 	// Clear the windows section as we dont want to forward to runc
 	spec.Windows = nil
 
+	log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> Completed")
 	return nil
 }