update dmverity tool to take directory as input to create a VHD

Signed-off-by: Heather Garvison <hgarvison@microsoft.com>

cmd/dmverity-vhd/README.md

@@ -28,12 +28,36 @@ Create VHDs:
 dmverity-vhd create -i alpine:3.12 -o alpine_3_12_layers
 ```
 
+Output:
+
+```text
+Layer VHD created at alpine_3_12_layers\1ad27bdd166b922492031b1938a4fb2f775e3d98c8f1b72051dad0570a4dd1b5.vhd
+```
+
+Create VHDs from a directory tarball:
+
+```bash
+dmverity-vhd create -i data.tar -o data_vhd -dir
+```
+
+Output:
+
+```text
+Directory VHD created at data_vhd\data.vhd
+```
+
 Compute root hashes:
 
 ```bash
 dmverity-vhd --docker roothash -i alpine:3.12
 ```
 
+Output:
+
+```text
+Layer 0 root hash: 71702a459fa5e6574337e014d9d3936bcf7cb448aaffe3814883caa01fbb4827
+```
+
 Compute root hashes with tarball:
 
 ```bash

cmd/dmverity-vhd/main.go

@@ -28,13 +28,14 @@ const usage = `dmverity-vhd is a command line tool for creating LCOW layer VHDs
 const (
 	usernameFlag       = "username"
 	passwordFlag       = "password"
-	imageFlag          = "image"
+	inputFlag          = "input"
 	verboseFlag        = "verbose"
 	outputDirFlag      = "out-dir"
 	dockerFlag         = "docker"
 	bufferedReaderFlag = "buffered-reader"
 	tarballFlag        = "tarball"
 	hashDeviceVhdFlag  = "hash-dev-vhd"
+	directoryVhdFlag   = "directory-vhd"
 	maxVHDSize         = dmverity.RecommendedVHDSizeGB
 )
 
@@ -305,7 +306,7 @@ func processRemoteImage(imageName string, username string, password string, onLa
 }
 
 func processImageLayers(ctx *cli.Context, onLayer LayerProcessor) (layerDigests map[int]string, layerIDs map[int]string, err error) {
-	imageName := ctx.String(imageFlag)
+	imageName := ctx.String(inputFlag)
 	tarballPath := ctx.GlobalString(tarballFlag)
 	useDocker := ctx.GlobalBool(dockerFlag)
 
@@ -431,8 +432,8 @@ var createVHDCommand = cli.Command{
 	Usage: "creates LCOW layer VHDs inside the output directory with dm-verity super block and merkle tree appended at the end",
 	Flags: []cli.Flag{
 		cli.StringFlag{
-			Name:     imageFlag + ",i",
-			Usage:    "Required: container image reference",
+			Name:     inputFlag + ",i",
+			Usage:    "Required: container image reference or path directory tarfile to create a VHD from",
 			Required: true,
 		},
 		cli.StringFlag{
@@ -452,13 +453,18 @@ var createVHDCommand = cli.Command{
 			Name:  hashDeviceVhdFlag + ",hdv",
 			Usage: "Optional: save hash-device as a VHD",
 		},
+		cli.BoolFlag{
+			Name:  directoryVhdFlag + ",dir",
+			Usage: "Optional: save directory tarfile as a VHD",
+		},
 	},
 	Action: func(ctx *cli.Context) error {
 		verbose := ctx.GlobalBool(verboseFlag)
 		if verbose {
 			log.SetLevel(log.DebugLevel)
 		}
 		verityHashDev := ctx.Bool(hashDeviceVhdFlag)
+		verityDir := ctx.Bool(directoryVhdFlag)
 
 		outDir := ctx.String(outputDirFlag)
 		if _, err := os.Stat(outDir); os.IsNotExist(err) {
@@ -468,41 +474,65 @@ var createVHDCommand = cli.Command{
 			}
 		}
 
-		createVHDLayer := func(layerID string, layerReader io.Reader) error {
-			return createVHD(layerID, layerReader, verityHashDev, outDir)
-		}
+		if verityDir {
+			dirName := ctx.String(inputFlag)
+			log.Debugf("creating VHD from directory tarball at: %q", dirName)
+			dirReader, err := fetchImageTarball(dirName)
+			if err != nil {
+				return fmt.Errorf("failed to get tar file reader from tarball %s: %w", dirName, err)
+			}
+			if err := createVHD(dirName, dirReader, verityHashDev, outDir); err != nil {
+				return fmt.Errorf("failed to create VHD from directory %s: %w", dirName, err)
+			}
+			sanitisedDirName := sanitiseVHDFilename(dirName)
+			src := filepath.Join(os.TempDir(), sanitisedDirName+".vhd")
+			if _, err := os.Stat(src); os.IsNotExist(err) {
+				return fmt.Errorf("directory VHD %s does not exist", src)
+			}
 
-		log.Debug("creating layer VHDs with dm-verity")
-		layerDigests, layerIDs, err := processImageLayers(ctx, createVHDLayer)
-		if err != nil {
-			return err
-		}
+			dst := filepath.Join(outDir, sanitisedDirName+".vhd")
+			if err := moveFile(src, dst); err != nil {
+				return err
+			}
 
-		// Move the VHDs to the output directory
-		// They can't immediately be in the output directory because they have
-		// temporary file names based on the layer id which isn't necessarily
-		// the layer digest
-		for layerNumber := 0; layerNumber < len(layerDigests); layerNumber++ {
-			layerDigest := layerDigests[layerNumber]
-			layerID := layerIDs[layerNumber]
-			sanitisedFileName := sanitiseVHDFilename(layerID)
-
-			suffixes := []string{".vhd"}
-
-			for _, srcSuffix := range suffixes {
-				src := filepath.Join(os.TempDir(), sanitisedFileName+srcSuffix)
-				if _, err := os.Stat(src); os.IsNotExist(err) {
-					return fmt.Errorf("layer VHD %s does not exist", src)
-				}
+			fmt.Fprintf(os.Stdout, "Directory VHD created at %s\n", dst)
+		} else {
+			createVHDLayer := func(layerID string, layerReader io.Reader) error {
+				return createVHD(layerID, layerReader, verityHashDev, outDir)
+			}
 
-				dst := filepath.Join(outDir, layerDigest+srcSuffix)
-				if err := moveFile(src, dst); err != nil {
-					return err
+			log.Debug("creating layer VHDs with dm-verity")
+			layerDigests, layerIDs, err := processImageLayers(ctx, createVHDLayer)
+			if err != nil {
+				return err
+			}
+
+			// Move the VHDs to the output directory
+			// They can't immediately be in the output directory because they have
+			// temporary file names based on the layer id which isn't necessarily
+			// the layer digest
+			for layerNumber := 0; layerNumber < len(layerDigests); layerNumber++ {
+				layerDigest := layerDigests[layerNumber]
+				layerID := layerIDs[layerNumber]
+				sanitisedFileName := sanitiseVHDFilename(layerID)
+
+				suffixes := []string{".vhd"}
+
+				for _, srcSuffix := range suffixes {
+					src := filepath.Join(os.TempDir(), sanitisedFileName+srcSuffix)
+					if _, err := os.Stat(src); os.IsNotExist(err) {
+						return fmt.Errorf("layer VHD %s does not exist", src)
+					}
+
+					dst := filepath.Join(outDir, layerDigest+srcSuffix)
+					if err := moveFile(src, dst); err != nil {
+						return err
+					}
+
+					fmt.Fprintf(os.Stdout, "Layer VHD created at %s\n", dst)
 				}
 
-				fmt.Fprintf(os.Stdout, "Layer VHD created at %s\n", dst)
 			}
-
 		}
 
 		return nil
@@ -514,7 +544,7 @@ var rootHashVHDCommand = cli.Command{
 	Usage: "compute root hashes for each LCOW layer VHD",
 	Flags: []cli.Flag{
 		cli.StringFlag{
-			Name:     imageFlag + ",i",
+			Name:     inputFlag + ",i",
 			Usage:    "Required: container image reference",
 			Required: true,
 		},

ext4/internal/compactext4/compact.go

@@ -538,7 +538,6 @@ func (w *Writer) lookup(name string, mustExist bool) (*inode, *inode, string, er
 // with the same permissions as that of it's parent directory. It is expected that the a
 // call to make these parent directories will be made at a later point with the correct
 // permissions, at that time the permissions of these directories will be updated.
-// We treat Atime, Mtime, Ctime, and Crtime in the same way.
 func (w *Writer) MakeParents(name string) error {
 	if err := w.finishInode(); err != nil {
 		return err
@@ -557,10 +556,10 @@ func (w *Writer) MakeParents(name string) error {
 		if _, ok := root.Children[dirname]; !ok {
 			f := &File{
 				Mode:     root.Mode,
-				Atime:    fsTimeToTime(root.Atime),
-				Mtime:    fsTimeToTime(root.Mtime),
-				Ctime:    fsTimeToTime(root.Ctime),
-				Crtime:   fsTimeToTime(root.Crtime),
+				Atime:    time.Now(),
+				Mtime:    time.Now(),
+				Ctime:    time.Now(),
+				Crtime:   time.Now(),
 				Size:     0,
 				Uid:      root.Uid,
 				Gid:      root.Gid,

ext4/tar2ext4/tar2ext4_test.go

@@ -1,9 +1,6 @@
 package tar2ext4
 
 import (
-	"crypto/sha256"
-	"fmt"
-	"io"
 	"path/filepath"
 	"testing"
 
@@ -165,76 +162,3 @@ func Test_TarHardlinkToSymlink(t *testing.T) {
 		t.Fatalf("failed to convert tar to layer vhd: %s", err)
 	}
 }
-
-func calcExt4Sha256(t *testing.T, layerTar *os.File) string {
-	t.Helper()
-	if _, err := layerTar.Seek(0, 0); err != nil {
-		t.Fatalf("failed to seek file: %s", err)
-	}
-
-	opts := []Option{ConvertWhiteout}
-
-	tmpExt4Path := filepath.Join(os.TempDir(), "test.ext4")
-	layerVhd, err := os.Create(tmpExt4Path)
-	if err != nil {
-		t.Fatalf("failed to create output VHD: %s", err)
-	}
-	defer os.Remove(tmpExt4Path)
-
-	if err := Convert(layerTar, layerVhd, opts...); err != nil {
-		t.Fatalf("failed to convert tar to layer vhd: %s", err)
-	}
-
-	if _, err := layerVhd.Seek(0, 0); err != nil {
-		t.Fatalf("failed to seek file: %s", err)
-	}
-
-	hasher := sha256.New()
-	if _, err = io.Copy(hasher, layerVhd); err != nil {
-		t.Fatalf("filed to initialize hasher: %s", err)
-	}
-
-	hash := hasher.Sum(nil)
-	return fmt.Sprintf("%x", hash)
-}
-
-// Test_MissingParentDirExpansion tests that we are correctly able to expand a layer tar file
-// even if its file does not include the parent directory in its file name.
-func Test_MissingParentDirExpansion(t *testing.T) {
-	tmpTarFilePath := filepath.Join(os.TempDir(), "test-layer.tar")
-	layerTar, err := os.Create(tmpTarFilePath)
-	if err != nil {
-		t.Fatalf("failed to create output file: %s", err)
-	}
-	defer os.Remove(tmpTarFilePath)
-
-	tw := tar.NewWriter(layerTar)
-	var file = struct {
-		path, body string
-	}{"foo/bar.txt", "inside bar.txt"}
-	hdr := &tar.Header{
-		Name:       file.path,
-		Mode:       0777,
-		Size:       int64(len(file.body)),
-		ModTime:    time.Now(),
-		AccessTime: time.Now(),
-		ChangeTime: time.Now(),
-	}
-	if err := tw.WriteHeader(hdr); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := tw.Write([]byte(file.body)); err != nil {
-		t.Fatal(err)
-	}
-	if err := tw.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Now import the tar file and check the conversion to ext4 is deterministic.
-	hash1 := calcExt4Sha256(t, layerTar)
-	hash2 := calcExt4Sha256(t, layerTar)
-
-	if hash1 != hash2 {
-		t.Fatalf("hash doesn't match")
-	}
-}

internal/uvm/create_lcow.go

@@ -471,10 +471,16 @@ func makeLCOWVMGSDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_
 		if opts.DmVerityMode {
 			logrus.Debug("makeLCOWVMGSDoc DmVerityMode true")
 			scsiController0 := guestrequest.ScsiControllerGuids[0]
-			doc.VirtualMachine.Devices.Scsi[scsiController0].Attachments["0"] = hcsschema.Attachment{
-				Type_:    "VirtualDisk",
-				Path:     dmVerityRootFsFullPath,
-				ReadOnly: true,
+			doc.VirtualMachine.Devices.Scsi = map[string]hcsschema.Scsi{
+				scsiController0: {
+					Attachments: map[string]hcsschema.Attachment{
+						"0": {
+							Type_:    "VirtualDisk",
+							Path:     dmVerityRootFsFullPath,
+							ReadOnly: true,
+						},
+					},
+				},
 			}
 			uvm.reservedSCSISlots = append(uvm.reservedSCSISlots, scsi.Slot{Controller: 0, LUN: 0})
 		}