[RFC] many: add "ref" in file customizations to support host resources

pkg/blueprint/fsnode_customizations.go

@@ -190,6 +190,15 @@ type FileCustomization struct {
 	Mode string `json:"mode,omitempty" toml:"mode,omitempty"`
 	// Data is the file content in plain text
 	Data string `json:"data,omitempty" toml:"data,omitempty"`
+
+	// Ref references the given files from the host, this makes
+	// the manifest alone no-longer portable (but future offline
+	// manifest bundles will fix that). It will still be
+	// reproducible as the manifest will include all the hashes of
+	// the content so any change will make the build fail.
+	//
+	// XXX: In this PoC only single files are supported.
+	Ref string `json:"ref" toml:"ref,omitempty"`
 }
 
 // Custom TOML unmarshalling for FileCustomization with validation
@@ -291,6 +300,10 @@ func (f *FileCustomization) UnmarshalJSON(data []byte) error {
 
 // ToFsNodeFile converts the FileCustomization to an fsnode.File
 func (f FileCustomization) ToFsNodeFile() (*fsnode.File, error) {
+	if f.Data != "" && f.Ref != "" {
+		return nil, fmt.Errorf("cannot specify both data %q and ref %q", f.Data, f.Ref)
+	}
+
 	var data []byte
 	if f.Data != "" {
 		data = []byte(f.Data)
@@ -309,6 +322,9 @@ func (f FileCustomization) ToFsNodeFile() (*fsnode.File, error) {
 		mode = common.ToPtr(os.FileMode(modeNum))
 	}
 
+	if f.Ref != "" {
+		return fsnode.NewFileForRef(f.Path, mode, f.User, f.Group, f.Ref)
+	}
 	return fsnode.NewFile(f.Path, mode, f.User, f.Group, data)
 }
 

pkg/customizations/fsnode/file.go

@@ -7,6 +7,8 @@ import (
 type File struct {
 	baseFsNode
 	data []byte
+
+	ref string
 }
 
 func (f *File) IsDir() bool {
@@ -20,9 +22,28 @@ func (f *File) Data() []byte {
 	return f.data
 }
 
+func (f *File) Ref() string {
+	return f.ref
+}
+
 // NewFile creates a new file with the given path, data, mode, user and group.
 // user and group can be either a string (user name/group name), an int64 (UID/GID) or nil.
 func NewFile(path string, mode *os.FileMode, user interface{}, group interface{}, data []byte) (*File, error) {
+	return newFile(path, mode, user, group, data, "")
+}
+
+// XXX: only here to keep the diff size down, introduce FsNodeOptions struct
+// instead that can be passed to NewFile.
+//
+// NewFleForRef creates a new file from the given "ref" (usually a local
+// file but potentially a URL later)
+func NewFileForRef(path string, mode *os.FileMode, user interface{}, group interface{}, ref string) (*File, error) {
+	// XXX: add code that clones the mode/user/group from the host
+	// as well (unless overriden by the user)
+	return newFile(path, mode, user, group, nil, ref)
+}
+
+func newFile(path string, mode *os.FileMode, user interface{}, group interface{}, data []byte, ref string) (*File, error) {
 	baseNode, err := newBaseFsNode(path, mode, user, group)
 
 	if err != nil {
@@ -32,5 +53,6 @@ func NewFile(path string, mode *os.FileMode, user interface{}, group interface{}
 	return &File{
 		baseFsNode: *baseNode,
 		data:       data,
+		ref:        ref,
 	}, nil
 }

pkg/hashutil/sha256sum.go

@@ -0,0 +1,26 @@
+package hashutil
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"os"
+)
+
+// Sha256sum() is a convenience wrapper to generate
+// the sha256 hex digest of a file. The hash is the
+// same as from the sha256sum util.
+func Sha256sum(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}

pkg/hashutil/sha256sum_test.go

@@ -0,0 +1,38 @@
+package hashutil_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/osbuild/images/pkg/hashutil"
+)
+
+func TestSha256sum(t *testing.T) {
+	for _, tc := range []struct {
+		inp      string
+		expected string
+	}{
+		// test vectors from
+		// https://www.di-mgt.com.au/sha_testvectors.html
+		{
+			"", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+		}, {
+			"abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+		}, {
+			"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
+			"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1",
+		},
+	} {
+		t.Run(tc.inp, func(t *testing.T) {
+			tmpf := filepath.Join(t.TempDir(), "inp.txt")
+			err := os.WriteFile(tmpf, []byte(tc.inp), 0644)
+			assert.NoError(t, err)
+			hash, err := hashutil.Sha256sum(tmpf)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expected, hash)
+		})
+	}
+}

pkg/manifest/manifest.go

@@ -162,12 +162,15 @@ func (m Manifest) Serialize(depsolvedSets map[string]dnfjson.DepsolveResult, con
 	var mergedInputs osbuild.SourceInputs
 	for _, pipeline := range m.pipelines {
 		pipelines = append(pipelines, pipeline.serialize())
-
 		mergedInputs.Commits = append(mergedInputs.Commits, pipeline.getOSTreeCommits()...)
 		mergedInputs.Depsolved.Packages = append(mergedInputs.Depsolved.Packages, depsolvedSets[pipeline.Name()].Packages...)
 		mergedInputs.Depsolved.Repos = append(mergedInputs.Depsolved.Repos, depsolvedSets[pipeline.Name()].Repos...)
 		mergedInputs.Containers = append(mergedInputs.Containers, pipeline.getContainerSpecs()...)
 		mergedInputs.InlineData = append(mergedInputs.InlineData, pipeline.getInline()...)
+		// XXX: only here to keep diff small
+		if pipelineWithFileRefs, ok := pipeline.(PipelineWithFileRefs); ok {
+			mergedInputs.FileRefs = append(mergedInputs.FileRefs, pipelineWithFileRefs.fileRefs()...)
+		}
 	}
 	for _, pipeline := range m.pipelines {
 		pipeline.serializeEnd()

pkg/manifest/os.go

@@ -897,8 +897,22 @@ func (p *OS) getInline() []string {
 
 	// inline data for custom files
 	for _, file := range p.Files {
-		inlineData = append(inlineData, string(file.Data()))
+		if file.Ref() == "" {
+			inlineData = append(inlineData, string(file.Data()))
+		}
 	}
 
 	return inlineData
 }
+
+func (p *OS) fileRefs() []string {
+	var fileRefs []string
+
+	for _, file := range p.Files {
+		if file.Ref() != "" {
+			fileRefs = append(fileRefs, file.Ref())
+		}
+	}
+
+	return fileRefs
+}

pkg/manifest/pipeline.go

@@ -71,6 +71,13 @@ type Pipeline interface {
 	getInline() []string
 }
 
+// XXX: mostly here to keep the diff down but it would be nice if we had
+// smaller interfaces in general
+type PipelineWithFileRefs interface {
+	// resources from the host
+	fileRefs() []string
+}
+
 // A Base represents the core functionality shared between each of the pipeline
 // implementations, and the Base struct must be embedded in each of them.
 type Base struct {

pkg/osbuild/fsnode.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/osbuild/images/pkg/customizations/fsnode"
+	"github.com/osbuild/images/pkg/hashutil"
 )
 
 // GenFileNodesStages generates the stages for a list of file nodes.
@@ -22,7 +23,16 @@ func GenFileNodesStages(files []*fsnode.File) []*Stage {
 	chownPaths := make(map[string]ChownStagePathOptions)
 
 	for _, file := range files {
-		fileDataChecksum := fmt.Sprintf("%x", sha256.Sum256(file.Data()))
+		var fileDataChecksum string
+		if file.Ref() == "" {
+			fileDataChecksum = fmt.Sprintf("%x", sha256.Sum256(file.Data()))
+		} else {
+			var err error
+			fileDataChecksum, err = hashutil.Sha256sum(file.Ref())
+			if err != nil {
+				panic(err)
+			}
+		}
 		copyStageInputKey := fmt.Sprintf("file-%s", fileDataChecksum)
 		copyStagePaths = append(copyStagePaths, CopyStagePath{
 			From: fmt.Sprintf("input://%s/sha256:%s", copyStageInputKey, fileDataChecksum),

pkg/osbuild/source.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/dnfjson"
+	"github.com/osbuild/images/pkg/hashutil"
 	"github.com/osbuild/images/pkg/ostree"
 	"github.com/osbuild/images/pkg/rpmmd"
 )
@@ -27,7 +28,10 @@ type SourceInputs struct {
 	Depsolved  dnfjson.DepsolveResult
 	Containers []container.Spec
 	Commits    []ostree.CommitSpec
+	// InlineData contans the inline data for fsnode.Files
 	InlineData []string
+	// FileRefs contains the references of paths/urls for fsnode.Files
+	FileRefs []string
 }
 
 // A Sources map contains all the sources made available to an osbuild run
@@ -170,5 +174,24 @@ func GenSources(inputs SourceInputs, rpmDownloader RpmDownloader) (Sources, erro
 		}
 	}
 
+	// collect host resources
+	if len(inputs.FileRefs) > 0 {
+		// XXX: fugly
+		curl := sources["org.osbuild.curl"].(*CurlSource)
+		if curl == nil {
+			curl = NewCurlSource()
+		}
+		for _, hostRes := range inputs.FileRefs {
+			checksum, err := hashutil.Sha256sum(hostRes)
+			if err != nil {
+				return nil, err
+			}
+			curl.Items["sha256:"+checksum] = &CurlSourceOptions{
+				URL: fmt.Sprintf("file:%s", hostRes),
+			}
+		}
+		sources["org.osbuild.curl"] = curl
+	}
+
 	return sources, nil
 }