Merge pull request #98 from hypnoglow/helm3-support-part2

Improve Helm v3 support

.golangci.yml

@@ -1,9 +1,11 @@
 linters:
   enable-all: true
   disable:
-    - lll
+    - dupl
     - funlen
     - gochecknoglobals
+    - godox
+    - lll
     - wsl
 
 issues:

Makefile

@@ -16,6 +16,10 @@ deps:
 build:
 	@./hack/build.sh $(CURDIR) $(PKG)
 
+.PHONY: build-local
+build-local:
+	HELM_S3_PLUGIN_VERSION=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") $(MAKE) build
+
 .PHONY: install
 install:
 	@./hack/install.sh

cmd/helms3/delete.go

@@ -2,14 +2,12 @@ package main
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/pkg/errors"
 
 	"github.com/hypnoglow/helm-s3/internal/awss3"
 	"github.com/hypnoglow/helm-s3/internal/awsutil"
 	"github.com/hypnoglow/helm-s3/internal/helmutil"
-	"github.com/hypnoglow/helm-s3/internal/index"
 )
 
 type deleteAction struct {
@@ -34,14 +32,14 @@ func (act deleteAction) Run(ctx context.Context) error {
 		return errors.WithMessage(err, "fetch current repo index")
 	}
 
-	idx := &index.Index{}
+	idx := helmutil.NewIndex()
 	if err := idx.UnmarshalBinary(b); err != nil {
 		return errors.WithMessage(err, "load index from downloaded file")
 	}
 
 	// Update index.
 
-	chartVersion, err := idx.Delete(act.name, act.version)
+	url, err := idx.Delete(act.name, act.version)
 	if err != nil {
 		return err
 	}
@@ -53,14 +51,12 @@ func (act deleteAction) Run(ctx context.Context) error {
 
 	// Delete the file from S3 and replace index file.
 
-	if len(chartVersion.URLs) < 1 {
-		return fmt.Errorf("chart version index record has no urls")
+	if url != "" {
+		if err := storage.Delete(ctx, url); err != nil {
+			return errors.WithMessage(err, "delete chart file from s3")
+		}
 	}
-	uri := chartVersion.URLs[0]
 
-	if err := storage.Delete(ctx, uri); err != nil {
-		return errors.WithMessage(err, "delete chart file from s3")
-	}
 	if err := storage.PutIndex(ctx, repoEntry.URL(), act.acl, idxReader); err != nil {
 		return errors.WithMessage(err, "upload new index to s3")
 	}

cmd/helms3/init.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/hypnoglow/helm-s3/internal/awss3"
 	"github.com/hypnoglow/helm-s3/internal/awsutil"
-	"github.com/hypnoglow/helm-s3/internal/index"
+	"github.com/hypnoglow/helm-s3/internal/helmutil"
 )
 
 type initAction struct {
@@ -16,7 +16,7 @@ type initAction struct {
 }
 
 func (act initAction) Run(ctx context.Context) error {
-	r, err := index.New().Reader()
+	r, err := helmutil.NewIndex().Reader()
 	if err != nil {
 		return errors.WithMessage(err, "get index reader")
 	}

cmd/helms3/push.go

@@ -2,20 +2,14 @@ package main
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
-	"k8s.io/helm/pkg/chartutil"
-	"k8s.io/helm/pkg/provenance"
-	"k8s.io/helm/pkg/repo"
 
 	"github.com/hypnoglow/helm-s3/internal/awss3"
 	"github.com/hypnoglow/helm-s3/internal/awsutil"
 	"github.com/hypnoglow/helm-s3/internal/helmutil"
-	"github.com/hypnoglow/helm-s3/internal/index"
 )
 
 var (
@@ -70,19 +64,19 @@ func (act pushAction) Run(ctx context.Context) error {
 	// Load chart, calculate required params like hash,
 	// and upload the chart right away.
 
-	chart, err := chartutil.LoadFile(fname)
+	chart, err := helmutil.LoadChart(fname)
 	if err != nil {
-		return fmt.Errorf("file %s is not a helm chart archive", fname)
+		return err
 	}
 
 	repoEntry, err := helmutil.LookupRepoEntry(act.repoName)
 	if err != nil {
 		return err
 	}
 
-	if cachedIndex, err := repo.LoadIndexFile(repoEntry.CacheFile()); err == nil {
+	if cachedIndex, err := helmutil.LoadIndex(repoEntry.CacheFile()); err == nil {
 		// if cached index exists, check if the same chart version exists in it.
-		if cachedIndex.Has(chart.Metadata.Name, chart.Metadata.Version) {
+		if cachedIndex.Has(chart.Name(), chart.Version()) {
 			if act.ignoreIfExists {
 				return nil
 			}
@@ -94,7 +88,7 @@ func (act pushAction) Run(ctx context.Context) error {
 		}
 	}
 
-	hash, err := provenance.DigestFile(fname)
+	hash, err := helmutil.DigestFile(fname)
 	if err != nil {
 		return errors.WithMessage(err, "get chart digest")
 	}
@@ -104,11 +98,6 @@ func (act pushAction) Run(ctx context.Context) error {
 		return errors.Wrap(err, "open chart file")
 	}
 
-	serializedChartMeta, err := json.Marshal(chart.Metadata)
-	if err != nil {
-		return errors.Wrap(err, "encode chart metadata to json")
-	}
-
 	exists, err := storage.Exists(ctx, repoEntry.URL()+"/"+fname)
 	if err != nil {
 		return errors.WithMessage(err, "check if chart already exists in the repository")
@@ -126,7 +115,11 @@ func (act pushAction) Run(ctx context.Context) error {
 	}
 
 	if !act.dryRun {
-		if _, err := storage.PutChart(ctx, repoEntry.URL()+"/"+fname, fchart, string(serializedChartMeta), act.acl, hash, act.contentType); err != nil {
+		chartMetaJSON, err := chart.Metadata().MarshalJSON()
+		if err != nil {
+			return err
+		}
+		if _, err := storage.PutChart(ctx, repoEntry.URL()+"/"+fname, fchart, string(chartMetaJSON), act.acl, hash, act.contentType); err != nil {
 			return errors.WithMessage(err, "upload chart to s3")
 		}
 	}
@@ -142,12 +135,11 @@ func (act pushAction) Run(ctx context.Context) error {
 		return errors.WithMessage(err, "fetch current repo index")
 	}
 
-	idx := &index.Index{}
+	idx := helmutil.NewIndex()
 	if err := idx.UnmarshalBinary(b); err != nil {
 		return errors.WithMessage(err, "load index from downloaded file")
 	}
-
-	if err := idx.AddOrReplace(chart.GetMetadata(), fname, repoEntry.URL(), hash); err != nil {
+	if err := idx.AddOrReplace(chart.Metadata().Value(), fname, repoEntry.URL(), hash); err != nil {
 		return errors.WithMessage(err, "add/replace chart in the index")
 	}
 	idx.SortEntries()

cmd/helms3/reindex.go

@@ -2,13 +2,13 @@ package main
 
 import (
 	"context"
+	"log"
 
 	"github.com/pkg/errors"
 
 	"github.com/hypnoglow/helm-s3/internal/awss3"
 	"github.com/hypnoglow/helm-s3/internal/awsutil"
 	"github.com/hypnoglow/helm-s3/internal/helmutil"
-	"github.com/hypnoglow/helm-s3/internal/index"
 )
 
 type reindexAction struct {
@@ -30,11 +30,13 @@ func (act reindexAction) Run(ctx context.Context) error {
 
 	items, errs := storage.Traverse(ctx, repoEntry.URL())
 
-	builtIndex := make(chan *index.Index, 1)
+	builtIndex := make(chan helmutil.Index, 1)
 	go func() {
-		idx := index.New()
+		idx := helmutil.NewIndex()
 		for item := range items {
-			idx.Add(item.Meta, item.Filename, repoEntry.URL(), item.Hash)
+			if err := idx.Add(item.Meta.Value(), item.Filename, repoEntry.URL(), item.Hash); err != nil {
+				log.Printf("[ERROR] failed to add chart to the index: %s", err)
+			}
 		}
 		idx.SortEntries()
 

go.mod

@@ -7,6 +7,7 @@ replace github.com/docker/docker => github.com/moby/moby v0.7.3-0.20190826074503
 
 require (
 	github.com/Masterminds/semver v1.5.0
+	github.com/Masterminds/semver/v3 v3.0.1
 	github.com/aws/aws-sdk-go v1.19.49
 	github.com/ghodss/yaml v1.0.0
 	github.com/minio/minio-go/v6 v6.0.40
@@ -18,4 +19,5 @@ require (
 	gopkg.in/ini.v1 v1.49.0 // indirect
 	helm.sh/helm/v3 v3.0.0
 	k8s.io/helm v2.16.1+incompatible
+	sigs.k8s.io/yaml v1.1.0
 )

hack/build.sh

@@ -11,7 +11,10 @@ if [ ! -e "${GOPATH}/src/${pkg}" ]; then
     ln -sfn "${projectRoot}" "${GOPATH}/src/${pkg}"
 fi
 
-version="$(cat plugin.yaml | grep "version" | cut -d '"' -f 2)"
+version="${HELM_S3_PLUGIN_VERSION:-}"
+if [ -z "${version}" ]; then
+  version="$(cat plugin.yaml | grep "version" | cut -d '"' -f 2)"
+fi
 
 cd "${GOPATH}/src/${pkg}"
-go build -o bin/helms3 -ldflags "-X main.version=${version}" ./cmd/helms3
+go build -o bin/helms3 -ldflags "-X main.version=${version}" ./cmd/helms3

internal/awss3/storage.go

@@ -3,7 +3,6 @@ package awss3
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/url"
@@ -16,9 +15,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/pkg/errors"
-	"k8s.io/helm/pkg/chartutil"
-	"k8s.io/helm/pkg/proto/hapi/chart"
-	"k8s.io/helm/pkg/provenance"
+
+	"github.com/hypnoglow/helm-s3/internal/helmutil"
 )
 
 const (
@@ -143,26 +141,29 @@ func (s *Storage) traverse(ctx context.Context, repoURI string, items chan<- Cha
 				buf := &bytes.Buffer{}
 				tr := io.TeeReader(objectOut.Body, buf)
 
-				ch, err := chartutil.LoadArchive(tr)
+				ch, err := helmutil.LoadArchive(tr)
 				objectOut.Body.Close()
 				if err != nil {
 					errs <- errors.Wrap(err, "load archive from s3 object")
 					return
 				}
 
-				reindexItem.Meta = ch.Metadata
-				reindexItem.Hash, err = provenance.Digest(buf)
+				digest, err := helmutil.Digest(buf)
 				if err != nil {
 					errs <- errors.WithMessage(err, "get chart hash")
 					return
 				}
+
+				reindexItem.Meta = ch.Metadata()
+				reindexItem.Hash = digest
 			} else {
-				reindexItem.Meta = &chart.Metadata{}
-				if err := json.Unmarshal([]byte(*serializedChartMeta), reindexItem.Meta); err != nil {
+				meta := helmutil.NewChartMetadata()
+				if err := meta.UnmarshalJSON([]byte(*serializedChartMeta)); err != nil {
 					errs <- errors.Wrap(err, "unserialize chart meta")
 					return
 				}
 
+				reindexItem.Meta = meta
 				reindexItem.Hash = *chartDigest
 			}
 
@@ -180,7 +181,7 @@ func (s *Storage) traverse(ctx context.Context, repoURI string, items chan<- Cha
 
 // ChartInfo contains info about particular chart.
 type ChartInfo struct {
-	Meta     *chart.Metadata
+	Meta     helmutil.ChartMetadata
 	Filename string
 	Hash     string
 }

internal/helmutil/chart.go

@@ -0,0 +1,55 @@
+package helmutil
+
+import (
+	"io"
+)
+
+// Chart describes a helm chart.
+type Chart interface {
+	// Name returns chart name.
+	// Example: "foo".
+	Name() string
+
+	// Version returns chart version.
+	// Example: "0.1.0".
+	Version() string
+
+	// Metadata returns chart metadata.
+	Metadata() ChartMetadata
+}
+
+// LoadChart returns chart loaded from the file system by path.
+func LoadChart(fpath string) (Chart, error) {
+	if IsHelm3() {
+		return loadChartV3(fpath)
+	}
+	return loadChartV2(fpath)
+}
+
+// LoadArchive returns chart loaded from the archive file reader.
+func LoadArchive(r io.Reader) (Chart, error) {
+	if IsHelm3() {
+		return loadArchiveV3(r)
+	}
+	return loadArchiveV2(r)
+}
+
+// ChartMetadata describes helm chart metadata.
+type ChartMetadata interface {
+	// MarshalJSON marshals chart metadata to JSON.
+	MarshalJSON() ([]byte, error)
+
+	// UnmarshalJSON unmarshals chart metadata from JSON.
+	UnmarshalJSON([]byte) error
+
+	// Value returns underlying chart metadata value.
+	Value() interface{}
+}
+
+// NewChartMetadata returns a new helm chart metadata.
+func NewChartMetadata() ChartMetadata {
+	if IsHelm3() {
+		return newChartMetadataV3()
+	}
+	return newChartMetadataV2()
+}