From 041cb8a90e8a399fda9c44f257c18141da43a634 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Tue, 31 May 2022 12:35:07 -0700 Subject: [PATCH 1/2] PatchSet improvements We're making PatchSet more generic and efficient. Before this change, PatchSet would not work with ConfigMaps. We're also removing a dependency on util/patch from Cluster API. PatchSet is also now safe for concurrent use. Signed-off-by: Ivan Mikushin --- pkg/v1/util/patchset/patchset.go | 49 ++++++++++++++++++++++----- pkg/v1/util/patchset/patchset_test.go | 35 ++++++++++++++++++- 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/pkg/v1/util/patchset/patchset.go b/pkg/v1/util/patchset/patchset.go index d6258eb1ba..232e5e0d16 100644 --- a/pkg/v1/util/patchset/patchset.go +++ b/pkg/v1/util/patchset/patchset.go @@ -6,11 +6,12 @@ package patchset import ( "context" + "reflect" + "sync" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" kerrors "k8s.io/apimachinery/pkg/util/errors" - "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -35,29 +36,53 @@ func New(c client.Client) PatchSet { } type patchSet struct { + sync.RWMutex client client.Client patchers map[types.UID]*patcher } type patcher struct { - obj client.Object - helper *patch.Helper + obj client.Object + beforeObj client.Object +} + +func (h *patcher) patch(ctx context.Context, c client.Client, obj client.Object) error { + for _, f := range []func() error{ + func() error { + return c.Patch(ctx, obj, client.MergeFromWithOptions(h.beforeObj, client.MergeFromWithOptimisticLock{})) + }, + func() error { + err := c.Status().Patch(ctx, obj, client.MergeFromWithOptions(h.beforeObj, client.MergeFromWithOptimisticLock{})) + return kerrors.FilterOut(err, apierrors.IsNotFound) // status resource may not exist + }, + } { + if err := f(); err != nil { + return err + } + h.beforeObj.SetResourceVersion(obj.GetResourceVersion()) // get the new resourceVersion after a successful patch + } + return nil } func (ps *patchSet) Add(obj client.Object) { + ps.Lock() + defer ps.Unlock() + uid := obj.GetUID() if _, exists := ps.patchers[uid]; exists { return } - helper, _ := patch.NewHelper(obj, ps.client) ps.patchers[uid] = &patcher{ - obj: obj, - helper: helper, + obj: obj, + beforeObj: obj.DeepCopyObject().(client.Object), } } -func (ps patchSet) Objects() map[types.UID]client.Object { +func (ps *patchSet) Objects() map[types.UID]client.Object { + ps.RLock() + defer ps.RUnlock() + result := make(map[types.UID]client.Object, len(ps.patchers)) for k, v := range ps.patchers { result[k] = v.obj @@ -65,10 +90,16 @@ func (ps patchSet) Objects() map[types.UID]client.Object { return result } -func (ps patchSet) Apply(ctx context.Context) error { +func (ps *patchSet) Apply(ctx context.Context) error { + ps.Lock() + defer ps.Unlock() + errs := make([]error, 0, len(ps.patchers)) for _, patcher := range ps.patchers { - if err := patcher.helper.Patch(ctx, patcher.obj); err != nil { + if reflect.DeepEqual(patcher.obj, patcher.beforeObj) { + continue + } + if err := patcher.patch(ctx, ps.client, patcher.obj); err != nil { if !patcher.obj.GetDeletionTimestamp().IsZero() && isNotFound(err) { continue } diff --git a/pkg/v1/util/patchset/patchset_test.go b/pkg/v1/util/patchset/patchset_test.go index 0b563bb0bc..da09eb0f5c 100644 --- a/pkg/v1/util/patchset/patchset_test.go +++ b/pkg/v1/util/patchset/patchset_test.go @@ -11,9 +11,13 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/uuid" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/controller-runtime/pkg/client" @@ -35,7 +39,7 @@ func TestPatchsetUnit(t *testing.T) { RunSpecs(t, "vlabel Package Unit Tests") } -var _ = Describe("Patchset", func() { +var _ = Describe("PatchSet", func() { var ( tkr1156 *runv1.TanzuKubernetesRelease tkr116 *runv1.TanzuKubernetesRelease @@ -103,6 +107,22 @@ var _ = Describe("Patchset", func() { }) }) + When("there's a conflict applying the patchset", func() { + BeforeEach(func() { + ps = New(&conflictedPatcher{}) + }) + + It("return a Conflict error", func() { + changedTKRs := []*runv1.TanzuKubernetesRelease{tkr116, tkr117} + for _, tkr := range changedTKRs { + tkr.Labels = labels.Set{"newLabel" + tkr.Name: ""} + } + err := ps.Apply(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(kerrors.FilterOut(err, apierrors.IsConflict)).To(BeNil()) + }) + }) + When("a patched object is slated for deletion", func() { BeforeEach(func() { tkr117.DeletionTimestamp = &metav1.Time{Time: time.Now()} @@ -140,6 +160,19 @@ func (p *countingPatcher) Patch(ctx context.Context, obj client.Object, patch cl return p.Client.Patch(ctx, obj, patch, opts...) } +type conflictedPatcher struct { + client.Client +} + +func (*conflictedPatcher) Patch(_ context.Context, obj client.Object, _ client.Patch, _ ...client.PatchOption) error { + gvk := obj.GetObjectKind().GroupVersionKind() + groupResource := schema.GroupResource{ + Group: gvk.Group, + Resource: gvk.Kind, + } + return apierrors.NewConflict(groupResource, obj.GetName(), errors.New("re-read the resource before patching")) +} + func newScheme() *runtime.Scheme { s := runtime.NewScheme() _ = runv1.AddToScheme(s) From f7e6dabc4cef5bda2231913e67a7db9020fada95 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Thu, 26 May 2022 19:38:52 -0700 Subject: [PATCH 2/2] Install TKR package contents directly TKR package contents get automatically installed without the use of Carvel PackageInstall resource (which doesn't support shared resources across packages). This is necessary because a TKR package may contain resources that are reused by other TKR packages, e.g. different TKRs shipping the same Kubernetes version may share some OSImages or addon packages (addon packages may even be reused across K8s versions). Inject Registry as an interface This improves testability and modularity of affected components, and reuse of Registry as a component. Now, both Fetcher and TKR Package Reconciler re-use the same Registry instance provided externally. Only fetch compatible TKR BOMs and TKR packages We're also now using compatibility information to download only compatible TKR packages and TKR BOMs into the management cluster. Signed-off-by: Ivan Mikushin --- .../source/tkr_source_controller.go | 2 +- pkg/v1/tkr/pkg/types/bommetadata.go | 4 +- .../tkr-source/compatibility/reconciler.go | 42 +-- .../tkr-source/fetcher/configurer.go | 76 ------ .../controller/tkr-source/fetcher/fetcher.go | 94 ++++--- .../controller/tkr-source/fetcher/helper.go | 20 -- pkg/v2/tkr/controller/tkr-source/main.go | 160 ++++++++---- .../controller/tkr-source/pkgcr/reconciler.go | 239 ++++++++++++++++-- .../tkr-source/pkgcr/reconciler_test.go | 101 +++++++- .../tkr-source/registry/registry.go | 157 ++++++++++++ pkg/v2/tkr/util/sets/strings.go | 55 ++++ pkg/v2/tkr/util/version/compatibility.go | 14 + 12 files changed, 719 insertions(+), 245 deletions(-) delete mode 100644 pkg/v2/tkr/controller/tkr-source/fetcher/configurer.go create mode 100644 pkg/v2/tkr/controller/tkr-source/registry/registry.go create mode 100644 pkg/v2/tkr/util/sets/strings.go create mode 100644 pkg/v2/tkr/util/version/compatibility.go diff --git a/pkg/v1/tkr/controllers/source/tkr_source_controller.go b/pkg/v1/tkr/controllers/source/tkr_source_controller.go index a1dd220126..70d30e2278 100644 --- a/pkg/v1/tkr/controllers/source/tkr_source_controller.go +++ b/pkg/v1/tkr/controllers/source/tkr_source_controller.go @@ -15,7 +15,6 @@ import ( "github.com/go-logr/logr" ctlimg "github.com/k14s/imgpkg/pkg/imgpkg/registry" "github.com/pkg/errors" - "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + "sigs.k8s.io/yaml" runv1 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha1" "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/constants" diff --git a/pkg/v1/tkr/pkg/types/bommetadata.go b/pkg/v1/tkr/pkg/types/bommetadata.go index 5d41904362..7cf1b7e62d 100644 --- a/pkg/v1/tkr/pkg/types/bommetadata.go +++ b/pkg/v1/tkr/pkg/types/bommetadata.go @@ -5,8 +5,8 @@ package types // ManagementClusterVersion contains kubernetes versions that are supported by the management cluster with a certain TKG version. type ManagementClusterVersion struct { - TKGVersion string `yaml:"version"` - SupportedKubernetesVersions []string `yaml:"supportedKubernetesVersions"` + TKGVersion string `json:"version"` + SupportedKubernetesVersions []string `json:"supportedKubernetesVersions"` } // CompatibilityMetadata contains tanzu release support matrix diff --git a/pkg/v2/tkr/controller/tkr-source/compatibility/reconciler.go b/pkg/v2/tkr/controller/tkr-source/compatibility/reconciler.go index d257c9ee29..79d1a95e91 100644 --- a/pkg/v2/tkr/controller/tkr-source/compatibility/reconciler.go +++ b/pkg/v2/tkr/controller/tkr-source/compatibility/reconciler.go @@ -9,7 +9,6 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" - "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" @@ -22,18 +21,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/source" + "sigs.k8s.io/yaml" runv1 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3" "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/constants" "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/types" "github.com/vmware-tanzu/tanzu-framework/pkg/v1/util/patchset" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/sets" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/version" ) type Reconciler struct { + version.Compatibility + Ctx context.Context Log logr.Logger Client client.Client + Config Config +} +type Compatibility struct { + Client client.Client Config Config } @@ -112,12 +120,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct } func (r *Reconciler) updateTKRCompatibleCondition(ctx context.Context, tkr *runv1.TanzuKubernetesRelease) error { - compatibleSet, err := r.getCompatibleSet(ctx) + compatibleSet, err := r.CompatibleVersions(ctx) if err != nil { return err } - if _, isCompatible := compatibleSet[tkr.Spec.Version]; isCompatible { + if compatibleSet.Has(tkr.Spec.Version) { conditions.MarkTrue(tkr, runv1.ConditionCompatible) return nil } @@ -125,38 +133,30 @@ func (r *Reconciler) updateTKRCompatibleCondition(ctx context.Context, tkr *runv return nil } -func (r *Reconciler) getCompatibleSet(ctx context.Context) (map[string]struct{}, error) { - mgmtClusterVersion, err := r.getManagementClusterVersion(ctx) +func (c *Compatibility) CompatibleVersions(ctx context.Context) (sets.StringSet, error) { + mgmtClusterVersion, err := c.getManagementClusterVersion(ctx) if err != nil { return nil, errors.Wrap(err, "failed to get the management cluster info") } - metadata, err := r.compatibilityMetadata(ctx) + metadata, err := c.compatibilityMetadata(ctx) if err != nil { return nil, errors.Wrapf(err, "failed to get BOM compatibility metadata") } for _, mgmtVersion := range metadata.ManagementClusterVersions { if mgmtClusterVersion == mgmtVersion.TKGVersion { - return stringSet(mgmtVersion.SupportedKubernetesVersions), nil + return sets.Strings(mgmtVersion.SupportedKubernetesVersions...), nil } } - return stringSet(nil), nil -} - -func stringSet(ss []string) map[string]struct{} { - result := make(map[string]struct{}, len(ss)) - for _, s := range ss { - result[s] = struct{}{} - } - return result + return sets.Strings(), nil } // getManagementClusterVersion get the version of the management cluster -func (r *Reconciler) getManagementClusterVersion(ctx context.Context) (string, error) { +func (c *Compatibility) getManagementClusterVersion(ctx context.Context) (string, error) { clusterList := &clusterv1.ClusterList{} - if err := r.Client.List(ctx, clusterList, client.HasLabels{constants.ManagementClusterRoleLabel}); err != nil { + if err := c.Client.List(ctx, clusterList, client.HasLabels{constants.ManagementClusterRoleLabel}); err != nil { return "", errors.Wrap(err, "failed to list clusters") } @@ -169,10 +169,10 @@ func (r *Reconciler) getManagementClusterVersion(ctx context.Context) (string, e return "", errors.New("failed to get management cluster info") } -func (r *Reconciler) compatibilityMetadata(ctx context.Context) (*types.CompatibilityMetadata, error) { +func (c *Compatibility) compatibilityMetadata(ctx context.Context) (*types.CompatibilityMetadata, error) { cm := &corev1.ConfigMap{} - cmObjectKey := client.ObjectKey{Namespace: r.Config.TKRNamespace, Name: constants.BOMMetadataConfigMapName} - if err := r.Client.Get(ctx, cmObjectKey, cm); err != nil { + cmObjectKey := client.ObjectKey{Namespace: c.Config.TKRNamespace, Name: constants.BOMMetadataConfigMapName} + if err := c.Client.Get(ctx, cmObjectKey, cm); err != nil { return nil, err } diff --git a/pkg/v2/tkr/controller/tkr-source/fetcher/configurer.go b/pkg/v2/tkr/controller/tkr-source/fetcher/configurer.go deleted file mode 100644 index 1adddae93a..0000000000 --- a/pkg/v2/tkr/controller/tkr-source/fetcher/configurer.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2022 VMware, Inc. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package fetcher - -import ( - "context" - "os" - "path" - - ctlimg "github.com/k14s/imgpkg/pkg/imgpkg/registry" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - k8serr "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - - "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/registry" -) - -const ( - configMapName = "tkr-controller-config" - caCertsKey = "caCerts" - registryCertsFile = "registry_certs" -) - -func (f *Fetcher) configure(ctx context.Context) error { - configMap := &corev1.ConfigMap{} - if err := f.Client.Get(ctx, - types.NamespacedName{Namespace: f.Config.TKRNamespace, Name: configMapName}, - configMap); !k8serr.IsNotFound(err) { - return errors.Wrapf(err, "unable to get the ConfigMap %s", configMapName) - } - - err := addTrustedCerts(configMap.Data[caCertsKey]) - if err != nil { - return errors.Wrap(err, "failed to add certs") - } - - f.registryOps = ctlimg.Opts{ - VerifyCerts: f.Config.VerifyRegistryCert, - Anon: true, - } - - // Add custom CA cert paths only if VerifyCerts is enabled - if f.registryOps.VerifyCerts { - registryCertPath, err := getRegistryCertFile() - if err == nil { - if _, err = os.Stat(registryCertPath); err == nil { - f.registryOps.CACertPaths = []string{registryCertPath} - } - } - } - - f.registry, err = registry.New(&f.registryOps) - return err -} - -func addTrustedCerts(certChain string) (err error) { - if certChain == "" { - return nil - } - - filePath, err := getRegistryCertFile() - if err != nil { - return err - } - - return os.WriteFile(filePath, []byte(certChain), 0644) -} -func getRegistryCertFile() (string, error) { - home, err := os.UserHomeDir() - if err != nil { - return "", errors.Wrap(err, "could not locate local tanzu dir") - } - return path.Join(home, registryCertsFile), nil -} diff --git a/pkg/v2/tkr/controller/tkr-source/fetcher/fetcher.go b/pkg/v2/tkr/controller/tkr-source/fetcher/fetcher.go index b85dc622c6..d746ddbf2f 100644 --- a/pkg/v2/tkr/controller/tkr-source/fetcher/fetcher.go +++ b/pkg/v2/tkr/controller/tkr-source/fetcher/fetcher.go @@ -14,23 +14,24 @@ import ( "time" "github.com/go-logr/logr" - ctlimg "github.com/k14s/imgpkg/pkg/imgpkg/registry" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kerrors "k8s.io/apimachinery/pkg/util/errors" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/yaml" kapppkgv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" - "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/constants" - "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/registry" "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/types" "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/pkgcr" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/registry" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/sets" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/version" ) type Fetcher struct { @@ -38,8 +39,9 @@ type Fetcher struct { Client client.Client Config Config - registryOps ctlimg.Opts - registry registry.Registry + Registry registry.Registry + + Compatibility version.Compatibility } // Config contains the controller manager context. @@ -48,7 +50,6 @@ type Config struct { BOMImagePath string BOMMetadataImagePath string TKRRepoImagePath string - VerifyRegistryCert bool TKRDiscoveryOption TKRDiscoveryIntervals } @@ -58,12 +59,11 @@ type TKRDiscoveryIntervals struct { ContinuousDiscoveryFrequency time.Duration } -func (f *Fetcher) Start(ctx context.Context) error { - f.Log.Info("Performing configuration setup") - if err := f.configure(ctx); err != nil { - return errors.Wrap(err, "failed to configure the controller") - } +func (f *Fetcher) SetupWithManager(m ctrl.Manager) error { + return m.Add(f) +} +func (f *Fetcher) Start(ctx context.Context) error { f.Log.Info("Performing an initial release discovery") f.initialReconcile(ctx, f.Config.TKRDiscoveryOption.InitialDiscoveryFrequency, InitialDiscoveryRetry) @@ -112,14 +112,18 @@ func (f *Fetcher) tkrDiscovery(ctx context.Context, frequency time.Duration) { } func (f *Fetcher) fetchAll(ctx context.Context) error { + fetchTKRCompatibilityDone := make(chan struct{}) return kerrors.AggregateGoroutines( func() error { + defer close(fetchTKRCompatibilityDone) return f.fetchTKRCompatibilityCM(ctx) }, func() error { + <-fetchTKRCompatibilityDone return f.fetchTKRBOMConfigMaps(ctx) }, func() error { + <-fetchTKRCompatibilityDone return f.fetchTKRPackages(ctx) }) } @@ -160,7 +164,7 @@ func (f *Fetcher) fetchTKRCompatibilityCM(ctx context.Context) error { func (f *Fetcher) fetchCompatibilityMetadata() (*types.CompatibilityMetadata, error) { f.Log.Info("Listing BOM metadata image tags", "image", f.Config.BOMMetadataImagePath) - tags, err := f.registry.ListImageTags(f.Config.BOMMetadataImagePath) + tags, err := f.Registry.ListImageTags(f.Config.BOMMetadataImagePath) if err != nil { return nil, errors.Wrap(err, "failed to list compatibility metadata image tags") } @@ -184,7 +188,7 @@ func (f *Fetcher) fetchCompatibilityMetadata() (*types.CompatibilityMetadata, er for i := len(tagNum) - 1; i >= 0; i-- { tagName := fmt.Sprintf("v%d", tagNum[i]) f.Log.Info("Fetching BOM metadata image", "image", f.Config.BOMMetadataImagePath, "tag", tagName) - metadataContent, err = f.registry.GetFile(fmt.Sprintf("%s:%s", f.Config.BOMMetadataImagePath, tagName), "") + metadataContent, err = f.Registry.GetFile(fmt.Sprintf("%s:%s", f.Config.BOMMetadataImagePath, tagName), "") if err == nil { if err = yaml.Unmarshal(metadataContent, &metadata); err == nil { break @@ -209,15 +213,18 @@ func (f *Fetcher) fetchTKRBOMConfigMaps(ctx context.Context) error { default: } + compatibleImageTags, err := f.compatibleImageTags(ctx) + if err != nil { + return err + } + f.Log.Info("Listing BOM image tags", "image", f.Config.BOMImagePath) - imageTags, err := f.registry.ListImageTags(f.Config.BOMImagePath) + imageTags, err := f.Registry.ListImageTags(f.Config.BOMImagePath) if err != nil { return errors.Wrap(err, "failed to list current available BOM image tags") } - tagMap := make(map[string]bool) - for _, tag := range imageTags { - tagMap[tag] = false - } + + tagsToDownload := compatibleImageTags.Intersect(sets.Strings(imageTags...)) cmList := &corev1.ConfigMapList{} if err := f.Client.List(ctx, cmList, &client.ListOptions{Namespace: f.Config.TKRNamespace}); err != nil { @@ -226,25 +233,18 @@ func (f *Fetcher) fetchTKRBOMConfigMaps(ctx context.Context) error { for i := range cmList.Items { if imageTag, ok := cmList.Items[i].ObjectMeta.Annotations[constants.BomConfigMapImageTagAnnotation]; ok { - if _, ok := tagMap[imageTag]; ok { - tagMap[imageTag] = true - } + tagsToDownload.Remove(imageTag) } } - var errs errorSlice - for tag, exist := range tagMap { - if !exist { - if err := f.createBOMConfigMap(ctx, tag); err != nil { - errs = append(errs, errors.Wrapf(err, "failed to create BOM ConfigMap for image %s", fmt.Sprintf("%s:%s", f.Config.BOMImagePath, tag))) - } + var errs []error + for tag := range tagsToDownload { + if err := f.createBOMConfigMap(ctx, tag); err != nil { + errs = append(errs, errors.Wrapf(err, "failed to create BOM ConfigMap for image %s", fmt.Sprintf("%s:%s", f.Config.BOMImagePath, tag))) } } - if len(errs) != 0 { - return errs - } f.Log.Info("Done reconciling BOM images", "image", f.Config.BOMImagePath) - return nil + return kerrors.NewAggregate(errs) } func (f *Fetcher) createBOMConfigMap(ctx context.Context, tag string) error { @@ -255,7 +255,7 @@ func (f *Fetcher) createBOMConfigMap(ctx context.Context, tag string) error { } f.Log.Info("Fetching BOM", "image", f.Config.BOMImagePath, "tag", tag) - bomContent, err := f.registry.GetFile(fmt.Sprintf("%s:%s", f.Config.BOMImagePath, tag), "") + bomContent, err := f.Registry.GetFile(fmt.Sprintf("%s:%s", f.Config.BOMImagePath, tag), "") if err != nil { return errors.Wrapf(err, "failed to get the BOM file from image %s:%s", f.Config.BOMImagePath, tag) } @@ -305,24 +305,38 @@ func (f *Fetcher) fetchTKRPackages(ctx context.Context) error { default: } + compatibleImageTags, err := f.compatibleImageTags(ctx) + if err != nil { + return err + } + f.Log.Info("Listing TKR Package Repository tags", "image", f.Config.TKRRepoImagePath) - imageTags, err := f.registry.ListImageTags(f.Config.TKRRepoImagePath) + imageTags, err := f.Registry.ListImageTags(f.Config.TKRRepoImagePath) if err != nil { return errors.Wrap(err, "failed to list current available TKR Package Repository image tags") } - var errs errorSlice - for _, tag := range imageTags { + imageTagsToPull := compatibleImageTags.Intersect(sets.Strings(imageTags...)) + + var errs []error + for tag := range imageTagsToPull { if err := f.createTKRPackages(ctx, tag); err != nil { errs = append(errs, errors.Wrapf(err, "failed to create TKR Package for image %s", fmt.Sprintf("%s:%s", f.Config.TKRRepoImagePath, tag))) } } - if len(errs) != 0 { - return errs - } f.Log.Info("Done fetching TKR Packages", "image", f.Config.TKRRepoImagePath) - return nil + return kerrors.NewAggregate(errs) +} + +func (f *Fetcher) compatibleImageTags(ctx context.Context) (sets.StringSet, error) { + compatibleTKRVersions, err := f.Compatibility.CompatibleVersions(ctx) + if err != nil { + return nil, err + } + return compatibleTKRVersions.Map(func(v string) string { + return strings.ReplaceAll(v, "+", "_") + }), nil } func (f *Fetcher) createTKRPackages(ctx context.Context, tag string) error { @@ -334,7 +348,7 @@ func (f *Fetcher) createTKRPackages(ctx context.Context, tag string) error { imageName := fmt.Sprintf("%s:%s", f.Config.TKRRepoImagePath, tag) f.Log.Info("Fetching TKR Package Repository imgpkg bundle", "image", imageName) - bundleContent, err := f.registry.GetFiles(imageName) + bundleContent, err := f.Registry.GetFiles(imageName) if err != nil { return errors.Wrapf(err, "failed to fetch the BOM file from image '%s'", imageName) } diff --git a/pkg/v2/tkr/controller/tkr-source/fetcher/helper.go b/pkg/v2/tkr/controller/tkr-source/fetcher/helper.go index a4a12b8c17..4f94f0458c 100644 --- a/pkg/v2/tkr/controller/tkr-source/fetcher/helper.go +++ b/pkg/v2/tkr/controller/tkr-source/fetcher/helper.go @@ -3,27 +3,7 @@ package fetcher -import ( - "strings" -) - const ( // InitialDiscoveryRetry is the number of retries for the initial TKR sync-up InitialDiscoveryRetry = 10 ) - -type errorSlice []error - -func (e errorSlice) Error() string { - if len(e) == 0 { - return "" - } - sb := &strings.Builder{} - for i, err := range e { - if i != 0 { - sb.WriteString(", ") - } - sb.WriteString(err.Error()) - } - return sb.String() -} diff --git a/pkg/v2/tkr/controller/tkr-source/main.go b/pkg/v2/tkr/controller/tkr-source/main.go index 884c417a6b..2ab9e07169 100644 --- a/pkg/v2/tkr/controller/tkr-source/main.go +++ b/pkg/v2/tkr/controller/tkr-source/main.go @@ -4,8 +4,11 @@ package main import ( + "context" "flag" + "fmt" "os" + "reflect" "time" corev1 "k8s.io/api/core/v1" @@ -20,12 +23,12 @@ import ( kapppkgiv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" kapppkgv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" - runv1 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3" "github.com/vmware-tanzu/tanzu-framework/pkg/v1/buildinfo" "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/compatibility" "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/fetcher" "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/pkgcr" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/registry" ) var ( @@ -41,17 +44,19 @@ func init() { utilruntime.Must(corev1.AddToScheme(scheme)) } -func main() { - var metricsAddr string - var tkrNamespace string - var tkrPkgServiceAccountName string - var bomImagePath string - var bomMetadataImagePath string - var tkrRepoImagePath string - var initTKRDiscoveryFreq int - var continuousTKRDiscoverFreq int - var skipVerifyRegistryCerts bool +var ( + metricsAddr string + tkrNamespace string + tkrPkgServiceAccountName string + bomImagePath string + bomMetadataImagePath string + tkrRepoImagePath string + initTKRDiscoveryFreq int + continuousTKRDiscoverFreq int + skipVerifyRegistryCerts bool +) +func init() { flag.StringVar(&metricsAddr, "metrics-bind-addr", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&tkrNamespace, "namespace", "tkg-system", "Namespace for TKR related resources") flag.StringVar(&tkrPkgServiceAccountName, "sa-name", "tkr-source-controller-manager-sa", "ServiceAccount name used by TKR PackageInstalls") @@ -63,16 +68,84 @@ func main() { flag.IntVar(&continuousTKRDiscoverFreq, "continuous-discover-frequency", 600, "Continuous TKR discovery frequency in seconds") flag.Parse() + setupLog.Info("Version", "version", buildinfo.Version, "buildDate", buildinfo.Date, "sha", buildinfo.SHA) + + registryConfig = registry.Config{ + TKRNamespace: tkrNamespace, + VerifyRegistryCert: !skipVerifyRegistryCerts, + } + fetcherConfig = fetcher.Config{ + TKRNamespace: tkrNamespace, + BOMImagePath: bomImagePath, + BOMMetadataImagePath: bomMetadataImagePath, + TKRRepoImagePath: tkrRepoImagePath, + TKRDiscoveryOption: fetcher.TKRDiscoveryIntervals{ + InitialDiscoveryFrequency: time.Duration(initTKRDiscoveryFreq) * time.Second, + ContinuousDiscoveryFrequency: time.Duration(continuousTKRDiscoverFreq) * time.Second, + }, + } + pkgcrConfig = pkgcr.Config{ + ServiceAccountName: tkrPkgServiceAccountName, + } + compatibilityConfig = compatibility.Config{ + TKRNamespace: tkrNamespace, + } +} + +var ( + registryConfig registry.Config + fetcherConfig fetcher.Config + pkgcrConfig pkgcr.Config + compatibilityConfig compatibility.Config +) + +func main() { opts := zap.Options{ Development: true, } opts.BindFlags(flag.CommandLine) - flag.Parse() - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - setupLog.Info("Version", "version", buildinfo.Version, "buildDate", buildinfo.Date, "sha", buildinfo.SHA) + mgr := createManager() + ctx := signals.SetupSignalHandler() + + tkrCompatibility := &compatibility.Compatibility{ + Client: mgr.GetClient(), + Config: compatibilityConfig, + } + registryInstance := registry.New(mgr.GetClient(), registryConfig) + fetcherInstance := &fetcher.Fetcher{ + Log: mgr.GetLogger().WithName("tkr-fetcher"), + Client: mgr.GetClient(), + Config: fetcherConfig, + Registry: registryInstance, + Compatibility: tkrCompatibility, + } + pkgcrReconciler := &pkgcr.Reconciler{ + Log: mgr.GetLogger().WithName("tkr-source"), + Client: mgr.GetClient(), + Config: pkgcrConfig, + Registry: registryInstance, + } + compatibilityReconciler := &compatibility.Reconciler{ + Ctx: ctx, + Log: mgr.GetLogger().WithName("tkr-compatibility"), + Client: mgr.GetClient(), + Config: compatibilityConfig, + Compatibility: tkrCompatibility, + } + + setupWithManager(mgr, []managedComponent{ + registryInstance, + fetcherInstance, + pkgcrReconciler, + compatibilityReconciler, + }) + startManager(ctx, mgr) +} + +func createManager() manager.Manager { // Setup Manager setupLog.Info("setting up manager") mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{ @@ -83,51 +156,32 @@ func main() { setupLog.Error(err, "unable to set up controller manager") os.Exit(1) } + return mgr +} - ctx := signals.SetupSignalHandler() - - if err := mgr.Add(&fetcher.Fetcher{ - Log: mgr.GetLogger().WithName("tkr-fetcher"), - Client: mgr.GetClient(), - Config: fetcher.Config{ - TKRNamespace: tkrNamespace, - BOMImagePath: bomImagePath, - BOMMetadataImagePath: bomMetadataImagePath, - TKRRepoImagePath: tkrRepoImagePath, - VerifyRegistryCert: !skipVerifyRegistryCerts, - TKRDiscoveryOption: fetcher.TKRDiscoveryIntervals{ - InitialDiscoveryFrequency: time.Duration(initTKRDiscoveryFreq) * time.Second, - ContinuousDiscoveryFrequency: time.Duration(continuousTKRDiscoverFreq) * time.Second, - }, - }, - }); err != nil { - setupLog.Error(err, "unable to add fetcher to controller manager") - os.Exit(1) +func setupWithManager(mgr manager.Manager, managedComponents []managedComponent) { + for _, c := range managedComponents { + setupLog.Info("setting up component", "type", fullTypeName(c)) + if err := c.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to setup component", "type", fullTypeName(c)) + os.Exit(1) + } } +} - if err := (&pkgcr.Reconciler{ - Log: mgr.GetLogger().WithName("tkr-source"), - Client: mgr.GetClient(), - Config: pkgcr.Config{ - ServiceAccountName: tkrPkgServiceAccountName, - }, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "TKR Package") - os.Exit(1) +func fullTypeName(c managedComponent) string { + cType := reflect.TypeOf(c) + for cType.Kind() == reflect.Ptr { + cType = cType.Elem() } + return fmt.Sprintf("%s.%s", cType.PkgPath(), cType.Name()) +} - if err := (&compatibility.Reconciler{ - Ctx: ctx, - Log: mgr.GetLogger().WithName("tkr-compatibility"), - Client: mgr.GetClient(), - Config: compatibility.Config{ - TKRNamespace: tkrNamespace, - }, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "TKR Compatibility") - os.Exit(1) - } +type managedComponent interface { + SetupWithManager(ctrl.Manager) error +} +func startManager(ctx context.Context, mgr manager.Manager) { setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "unable to run manager") diff --git a/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler.go b/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler.go index b86022ce29..32bae52e6b 100644 --- a/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler.go +++ b/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler.go @@ -6,22 +6,28 @@ package pkgcr import ( "context" + "fmt" "reflect" + "strings" "github.com/go-logr/logr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/kind/pkg/errors" + "sigs.k8s.io/yaml" - kapppkgiv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" kapppkgv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" - versionsv1 "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/versions/v1alpha1" + "github.com/vmware-tanzu/tanzu-framework/pkg/v1/util/patchset" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/registry" "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/version" ) @@ -30,20 +36,28 @@ type Reconciler struct { Client client.Client Config Config + + Registry registry.Registry } type Config struct { ServiceAccountName string } +type InstallData struct { + ObservedGeneration int64 + Success bool +} + const ( - LabelTKRPackage = "run.tanzu.vmware.com/tkr-package" + LabelTKRPackage = "run.tanzu.vmware.com/tkr-package" + FieldInstallData = "installData" ) func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&kapppkgv1.Package{}, builder.WithPredicates(hasTKRPackageLabelPredicate, predicate.GenerationChangedPredicate{})). - Owns(&kapppkgiv1.PackageInstall{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Owns(&corev1.ConfigMap{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Named("tkr_source"). Complete(r) } @@ -61,30 +75,74 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct if apierrors.IsNotFound(err) { return ctrl.Result{}, nil } - return ctrl.Result{}, err + return ctrl.Result{}, errors.Wrapf(err, "getting Package '%s'", req) } if !pkg.DeletionTimestamp.IsZero() { return ctrl.Result{}, nil } - pkgi := r.packageInstall(pkg) + done, err := r.install(ctx, pkg) + return ctrl.Result{Requeue: !done}, errors.Wrapf(err, "failed to install TKR package '%s'", pkg.Name) +} - r.Log.Info("installing TKR package", "name", pkg.Spec.RefName, "version", pkg.Spec.Version) +var pkgAPIVersion, pkgKind = kapppkgv1.SchemeGroupVersion.WithKind(reflect.TypeOf(kapppkgv1.Package{}).Name()).ToAPIVersionAndKind() +var cmAPIVersion, cmKind = corev1.SchemeGroupVersion.WithKind(reflect.TypeOf(corev1.ConfigMap{}).Name()).ToAPIVersionAndKind() - if err := r.Client.Create(ctx, pkgi); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, errors.Wrap(err, "failed to create PackageInstall") +func (r *Reconciler) install(ctx context.Context, pkg *kapppkgv1.Package) (done bool, err error) { + r.Log.Info("Processing TKR package", "name", pkg.Name) + + cm, err := r.createCM(ctx, pkg) + if cm == nil { + return false, err // err == nil is possible + } + + installData := parseInstallData(cm.Data[FieldInstallData]) + if installData.Success { + r.Log.Info("Already installed TKR package", "name", pkg.Name) + return true, nil } - return ctrl.Result{}, nil + done, err = r.doInstall(ctx, pkg, cm) + if done { + r.Log.Info("Installed TKR package", "name", pkg.Name) + } + return done, err } -var pkgAPIVersion, pkgKind = kapppkgv1.SchemeGroupVersion.WithKind(reflect.TypeOf(kapppkgv1.Package{}).Name()).ToAPIVersionAndKind() +func (r *Reconciler) createCM(ctx context.Context, pkg *kapppkgv1.Package) (*corev1.ConfigMap, error) { + cm0 := cmForPkg(pkg) + if err := r.Client.Create(ctx, cm0); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, errors.Wrapf(err, "creating ConfigMap '%s'", objKey(cm0)) + } + + cm := &corev1.ConfigMap{} + if err := r.Client.Get(ctx, objKey(cm0), cm); err != nil { + return nil, errors.Wrapf(err, "getting ConfigMap '%s'", objKey(cm0)) + } + + installData := parseInstallData(cm.Data[FieldInstallData]) + if installData == nil || installData.ObservedGeneration != pkg.GetGeneration() { + err := r.Client.Delete(ctx, cm) + err = kerrors.FilterOut(err, apierrors.IsNotFound) + return nil, errors.Wrapf(err, "deleting ConfigMap '%s'", objKey(cm)) // err == nil is possible + } + + return cm, nil + } -func (r *Reconciler) packageInstall(pkg *kapppkgv1.Package) *kapppkgiv1.PackageInstall { - return &kapppkgiv1.PackageInstall{ + return cm0, nil +} + +func objKey(o client.Object) client.ObjectKey { + return client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} +} + +func cmForPkg(pkg *kapppkgv1.Package) *corev1.ConfigMap { + return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: "tkr-" + version.Label(pkg.Spec.Version), Namespace: pkg.Namespace, + Name: fmt.Sprintf("tkr-%s", version.Label(pkg.Spec.Version)), OwnerReferences: []metav1.OwnerReference{{ APIVersion: pkgAPIVersion, Kind: pkgKind, @@ -92,15 +150,150 @@ func (r *Reconciler) packageInstall(pkg *kapppkgv1.Package) *kapppkgiv1.PackageI UID: pkg.UID, Controller: pointer.BoolPtr(true), }}, - }, - Spec: kapppkgiv1.PackageInstallSpec{ - ServiceAccountName: r.Config.ServiceAccountName, - PackageRef: &kapppkgiv1.PackageRef{ - RefName: pkg.Spec.RefName, - VersionSelection: &versionsv1.VersionSelectionSemver{ - Constraints: pkg.Spec.Version, - }, + Labels: map[string]string{ + LabelTKRPackage: pkg.Labels[LabelTKRPackage], }, }, + Data: map[string]string{ + FieldInstallData: marshalInstallData(&InstallData{ + ObservedGeneration: pkg.Generation, + }), + }, + } +} + +func parseInstallData(s string) *InstallData { + installData := &InstallData{} + _ = yaml.Unmarshal([]byte(s), installData) + return installData +} + +func marshalInstallData(installData *InstallData) string { + result, _ := yaml.Marshal(installData) + return string(result) +} + +func (r *Reconciler) doInstall(ctx context.Context, pkg *kapppkgv1.Package, cm *corev1.ConfigMap) (done bool, retErr error) { + ps := patchset.New(r.Client) + defer func() { + if err := ps.Apply(ctx); err != nil { + err = kerrors.FilterOut(err, apierrors.IsConflict) + done, retErr = false, errors.Wrap(err, "applying patchset") // err == nil is possible + } + }() + + installData := parseInstallData(cm.Data[FieldInstallData]) + defer func() { + cm.Data[FieldInstallData] = marshalInstallData(installData) + }() + + packageContent, err := r.fetchPackageContent(pkg) + if err != nil { + return false, err + } + + for path, bytes := range packageContent { + if strings.HasPrefix(path, ".") { + continue + } + u, err := parseObject(cm, bytes) + if err != nil { + r.Log.Error(err, "Failed to parse an object from package", "pkg", pkg.Name, "path", path) + continue + } + if err = r.create(ctx, u); err != nil { + r.Log.Error(err, "Failed to create an object from package", "pkg", pkg.Name, "path", path) + return false, err + } + } + + ps.Add(cm) + installData.Success = true + + return true, nil +} + +func parseObject(cm *corev1.ConfigMap, bytes []byte) (*unstructured.Unstructured, error) { + u := &unstructured.Unstructured{} + if err := yaml.Unmarshal(bytes, u); err != nil { + return nil, err + } + u.SetNamespace(cm.Namespace) + addOwnerRefs(u, []metav1.OwnerReference{{ + APIVersion: cmAPIVersion, + Kind: cmKind, + Name: cm.Name, + UID: cm.UID, + }}) + return u, nil +} + +func (r *Reconciler) fetchPackageContent(pkg *kapppkgv1.Package) (map[string][]byte, error) { + if pkg.Spec.Template.Spec == nil { + return nil, nil + } + for _, fetch := range pkg.Spec.Template.Spec.Fetch { + if fetch.ImgpkgBundle == nil { + return nil, nil + } + files, err := r.Registry.GetFiles(fetch.ImgpkgBundle.Image) + if err != nil { + return nil, err + } + return files, nil // nolint:staticcheck // loop is unconditionally terminated: there's only one Fetch + } + return nil, nil +} + +func (r *Reconciler) create(ctx context.Context, u *unstructured.Unstructured) error { + r.Log.Info("Creating object", "GVK", u.GetObjectKind().GroupVersionKind(), "objectKey", objKey(u)) + for { + if err := r.Client.Create(ctx, u); err != nil { + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf(err, "creating object '%s', named '%s'", u.GetObjectKind().GroupVersionKind(), objKey(u)) + } + if err := r.patchExisting(ctx, u); err != nil { + if err := kerrors.FilterOut(err, apierrors.IsConflict, apierrors.IsNotFound); err != nil { + return err // not IsConflict, not IsNotFound + } + r.Log.Info("Re-trying to create object", "GVK", u.GetObjectKind().GroupVersionKind(), "objectKey", objKey(u)) + continue // try to create again + } + } + return nil + } +} + +func (r *Reconciler) patchExisting(ctx context.Context, u *unstructured.Unstructured) error { + existing := &unstructured.Unstructured{} + existing.SetAPIVersion(u.GetAPIVersion()) + existing.SetKind(u.GetKind()) + if err := r.Client.Get(ctx, objKey(u), existing); err != nil { + return errors.Wrapf(err, "getting object '%s', named '%s'", u.GetObjectKind().GroupVersionKind(), objKey(u)) + } + + ps := patchset.New(r.Client) + ps.Add(existing) + + addOwnerRefs(existing, u.GetOwnerReferences()) + + if err := ps.Apply(ctx); err != nil { + return err + } + return nil +} + +func addOwnerRefs(object client.Object, ownerRefs []metav1.OwnerReference) { + switch object.GetObjectKind().GroupVersionKind().Kind { + case "TanzuKubernetesRelease", "OSImage": + return // not adding ownerRef to cluster scoped resources + } + for _, ownerRef := range ownerRefs { + for _, r := range object.GetOwnerReferences() { + if r == ownerRef { + return + } + } + object.SetOwnerReferences(append(object.GetOwnerReferences(), ownerRef)) } } diff --git a/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler_test.go b/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler_test.go index 0f256f92ba..88a1bbdb7f 100644 --- a/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler_test.go +++ b/pkg/v2/tkr/controller/tkr-source/pkgcr/reconciler_test.go @@ -21,9 +21,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - kapppkgiv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/packaging/v1alpha1" + kappctrlv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1" kapppkgv1 "github.com/vmware-tanzu/carvel-kapp-controller/pkg/apiserver/apis/datapackaging/v1alpha1" runv1 "github.com/vmware-tanzu/tanzu-framework/apis/run/v1alpha3" + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/controller/tkr-source/registry" "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/testdata" "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/version" ) @@ -44,6 +45,7 @@ const tkrServiceAccount = "tkr-service-account" var _ = Describe("Reconciler", func() { var ( r *Reconciler + reg registry.Registry objects []client.Object ctx context.Context pkg *kapppkgv1.Package @@ -60,11 +62,14 @@ var _ = Describe("Reconciler", func() { Config: Config{ ServiceAccountName: tkrServiceAccount, }, + Registry: reg, } }) When("a TKR Package hasn't been installed yet", func() { var isTKR int + var tkrName string + var tkrVersion string BeforeEach(func() { pkg = genPkg() @@ -75,6 +80,18 @@ var _ = Describe("Reconciler", func() { } } objects = []client.Object{pkg} + tkrVersion = pkg.Spec.Version + tkrName = version.Label(tkrVersion) + reg = fakeRegistry{ + imageParams: map[string]struct { + tkrVersion string + k8sVersion string + }{ + pkg.Spec.Template.Spec.Fetch[0].ImgpkgBundle.Image: { + tkrVersion: tkrVersion, + k8sVersion: pkg.Spec.Version, + }, + }} }) repeat(100, func() { @@ -84,28 +101,31 @@ var _ = Describe("Reconciler", func() { Expect(err).ToNot(HaveOccurred()) } - pkgi := &kapppkgiv1.PackageInstall{} + cm := &corev1.ConfigMap{} err := r.Client.Get(ctx, client.ObjectKey{Namespace: pkg.Namespace, Name: fmt.Sprintf("tkr-%s", version.Label(pkg.Spec.Version))}, - pkgi) + cm) switch hasTKRPackageLabel(pkg) { case true: Expect(err).ToNot(HaveOccurred()) - Expect(pkgi.Spec.ServiceAccountName).To(Equal(r.Config.ServiceAccountName)) - Expect(pkgi.Spec.PackageRef.RefName).To(Equal(pkg.Spec.RefName)) - Expect(pkgi.Spec.PackageRef.VersionSelection.Constraints).To(Equal(pkg.Spec.Version)) + + tkr := &runv1.TanzuKubernetesRelease{} + Expect(r.Client.Get(ctx, installedObjectName(pkg, tkrName), tkr)).To(Succeed()) + case false: Expect(err).To(HaveOccurred()) Expect(errors.IsNotFound(err)).To(BeTrue()) } - - Expect(pkgi) }) }) }) }) +func installedObjectName(pkg *kapppkgv1.Package, name string) client.ObjectKey { + return client.ObjectKey{Namespace: pkg.Namespace, Name: name} +} + func genPkg() *kapppkgv1.Package { name := rand.String(10) v := fmt.Sprintf("%v.%v.%v", rand.Intn(2), rand.Intn(10), rand.Intn(10)) @@ -117,6 +137,15 @@ func genPkg() *kapppkgv1.Package { Spec: kapppkgv1.PackageSpec{ RefName: name, Version: v, + Template: kapppkgv1.AppTemplateSpec{ + Spec: &kappctrlv1.AppSpec{ + Fetch: []kappctrlv1.AppFetch{{ + ImgpkgBundle: &kappctrlv1.AppFetchImgpkgBundle{ + Image: fmt.Sprintf("example.org/%s", rand.String(13)), + }, + }}, + }, + }, }, } } @@ -125,7 +154,6 @@ func initScheme() *runtime.Scheme { scheme := runtime.NewScheme() utilruntime.Must(runv1.AddToScheme(scheme)) utilruntime.Must(kapppkgv1.AddToScheme(scheme)) - utilruntime.Must(kapppkgiv1.AddToScheme(scheme)) utilruntime.Must(corev1.AddToScheme(scheme)) return scheme } @@ -145,3 +173,58 @@ func (u uidSetter) Create(ctx context.Context, obj client.Object, opts ...client obj.(metav1.Object).SetUID(uuid.NewUUID()) return u.Client.Create(ctx, obj, opts...) } + +type fakeRegistry struct { + registry.Registry + + imageParams map[string]struct { + tkrVersion string + k8sVersion string + } +} + +func (r fakeRegistry) GetFiles(image string) (map[string][]byte, error) { + params := r.imageParams[image] + return map[string][]byte{ + "config/tkr.yaml": []byte(tkrStr(params.tkrVersion, params.k8sVersion)), + "config/garbage.yaml": []byte(garbageStr), + }, nil +} + +func tkrStr(tkrVersion, k8sVersion string) string { + return fmt.Sprintf(` +kind: TanzuKubernetesRelease +apiVersion: run.tanzu.vmware.com/v1alpha3 +metadata: + name: %s +spec: + version: %s + kubernetes: + version: %s + imageRepository: projects.registry.vmware.com/tkg + etcd: + imageTag: v3.5.2_vmware.4 + pause: + imageTag: "3.6" + coredns: + imageTag: v1.8.6_vmware.5 + osImages: + - name: ubuntu-2004-amd64-vmi-k8s-v1.23.5---vmware.1-tkg.1-zshippable + - name: photon-3-amd64-vmi-k8s-v1.23.5---vmware.1-tkg.1-zshippable + bootstrapPackages: + - name: antrea.tanzu.vmware.com.1.2.3+vmware.4-tkg.2-advanced-zshippable + - name: vsphere-pv-csi.tanzu.vmware.com.2.4.0+vmware.1-tkg.1-zshippable + - name: vsphere-cpi.tanzu.vmware.com.1.22.6+vmware.1-tkg.1-zshippable + - name: kapp-controller.tanzu.vmware.com.0.34.0+vmware.1-tkg.1-zshippable + - name: guest-cluster-auth-service.tanzu.vmware.com.1.0.0+tkg.1-zshippable + - name: metrics-server.tanzu.vmware.com.0.5.1+vmware.1-tkg.2-zshippable + - name: secretgen-controller.tanzu.vmware.com.0.8.0+vmware.1-tkg.1-zshippable + - name: pinniped.tanzu.vmware.com.0.12.1+vmware.1-tkg.1-zshippable + - name: capabilities.tanzu.vmware.com.0.22.0-dev-57-gd9465b25+vmware.1 + - name: calico.tanzu.vmware.com.3.22.1+vmware.1-tkg.1-zshippable +`, version.Label(tkrVersion), tkrVersion, k8sVersion) +} + +const garbageStr = ` +(This (not even YAML)) +` diff --git a/pkg/v2/tkr/controller/tkr-source/registry/registry.go b/pkg/v2/tkr/controller/tkr-source/registry/registry.go new file mode 100644 index 0000000000..5f926987e9 --- /dev/null +++ b/pkg/v2/tkr/controller/tkr-source/registry/registry.go @@ -0,0 +1,157 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package registry provides the Registry interface and implementation. +package registry + +import ( + "context" + "os" + "path" + "sync" + + ctlimg "github.com/k14s/imgpkg/pkg/imgpkg/registry" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/tanzu-framework/pkg/v1/tkr/pkg/registry" +) + +const ( + configMapName = "tkr-controller-config" + caCertsKey = "caCerts" + registryCertsFile = "registry_certs" +) + +// Registry defines the registry interface +type Registry interface { + // ListImageTags lists all tags of the given image. + ListImageTags(imageName string) ([]string, error) + // GetFile gets the file content bundled in the given image:tag. + // If filename is empty, it will get the first file. + GetFile(imageWithTag string, filename string) ([]byte, error) + // GetFiles get all the files content bundled in the given image:tag. + GetFiles(imageWithTag string) (map[string][]byte, error) + // DownloadBundle downloads OCI bundle similar to `imgpkg pull -b` command + // It is recommended to use this function when downloading imgpkg bundle + DownloadBundle(imageName, outputDir string) error +} + +type impl struct { + Registry + + Client client.Client + Config Config + + sync.Once + initDone chan struct{} +} + +var _ Registry = (*impl)(nil) + +// New returns a new Registry instance. +func New(c client.Client, config Config) *impl { // nolint:revive // unexported-return: *impl implements a public interface + return &impl{ + Client: c, + Config: config, + initDone: make(chan struct{}), + } +} + +// Config contains the controller manager context. +type Config struct { + TKRNamespace string + VerifyRegistryCert bool +} + +func (r *impl) SetupWithManager(m ctrl.Manager) error { + return m.Add(r) +} + +func (r *impl) Start(ctx context.Context) error { + var err error + r.Do(func() { + err = r.configure(ctx) + if err == nil { + close(r.initDone) + } + }) + return err +} + +func (r *impl) configure(ctx context.Context) error { + configMap := &corev1.ConfigMap{} + if err := r.Client.Get(ctx, + types.NamespacedName{Namespace: r.Config.TKRNamespace, Name: configMapName}, + configMap); !k8serr.IsNotFound(err) { + return errors.Wrapf(err, "unable to get the ConfigMap %s", configMapName) + } + + err := addTrustedCerts(configMap.Data[caCertsKey]) + if err != nil { + return errors.Wrap(err, "failed to add certs") + } + + registryOps := &ctlimg.Opts{ + VerifyCerts: r.Config.VerifyRegistryCert, + Anon: true, + } + + // Add custom CA cert paths only if VerifyCerts is enabled + if registryOps.VerifyCerts { + registryCertPath, err := getRegistryCertFile() + if err == nil { + if _, err = os.Stat(registryCertPath); err == nil { + registryOps.CACertPaths = []string{registryCertPath} + } + } + } + + r.Registry, err = registry.New(registryOps) + return err +} + +func addTrustedCerts(certChain string) (err error) { + if certChain == "" { + return nil + } + + filePath, err := getRegistryCertFile() + if err != nil { + return err + } + + return os.WriteFile(filePath, []byte(certChain), 0644) +} + +func getRegistryCertFile() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", errors.Wrap(err, "could not locate local tanzu dir") + } + return path.Join(home, registryCertsFile), nil +} + +func (r *impl) ListImageTags(imageName string) ([]string, error) { + <-r.initDone + return r.Registry.ListImageTags(imageName) +} + +func (r *impl) GetFile(imageWithTag, filename string) ([]byte, error) { + <-r.initDone + return r.Registry.GetFile(imageWithTag, filename) +} + +func (r *impl) GetFiles(imageWithTag string) (map[string][]byte, error) { + <-r.initDone + return r.Registry.GetFiles(imageWithTag) +} + +func (r *impl) DownloadBundle(imageName, outputDir string) error { + <-r.initDone + return r.Registry.DownloadBundle(imageName, outputDir) +} diff --git a/pkg/v2/tkr/util/sets/strings.go b/pkg/v2/tkr/util/sets/strings.go new file mode 100644 index 0000000000..3c4aafc676 --- /dev/null +++ b/pkg/v2/tkr/util/sets/strings.go @@ -0,0 +1,55 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package sets provides sets of different types, e.g. StringSet. +package sets + +type StringSet map[string]struct{} + +func Strings(ss ...string) StringSet { + r := make(StringSet, len(ss)) + return r.Add(ss...) +} + +func (set StringSet) Add(ss ...string) StringSet { + for _, s := range ss { + set[s] = struct{}{} + } + return set +} + +func (set StringSet) Remove(ss ...string) StringSet { + for _, s := range ss { + delete(set, s) + } + return set +} + +func (set StringSet) Has(s string) bool { + _, has := set[s] + return has +} + +func (set StringSet) Intersect(other StringSet) StringSet { + return set.Filter(func(s string) bool { + return other.Has(s) + }) +} + +func (set StringSet) Map(f func(s string) string) StringSet { + r := make(StringSet, len(set)) + for s := range set { + r[f(s)] = struct{}{} + } + return r +} + +func (set StringSet) Filter(f func(s string) bool) StringSet { + r := make(StringSet, len(set)) + for s := range set { + if f(s) { + r[s] = struct{}{} + } + } + return r +} diff --git a/pkg/v2/tkr/util/version/compatibility.go b/pkg/v2/tkr/util/version/compatibility.go new file mode 100644 index 0000000000..31744e7b01 --- /dev/null +++ b/pkg/v2/tkr/util/version/compatibility.go @@ -0,0 +1,14 @@ +// Copyright 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package version + +import ( + "context" + + "github.com/vmware-tanzu/tanzu-framework/pkg/v2/tkr/util/sets" +) + +type Compatibility interface { + CompatibleVersions(ctx context.Context) (sets.StringSet, error) +}