From 90d8ae707c3b56353c175af1dc02665a9a16cdda Mon Sep 17 00:00:00 2001
From: jay vyas <jvyas@vmware.com>
Date: Tue, 22 Nov 2022 14:44:29 +0100
Subject: [PATCH 1/6] [DO NOT MERGE] WIP:redo of BUILDKIT caching PR

---
 addons/Dockerfile                             | 13 +++++---
 .../manifests/samples/simple-app/Dockerfile   |  3 ++
 capabilities/controller/Dockerfile            |  9 ++++--
 common.mk                                     |  5 ++++
 featuregates/controller/Dockerfile            | 12 ++++++--
 object-propagation/Dockerfile                 | 14 +++++++--
 pinniped-components/post-deploy/Dockerfile    | 11 +++++--
 .../tanzu-auth-controller-manager/Dockerfile  | 11 +++++--
 pkg/v1/tkr/Dockerfile                         |  8 +++--
 tkg/clusterclient/poller.go                   | 30 +++++++++++++++----
 tkg/vsphere-template-resolver/Dockerfile      |  9 ++++--
 tkr/controller/tkr-source/Dockerfile          |  5 ++++
 tkr/controller/tkr-status/Dockerfile          |  6 ++++
 tkr/webhook/cluster/tkr-resolver/Dockerfile   | 11 +++++--
 tkr/webhook/tkr-conversion/Dockerfile         | 10 +++++--
 15 files changed, 126 insertions(+), 31 deletions(-)

diff --git a/addons/Dockerfile b/addons/Dockerfile
index 2f9dbe7c0d..359685dde3 100644
--- a/addons/Dockerfile
+++ b/addons/Dockerfile
@@ -1,26 +1,31 @@
 # Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
-
 WORKDIR /workspace
-
 # Copy the go source
 COPY ./ ./
 
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN go mod download
+RUN  --mount=$CACHE_GOLANG --mount=$CACHE_GOMOD go mod download
 
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN cd addons && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o bin/manager main.go
+RUN $CACHE cd addons && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -ldflags "$LD_FLAGS" -o bin/manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/addons/manifests/samples/simple-app/Dockerfile b/addons/manifests/samples/simple-app/Dockerfile
index d847571323..9202de6e93 100644
--- a/addons/manifests/samples/simple-app/Dockerfile
+++ b/addons/manifests/samples/simple-app/Dockerfile
@@ -1,5 +1,8 @@
 # Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+
 FROM scratch
 COPY templates/* ./
diff --git a/capabilities/controller/Dockerfile b/capabilities/controller/Dockerfile
index bc26740015..21b33aee86 100644
--- a/capabilities/controller/Dockerfile
+++ b/capabilities/controller/Dockerfile
@@ -4,7 +4,10 @@
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
-
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
 
@@ -17,12 +20,12 @@ COPY apis/ apis/
 COPY cli/runtime cli/runtime
 
 WORKDIR capabilities/controller
-RUN  go mod download
+RUN $CACHE go mod download
 
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/common.mk b/common.mk
index 8c414af8cc..9534b4d694 100644
--- a/common.mk
+++ b/common.mk
@@ -10,6 +10,11 @@ ROOT_DIR := $(shell git rev-parse --show-toplevel)
 RELATIVE_ROOT ?= .
 CONTROLLER_GEN_SRC ?= "./..."
 
+# Framework has lots of components, and many build steps that are disparate.
+# Use docker buildkit so that we get faster image builds and skip redundant work.
+# TODO: We need to measure what this speeds up, and what it doesn't (and why).
+export DOCKER_BUILDKIT := 1
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
diff --git a/featuregates/controller/Dockerfile b/featuregates/controller/Dockerfile
index ae4d04dd4d..ac762d4362 100644
--- a/featuregates/controller/Dockerfile
+++ b/featuregates/controller/Dockerfile
@@ -1,5 +1,11 @@
-# Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# Enable Buildkit
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
+
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -19,7 +25,7 @@ COPY cli/runtime cli/runtime
 COPY capabilities/client capabilities/client
 
 WORKDIR featuregates/controller
-RUN  go mod download
+RUN $CACHE go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -31,7 +37,7 @@ COPY ./featuregates/controller/main.go ./main.go
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/object-propagation/Dockerfile b/object-propagation/Dockerfile
index fb52dd0995..5b19311883 100644
--- a/object-propagation/Dockerfile
+++ b/object-propagation/Dockerfile
@@ -1,6 +1,14 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}"
+
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
@@ -18,7 +26,7 @@ COPY object-propagation/go.mod object-propagation/go.mod
 COPY object-propagation/go.sum object-propagation/go.sum
 
 WORKDIR /workspace/object-propagation
-RUN  go mod download
+RUN $CACHE go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +36,7 @@ COPY object-propagation/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager main.go
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -37,4 +45,4 @@ WORKDIR /
 COPY --from=builder /workspace/object-propagation/manager .
 USER nonroot:nonroot
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/manager"]
\ No newline at end of file
diff --git a/pinniped-components/post-deploy/Dockerfile b/pinniped-components/post-deploy/Dockerfile
index 19372f9e2e..d065e86d04 100644
--- a/pinniped-components/post-deploy/Dockerfile
+++ b/pinniped-components/post-deploy/Dockerfile
@@ -5,6 +5,11 @@
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
 
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" 
+
 # Build the post-deploy binary
 FROM $BUILDER_BASE_IMAGE as builder
 
@@ -14,7 +19,7 @@ WORKDIR /workspace
 COPY tanzu-auth-controller-manager tanzu-auth-controller-manager
 COPY post-deploy/go.mod post-deploy/go.mod
 COPY post-deploy/go.sum post-deploy/go.sum
-RUN cd post-deploy && go mod download
+RUN $CACHE cd post-deploy && go mod download
 
 # Copy the source
 COPY post-deploy/cmd/ post-deploy/cmd/
@@ -22,7 +27,7 @@ COPY post-deploy/pkg/ post-deploy/pkg/
 COPY post-deploy/Makefile post-deploy/Makefile
 #COPY .git/ .git/
 
-RUN make native -C post-deploy
+RUN $CACHE make native -C post-deploy
 
 # Support older deployment YAMLs by providing symlink for the historic job name without -job.
 # The subdirectory here is to ensure that the docker COPY command does not follow the symlink
@@ -36,4 +41,4 @@ WORKDIR /
 COPY --from=builder /tmp/links/ .
 COPY --from=builder /workspace/post-deploy/tkg-pinniped-post-deploy-job .
 COPY --from=builder /workspace/post-deploy/tkg-pinniped-post-deploy-controller .
-CMD ["/tkg-pinniped-post-deploy-job"]
+CMD ["/tkg-pinniped-post-deploy-job"]
\ No newline at end of file
diff --git a/pinniped-components/tanzu-auth-controller-manager/Dockerfile b/pinniped-components/tanzu-auth-controller-manager/Dockerfile
index 7a0541ee91..427b106cd4 100644
--- a/pinniped-components/tanzu-auth-controller-manager/Dockerfile
+++ b/pinniped-components/tanzu-auth-controller-manager/Dockerfile
@@ -1,6 +1,13 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" 
+ 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
@@ -12,7 +19,7 @@ WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
-RUN go mod download
+RUN  --mount=$CACHE_GOLANG --mount=$CACHE_GOMOD go mod download
 
 # Copy the source
 COPY main.go main.go
@@ -21,7 +28,7 @@ COPY controllers/ controllers/
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o tanzu-auth-controller-manager .
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o tanzu-auth-controller-manager .
 
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
diff --git a/pkg/v1/tkr/Dockerfile b/pkg/v1/tkr/Dockerfile
index 14bc0db977..994b0d319f 100644
--- a/pkg/v1/tkr/Dockerfile
+++ b/pkg/v1/tkr/Dockerfile
@@ -1,5 +1,9 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -18,7 +22,7 @@ COPY pkg/v1/tkr/go.mod pkg/v1/tkr/go.mod
 COPY pkg/v1/tkr/go.sum pkg/v1/tkr/go.sum
 
 WORKDIR /workspace/pkg/v1/tkr
-RUN  go mod download
+RUN $CACHE go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +32,7 @@ COPY pkg/v1/tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkg/clusterclient/poller.go b/tkg/clusterclient/poller.go
index 7e7a4e2d0f..59b453348f 100644
--- a/tkg/clusterclient/poller.go
+++ b/tkg/clusterclient/poller.go
@@ -95,11 +95,31 @@ func (p *pollerProxy) PollImmediateInfiniteWithGetter(interval time.Duration, ge
 		return true, nil
 	}
 
-	errPoll := p.PollImmediateInfinite(interval, pollerFunc)
-	if errPoll != nil {
-		// note: this function will return actual error which is thrown by getterFunc
-		// and will not return error of the PollImmediateInfinite call
-		return err
+	timeoutReached := false
+	ch := time.After(interval)
+	// dont return prematurely.  workaround to the fact that
+	// apimachinery will prematurely return if any errors, but
+	// we want to be forgiving of race conditions for looking up the vsphere secrets, for example.
+	go func() {
+		<- ch 
+		timeoutReached = true
 	}
+	for timeoutReached {
+		// note: APIMachinery will return immediately on failure, however, we don't want
+		// to always return immediately.
+		errPoll := p.PollImmediateInfinite(interval, pollerFunc)
+		if errPoll != nil {
+			// note: this function will return actual error which is thrown by getterFunc
+			// and will not return error of the PollImmediateInfinite call.
+			// note: caller shouldnt necessarily terminate on errors...
+			log.Info(errPoll.Error() + "underlying error:" + err.Error())
+			// not returning error yet...
+		} else {
+			log.V(6).Info("no errors, but not ready yet, retrying")
+		}
+
+
+	}
+
 	return nil
 }
diff --git a/tkg/vsphere-template-resolver/Dockerfile b/tkg/vsphere-template-resolver/Dockerfile
index f7752ab1b9..f72033de9a 100644
--- a/tkg/vsphere-template-resolver/Dockerfile
+++ b/tkg/vsphere-template-resolver/Dockerfile
@@ -1,9 +1,14 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
@@ -26,7 +31,7 @@ COPY tkg/go.mod tkg/go.mod
 COPY tkg/go.sum tkg/go.sum
 
 WORKDIR /workspace/tkg
-RUN  go mod download
+RUN $CACHE go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -36,7 +41,7 @@ COPY tkg/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager vsphere-template-resolver/main.go
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager vsphere-template-resolver/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/controller/tkr-source/Dockerfile b/tkr/controller/tkr-source/Dockerfile
index 5737ee4b95..0755aedb38 100644
--- a/tkr/controller/tkr-source/Dockerfile
+++ b/tkr/controller/tkr-source/Dockerfile
@@ -1,9 +1,14 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
diff --git a/tkr/controller/tkr-status/Dockerfile b/tkr/controller/tkr-status/Dockerfile
index 99b3ff30c2..3040bd48fa 100644
--- a/tkr/controller/tkr-status/Dockerfile
+++ b/tkr/controller/tkr-status/Dockerfile
@@ -1,5 +1,11 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4
+
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
diff --git a/tkr/webhook/cluster/tkr-resolver/Dockerfile b/tkr/webhook/cluster/tkr-resolver/Dockerfile
index d98680b5d1..6b947029d9 100644
--- a/tkr/webhook/cluster/tkr-resolver/Dockerfile
+++ b/tkr/webhook/cluster/tkr-resolver/Dockerfile
@@ -1,10 +1,17 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
 
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
+
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
 
@@ -18,7 +25,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN  go mod download
+RUN $CACHE go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +35,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/cluster/tkr-resolver/main.go
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/cluster/tkr-resolver/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/webhook/tkr-conversion/Dockerfile b/tkr/webhook/tkr-conversion/Dockerfile
index 679429828f..7f2357786d 100644
--- a/tkr/webhook/tkr-conversion/Dockerfile
+++ b/tkr/webhook/tkr-conversion/Dockerfile
@@ -1,5 +1,11 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4
+
+ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
+ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
+ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -18,7 +24,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN  go mod download
+RUN $CACHE go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +34,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
+RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

From b90d0846896098fa3357a276207f9a60e1b3f9f1 Mon Sep 17 00:00:00 2001
From: jay vyas <jvyas@vmware.com>
Date: Tue, 22 Nov 2022 16:17:06 +0100
Subject: [PATCH 2/6] WIP: seeing if i can get tkr-source to compile

---
 tkr/controller/tkr-source/Dockerfile        | 16 ++++++++++------
 tkr/controller/tkr-status/Dockerfile        |  4 ++--
 tkr/webhook/cluster/tkr-resolver/Dockerfile |  2 +-
 tkr/webhook/tkr-conversion/Dockerfile       |  8 +++++---
 4 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/tkr/controller/tkr-source/Dockerfile b/tkr/controller/tkr-source/Dockerfile
index 0755aedb38..965ec5d1b3 100644
--- a/tkr/controller/tkr-source/Dockerfile
+++ b/tkr/controller/tkr-source/Dockerfile
@@ -5,10 +5,6 @@
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
@@ -22,8 +18,15 @@ COPY util/ util/
 COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
+ENV CACHE_GOBUILD="type=cache,target=/root/.cache/go-build"
+ENV CACHE_GOLANG="cache,target=/root/.local/share/golang"
+ENV CACHE_GOMOD="type=cache,target=/go/pkg/mod"
+
+ENV CACHE="--mount=${CACHE_GOLANG}"
+#" --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}"
+
 WORKDIR /workspace/tkr
-RUN  go mod download
+RUN $CACHE go mod download 
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -33,7 +36,8 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-source/main.go
+RUN $CACHE \
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-source/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/controller/tkr-status/Dockerfile b/tkr/controller/tkr-status/Dockerfile
index 3040bd48fa..f3b2c09bfa 100644
--- a/tkr/controller/tkr-status/Dockerfile
+++ b/tkr/controller/tkr-status/Dockerfile
@@ -24,7 +24,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN  go mod download
+RUN \${CACHE} go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -34,7 +34,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-status/main.go
+RUN \${CACHE}  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-status/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/webhook/cluster/tkr-resolver/Dockerfile b/tkr/webhook/cluster/tkr-resolver/Dockerfile
index 6b947029d9..967161ac01 100644
--- a/tkr/webhook/cluster/tkr-resolver/Dockerfile
+++ b/tkr/webhook/cluster/tkr-resolver/Dockerfile
@@ -25,7 +25,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN $CACHE go mod download
+RUN \${CACHE} go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
diff --git a/tkr/webhook/tkr-conversion/Dockerfile b/tkr/webhook/tkr-conversion/Dockerfile
index 7f2357786d..5c1f3c6ed4 100644
--- a/tkr/webhook/tkr-conversion/Dockerfile
+++ b/tkr/webhook/tkr-conversion/Dockerfile
@@ -5,7 +5,9 @@
 ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
 ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
 ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
+ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}"
+
+# Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -24,7 +26,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN $CACHE go mod download
+RUN \$CACHE go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -34,7 +36,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
+RUN ${CACHE} CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

From c66db041e2e0e97deba8326b0b1a29845d1e37c1 Mon Sep 17 00:00:00 2001
From: jay vyas <jvyas@vmware.com>
Date: Tue, 22 Nov 2022 22:15:52 +0100
Subject: [PATCH 3/6] updated caches to use the old way

---
 addons/Dockerfile                             |  4 ++--
 capabilities/controller/Dockerfile            |  4 ++--
 featuregates/controller/Dockerfile            |  4 ++--
 object-propagation/Dockerfile                 |  4 ++--
 pinniped-components/post-deploy/Dockerfile    |  4 ++--
 .../tanzu-auth-controller-manager/Dockerfile  |  4 ++--
 pkg/v1/tkr/Dockerfile                         |  4 ++--
 tkg/vsphere-template-resolver/Dockerfile      |  4 ++--
 tkr/controller/tkr-source/Dockerfile          | 19 ++++++++++---------
 tkr/controller/tkr-status/Dockerfile          |  4 ++--
 tkr/webhook/cluster/tkr-resolver/Dockerfile   |  4 ++--
 tkr/webhook/tkr-conversion/Dockerfile         |  4 ++--
 12 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/addons/Dockerfile b/addons/Dockerfile
index 359685dde3..5d1153b78d 100644
--- a/addons/Dockerfile
+++ b/addons/Dockerfile
@@ -20,12 +20,12 @@ COPY ./ ./
 
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN  --mount=$CACHE_GOLANG --mount=$CACHE_GOMOD go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE cd addons && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -ldflags "$LD_FLAGS" -o bin/manager main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod cd addons && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -ldflags "$LD_FLAGS" -o bin/manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/capabilities/controller/Dockerfile b/capabilities/controller/Dockerfile
index 21b33aee86..726db364a3 100644
--- a/capabilities/controller/Dockerfile
+++ b/capabilities/controller/Dockerfile
@@ -20,12 +20,12 @@ COPY apis/ apis/
 COPY cli/runtime cli/runtime
 
 WORKDIR capabilities/controller
-RUN $CACHE go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/featuregates/controller/Dockerfile b/featuregates/controller/Dockerfile
index ac762d4362..62839ea36f 100644
--- a/featuregates/controller/Dockerfile
+++ b/featuregates/controller/Dockerfile
@@ -25,7 +25,7 @@ COPY cli/runtime cli/runtime
 COPY capabilities/client capabilities/client
 
 WORKDIR featuregates/controller
-RUN $CACHE go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -37,7 +37,7 @@ COPY ./featuregates/controller/main.go ./main.go
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
 
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/object-propagation/Dockerfile b/object-propagation/Dockerfile
index 5b19311883..2ca9f79ad3 100644
--- a/object-propagation/Dockerfile
+++ b/object-propagation/Dockerfile
@@ -26,7 +26,7 @@ COPY object-propagation/go.mod object-propagation/go.mod
 COPY object-propagation/go.sum object-propagation/go.sum
 
 WORKDIR /workspace/object-propagation
-RUN $CACHE go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -36,7 +36,7 @@ COPY object-propagation/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/pinniped-components/post-deploy/Dockerfile b/pinniped-components/post-deploy/Dockerfile
index d065e86d04..210750fb10 100644
--- a/pinniped-components/post-deploy/Dockerfile
+++ b/pinniped-components/post-deploy/Dockerfile
@@ -19,7 +19,7 @@ WORKDIR /workspace
 COPY tanzu-auth-controller-manager tanzu-auth-controller-manager
 COPY post-deploy/go.mod post-deploy/go.mod
 COPY post-deploy/go.sum post-deploy/go.sum
-RUN $CACHE cd post-deploy && go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod cd post-deploy && go mod download
 
 # Copy the source
 COPY post-deploy/cmd/ post-deploy/cmd/
@@ -27,7 +27,7 @@ COPY post-deploy/pkg/ post-deploy/pkg/
 COPY post-deploy/Makefile post-deploy/Makefile
 #COPY .git/ .git/
 
-RUN $CACHE make native -C post-deploy
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod make native -C post-deploy
 
 # Support older deployment YAMLs by providing symlink for the historic job name without -job.
 # The subdirectory here is to ensure that the docker COPY command does not follow the symlink
diff --git a/pinniped-components/tanzu-auth-controller-manager/Dockerfile b/pinniped-components/tanzu-auth-controller-manager/Dockerfile
index 427b106cd4..8439f77ba1 100644
--- a/pinniped-components/tanzu-auth-controller-manager/Dockerfile
+++ b/pinniped-components/tanzu-auth-controller-manager/Dockerfile
@@ -19,7 +19,7 @@ WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
-RUN  --mount=$CACHE_GOLANG --mount=$CACHE_GOMOD go mod download
+RUN  --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 
 # Copy the source
 COPY main.go main.go
@@ -28,7 +28,7 @@ COPY controllers/ controllers/
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o tanzu-auth-controller-manager .
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o tanzu-auth-controller-manager .
 
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
diff --git a/pkg/v1/tkr/Dockerfile b/pkg/v1/tkr/Dockerfile
index 994b0d319f..f851cd9c59 100644
--- a/pkg/v1/tkr/Dockerfile
+++ b/pkg/v1/tkr/Dockerfile
@@ -22,7 +22,7 @@ COPY pkg/v1/tkr/go.mod pkg/v1/tkr/go.mod
 COPY pkg/v1/tkr/go.sum pkg/v1/tkr/go.sum
 
 WORKDIR /workspace/pkg/v1/tkr
-RUN $CACHE go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -32,7 +32,7 @@ COPY pkg/v1/tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkg/vsphere-template-resolver/Dockerfile b/tkg/vsphere-template-resolver/Dockerfile
index f72033de9a..4caa74926a 100644
--- a/tkg/vsphere-template-resolver/Dockerfile
+++ b/tkg/vsphere-template-resolver/Dockerfile
@@ -31,7 +31,7 @@ COPY tkg/go.mod tkg/go.mod
 COPY tkg/go.sum tkg/go.sum
 
 WORKDIR /workspace/tkg
-RUN $CACHE go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -41,7 +41,7 @@ COPY tkg/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager vsphere-template-resolver/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager vsphere-template-resolver/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/controller/tkr-source/Dockerfile b/tkr/controller/tkr-source/Dockerfile
index 965ec5d1b3..fc236994d9 100644
--- a/tkr/controller/tkr-source/Dockerfile
+++ b/tkr/controller/tkr-source/Dockerfile
@@ -18,15 +18,15 @@ COPY util/ util/
 COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
-ENV CACHE_GOBUILD="type=cache,target=/root/.cache/go-build"
-ENV CACHE_GOLANG="cache,target=/root/.local/share/golang"
-ENV CACHE_GOMOD="type=cache,target=/go/pkg/mod"
-
-ENV CACHE="--mount=${CACHE_GOLANG}"
-#" --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}"
+#ENV CACHE_GOBUILD='--mount="type=cache,target=/root/.cache/go-build"'
+#ENV CACHE_GOLANG='--mount="type=cache,target=/root/.local/share/golang"'
+#ENV CACHE_GOMOD='--mount="type=cache,target=/go/pkg/mod"'
 
 WORKDIR /workspace/tkr
-RUN $CACHE go mod download 
+
+########### RUN --mount="type=cache,target=/root/.local/share/golang" go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -36,8 +36,9 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE \
-    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-source/main.go
+
+# RUN --mount=type=cache,target=/root/.cache/go-build \
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-source/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/controller/tkr-status/Dockerfile b/tkr/controller/tkr-status/Dockerfile
index f3b2c09bfa..740d9587b1 100644
--- a/tkr/controller/tkr-status/Dockerfile
+++ b/tkr/controller/tkr-status/Dockerfile
@@ -24,7 +24,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN \${CACHE} go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -34,7 +34,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN \${CACHE}  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-status/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-status/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/webhook/cluster/tkr-resolver/Dockerfile b/tkr/webhook/cluster/tkr-resolver/Dockerfile
index 967161ac01..2733720bfe 100644
--- a/tkr/webhook/cluster/tkr-resolver/Dockerfile
+++ b/tkr/webhook/cluster/tkr-resolver/Dockerfile
@@ -25,7 +25,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN \${CACHE} go mod download
+RUN aaa\${CACHE} go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -35,7 +35,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN $CACHE CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/cluster/tkr-resolver/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/cluster/tkr-resolver/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
diff --git a/tkr/webhook/tkr-conversion/Dockerfile b/tkr/webhook/tkr-conversion/Dockerfile
index 5c1f3c6ed4..4840143fb2 100644
--- a/tkr/webhook/tkr-conversion/Dockerfile
+++ b/tkr/webhook/tkr-conversion/Dockerfile
@@ -26,7 +26,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN \$CACHE go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -36,7 +36,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN ${CACHE} CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

From 8e0f47fe14e1fbbc466ca982844df986bc5fc1db Mon Sep 17 00:00:00 2001
From: jay vyas <jvyas@vmware.com>
Date: Wed, 23 Nov 2022 08:46:17 +0100
Subject: [PATCH 4/6] Cleanup and remove dbeugging

---
 addons/Dockerfile                                     |  3 ---
 capabilities/controller/Dockerfile                    |  5 +----
 featuregates/controller/Dockerfile                    |  4 ----
 object-propagation/Dockerfile                         |  5 -----
 pinniped-components/post-deploy/Dockerfile            |  7 ++-----
 .../tanzu-auth-controller-manager/Dockerfile          |  6 +-----
 pkg/v1/tkr/Dockerfile                                 |  5 +----
 tkg/vsphere-template-resolver/Dockerfile              |  4 ----
 tkr/controller/tkr-source/Dockerfile                  | 11 +++++++----
 tkr/controller/tkr-status/Dockerfile                  |  7 +------
 tkr/webhook/cluster/tkr-resolver/Dockerfile           |  7 +------
 tkr/webhook/tkr-conversion/Dockerfile                 |  5 -----
 12 files changed, 14 insertions(+), 55 deletions(-)

diff --git a/addons/Dockerfile b/addons/Dockerfile
index 5d1153b78d..a7a76cb5ec 100644
--- a/addons/Dockerfile
+++ b/addons/Dockerfile
@@ -7,9 +7,6 @@
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
diff --git a/capabilities/controller/Dockerfile b/capabilities/controller/Dockerfile
index 726db364a3..f22cfa7803 100644
--- a/capabilities/controller/Dockerfile
+++ b/capabilities/controller/Dockerfile
@@ -4,10 +4,7 @@
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" 
+
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
 
diff --git a/featuregates/controller/Dockerfile b/featuregates/controller/Dockerfile
index 62839ea36f..e1a178dab3 100644
--- a/featuregates/controller/Dockerfile
+++ b/featuregates/controller/Dockerfile
@@ -1,9 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
 # Enable Buildkit
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # syntax=docker/dockerfile:1.4
 
diff --git a/object-propagation/Dockerfile b/object-propagation/Dockerfile
index 2ca9f79ad3..2a4c02e6c4 100644
--- a/object-propagation/Dockerfile
+++ b/object-propagation/Dockerfile
@@ -4,11 +4,6 @@
 # Enable Buildkit
 # syntax=docker/dockerfile:1.4
 
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}"
-
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
diff --git a/pinniped-components/post-deploy/Dockerfile b/pinniped-components/post-deploy/Dockerfile
index 210750fb10..8b0508f413 100644
--- a/pinniped-components/post-deploy/Dockerfile
+++ b/pinniped-components/post-deploy/Dockerfile
@@ -1,15 +1,11 @@
 # Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker.io/docker/dockerfile:1.3.0
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
 
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" 
-
 # Build the post-deploy binary
 FROM $BUILDER_BASE_IMAGE as builder
 
@@ -19,6 +15,7 @@ WORKDIR /workspace
 COPY tanzu-auth-controller-manager tanzu-auth-controller-manager
 COPY post-deploy/go.mod post-deploy/go.mod
 COPY post-deploy/go.sum post-deploy/go.sum
+
 RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod cd post-deploy && go mod download
 
 # Copy the source
diff --git a/pinniped-components/tanzu-auth-controller-manager/Dockerfile b/pinniped-components/tanzu-auth-controller-manager/Dockerfile
index 8439f77ba1..84a2bbde09 100644
--- a/pinniped-components/tanzu-auth-controller-manager/Dockerfile
+++ b/pinniped-components/tanzu-auth-controller-manager/Dockerfile
@@ -3,11 +3,7 @@
 
 # Enable Buildkit
 # syntax=docker/dockerfile:1.4
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" 
- 
+
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
diff --git a/pkg/v1/tkr/Dockerfile b/pkg/v1/tkr/Dockerfile
index f851cd9c59..7d83033695 100644
--- a/pkg/v1/tkr/Dockerfile
+++ b/pkg/v1/tkr/Dockerfile
@@ -1,9 +1,6 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
+# syntax=docker.io/docker/dockerfile:1.3.0
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
diff --git a/tkg/vsphere-template-resolver/Dockerfile b/tkg/vsphere-template-resolver/Dockerfile
index 4caa74926a..b7794ae350 100644
--- a/tkg/vsphere-template-resolver/Dockerfile
+++ b/tkg/vsphere-template-resolver/Dockerfile
@@ -5,10 +5,6 @@
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
diff --git a/tkr/controller/tkr-source/Dockerfile b/tkr/controller/tkr-source/Dockerfile
index fc236994d9..2d61c2748b 100644
--- a/tkr/controller/tkr-source/Dockerfile
+++ b/tkr/controller/tkr-source/Dockerfile
@@ -18,13 +18,16 @@ COPY util/ util/
 COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
-#ENV CACHE_GOBUILD='--mount="type=cache,target=/root/.cache/go-build"'
-#ENV CACHE_GOLANG='--mount="type=cache,target=/root/.local/share/golang"'
-#ENV CACHE_GOMOD='--mount="type=cache,target=/go/pkg/mod"'
+# TODO: @jayunit100 ~ I Tried this but it didnt work, filed an upstream issue https://github.com/moby/buildkit/issues/3301 
+# ENV CACHE_GOBUILD='--mount="type=cache,target=/root/.cache/go-build"'
+# ENV CACHE_GOLANG='--mount="type=cache,target=/root/.local/share/golang"'
+# ENV CACHE_GOMOD='--mount="type=cache,target=/go/pkg/mod"'
+# ENV CACHE="$CACHE_GOMOD $CACHE_GOLANG $CACHE_GOBUILD"
+# RUN $CACHE go mod download
+# Since this failed, we copy paste 3 --mount commands explicitly on line 31 below.  Someday woudl be nice to generalize all of that into a interpolated call.
 
 WORKDIR /workspace/tkr
 
-########### RUN --mount="type=cache,target=/root/.local/share/golang" go mod download
 RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 
 # cache deps before building and copying source so that we don't need to re-download as much
diff --git a/tkr/controller/tkr-status/Dockerfile b/tkr/controller/tkr-status/Dockerfile
index 740d9587b1..a7fb2376a5 100644
--- a/tkr/controller/tkr-status/Dockerfile
+++ b/tkr/controller/tkr-status/Dockerfile
@@ -1,11 +1,6 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
-# syntax=docker/dockerfile:1.4
-
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
+# syntax=docker/dockerfile:1.4 
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
diff --git a/tkr/webhook/cluster/tkr-resolver/Dockerfile b/tkr/webhook/cluster/tkr-resolver/Dockerfile
index 2733720bfe..f5974a980b 100644
--- a/tkr/webhook/cluster/tkr-resolver/Dockerfile
+++ b/tkr/webhook/cluster/tkr-resolver/Dockerfile
@@ -7,11 +7,6 @@
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
 
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}" # Copyright 2021 VMware, Inc. All Rights Reserved.
-
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
 
@@ -25,7 +20,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN aaa\${CACHE} go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
diff --git a/tkr/webhook/tkr-conversion/Dockerfile b/tkr/webhook/tkr-conversion/Dockerfile
index 4840143fb2..e527dad410 100644
--- a/tkr/webhook/tkr-conversion/Dockerfile
+++ b/tkr/webhook/tkr-conversion/Dockerfile
@@ -2,11 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # syntax=docker/dockerfile:1.4
 
-ARG CACHE_GOBUILD=type=cache,target=/root/.cache/go-build
-ARG CACHE_GOLANG=cache,target=/root/.local/share/golang
-ARG CACHE_GOMOD=type=cache,target=/go/pkg/mod
-ARG CACHE="--mount=${CACHE_GOLANG} --mount=${CACHE_GOBUILD} --mount=${CACHE_GOMOD}"
-
 # Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build from publicly reachable source by default, but allow people to re-build images on

From 9f2e2ce0cb6f7563aa3ac7176ab2793cf9e79193 Mon Sep 17 00:00:00 2001
From: jay vyas <jvyas@vmware.com>
Date: Wed, 23 Nov 2022 17:40:50 +0100
Subject: [PATCH 5/6] revert poller.go change, irrelevant

---
 tkg/clusterclient/poller.go | 30 +++++-------------------------
 1 file changed, 5 insertions(+), 25 deletions(-)

diff --git a/tkg/clusterclient/poller.go b/tkg/clusterclient/poller.go
index 59b453348f..7e7a4e2d0f 100644
--- a/tkg/clusterclient/poller.go
+++ b/tkg/clusterclient/poller.go
@@ -95,31 +95,11 @@ func (p *pollerProxy) PollImmediateInfiniteWithGetter(interval time.Duration, ge
 		return true, nil
 	}
 
-	timeoutReached := false
-	ch := time.After(interval)
-	// dont return prematurely.  workaround to the fact that
-	// apimachinery will prematurely return if any errors, but
-	// we want to be forgiving of race conditions for looking up the vsphere secrets, for example.
-	go func() {
-		<- ch 
-		timeoutReached = true
-	}
-	for timeoutReached {
-		// note: APIMachinery will return immediately on failure, however, we don't want
-		// to always return immediately.
-		errPoll := p.PollImmediateInfinite(interval, pollerFunc)
-		if errPoll != nil {
-			// note: this function will return actual error which is thrown by getterFunc
-			// and will not return error of the PollImmediateInfinite call.
-			// note: caller shouldnt necessarily terminate on errors...
-			log.Info(errPoll.Error() + "underlying error:" + err.Error())
-			// not returning error yet...
-		} else {
-			log.V(6).Info("no errors, but not ready yet, retrying")
-		}
-
-
+	errPoll := p.PollImmediateInfinite(interval, pollerFunc)
+	if errPoll != nil {
+		// note: this function will return actual error which is thrown by getterFunc
+		// and will not return error of the PollImmediateInfinite call
+		return err
 	}
-
 	return nil
 }

From 4f83805e4f52d810a16ee580d66932d57b8e3a52 Mon Sep 17 00:00:00 2001
From: jay vyas <jvyas@vmware.com>
Date: Wed, 23 Nov 2022 21:21:57 +0100
Subject: [PATCH 6/6] woudl typo fix

---
 tkr/controller/tkr-source/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tkr/controller/tkr-source/Dockerfile b/tkr/controller/tkr-source/Dockerfile
index 2d61c2748b..e70a34ccfe 100644
--- a/tkr/controller/tkr-source/Dockerfile
+++ b/tkr/controller/tkr-source/Dockerfile
@@ -24,7 +24,7 @@ COPY tkr/go.sum tkr/go.sum
 # ENV CACHE_GOMOD='--mount="type=cache,target=/go/pkg/mod"'
 # ENV CACHE="$CACHE_GOMOD $CACHE_GOLANG $CACHE_GOBUILD"
 # RUN $CACHE go mod download
-# Since this failed, we copy paste 3 --mount commands explicitly on line 31 below.  Someday woudl be nice to generalize all of that into a interpolated call.
+# Since this failed, we copy paste 3 --mount commands explicitly on line 31 below.  Someday would be nice to generalize all of that into a interpolated call.
 
 WORKDIR /workspace/tkr