redo of BUILDKIT caching PR

addons/Dockerfile

@@ -1,26 +1,28 @@
 # Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
 
 # Build the manager binary
 FROM $BUILDER_BASE_IMAGE as builder
-
 WORKDIR /workspace
-
 # Copy the go source
 COPY ./ ./
 
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN cd addons && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o bin/manager main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod cd addons && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -ldflags "$LD_FLAGS" -o bin/manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

addons/manifests/samples/simple-app/Dockerfile

@@ -1,5 +1,8 @@
 # Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+
 FROM scratch
 COPY templates/* ./

capabilities/controller/Dockerfile

@@ -17,12 +17,12 @@ COPY apis/ apis/
 COPY cli/runtime cli/runtime
 
 WORKDIR capabilities/controller
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

common.mk

@@ -10,6 +10,11 @@ ROOT_DIR := $(shell git rev-parse --show-toplevel)
 RELATIVE_ROOT ?= .
 CONTROLLER_GEN_SRC ?= "./..."
 
+# Framework has lots of components, and many build steps that are disparate.
+# Use docker buildkit so that we get faster image builds and skip redundant work.
+# TODO: We need to measure what this speeds up, and what it doesn't (and why).
+export DOCKER_BUILDKIT := 1
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin

featuregates/controller/Dockerfile

@@ -1,5 +1,7 @@
-# Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# Enable Buildkit
+
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -19,7 +21,7 @@ COPY cli/runtime cli/runtime
 COPY capabilities/client capabilities/client
 
 WORKDIR featuregates/controller
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -31,7 +33,7 @@ COPY ./featuregates/controller/main.go ./main.go
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

object-propagation/Dockerfile

@@ -1,6 +1,9 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
@@ -18,7 +21,7 @@ COPY object-propagation/go.mod object-propagation/go.mod
 COPY object-propagation/go.sum object-propagation/go.sum
 
 WORKDIR /workspace/object-propagation
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +31,7 @@ COPY object-propagation/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -37,4 +40,4 @@ WORKDIR /
 COPY --from=builder /workspace/object-propagation/manager .
 USER nonroot:nonroot
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/manager"]

pinniped-components/post-deploy/Dockerfile

@@ -1,5 +1,6 @@
 # Copyright 2021 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker.io/docker/dockerfile:1.3.0
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -14,15 +15,16 @@ WORKDIR /workspace
 COPY tanzu-auth-controller-manager tanzu-auth-controller-manager
 COPY post-deploy/go.mod post-deploy/go.mod
 COPY post-deploy/go.sum post-deploy/go.sum
-RUN cd post-deploy && go mod download
+
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod cd post-deploy && go mod download
 
 # Copy the source
 COPY post-deploy/cmd/ post-deploy/cmd/
 COPY post-deploy/pkg/ post-deploy/pkg/
 COPY post-deploy/Makefile post-deploy/Makefile
 #COPY .git/ .git/
 
-RUN make native -C post-deploy
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod make native -C post-deploy
 
 # Support older deployment YAMLs by providing symlink for the historic job name without -job.
 # The subdirectory here is to ensure that the docker COPY command does not follow the symlink
@@ -36,4 +38,4 @@ WORKDIR /
 COPY --from=builder /tmp/links/ .
 COPY --from=builder /workspace/post-deploy/tkg-pinniped-post-deploy-job .
 COPY --from=builder /workspace/post-deploy/tkg-pinniped-post-deploy-controller .
-CMD ["/tkg-pinniped-post-deploy-job"]
+CMD ["/tkg-pinniped-post-deploy-job"]

pinniped-components/tanzu-auth-controller-manager/Dockerfile

@@ -1,6 +1,9 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
+
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
 ARG BUILDER_BASE_IMAGE=golang:1.18
@@ -12,7 +15,7 @@ WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
-RUN go mod download
+RUN  --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 
 # Copy the source
 COPY main.go main.go
@@ -21,7 +24,7 @@ COPY controllers/ controllers/
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o tanzu-auth-controller-manager .
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o tanzu-auth-controller-manager .
 
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /

pkg/v1/tkr/Dockerfile

@@ -1,5 +1,6 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker.io/docker/dockerfile:1.3.0
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -18,7 +19,7 @@ COPY pkg/v1/tkr/go.mod pkg/v1/tkr/go.mod
 COPY pkg/v1/tkr/go.sum pkg/v1/tkr/go.sum
 
 WORKDIR /workspace/pkg/v1/tkr
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +29,7 @@ COPY pkg/v1/tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager ./main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

tkg/vsphere-template-resolver/Dockerfile

@@ -1,5 +1,6 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -26,7 +27,7 @@ COPY tkg/go.mod tkg/go.mod
 COPY tkg/go.sum tkg/go.sum
 
 WORKDIR /workspace/tkg
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -36,7 +37,7 @@ COPY tkg/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager vsphere-template-resolver/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager vsphere-template-resolver/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

tkr/controller/tkr-source/Dockerfile

@@ -1,5 +1,6 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -17,8 +18,18 @@ COPY util/ util/
 COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
+# TODO: @jayunit100 ~ I Tried this but it didnt work, filed an upstream issue https://github.com/moby/buildkit/issues/3301 
+# ENV CACHE_GOBUILD='--mount="type=cache,target=/root/.cache/go-build"'
+# ENV CACHE_GOLANG='--mount="type=cache,target=/root/.local/share/golang"'
+# ENV CACHE_GOMOD='--mount="type=cache,target=/go/pkg/mod"'
+# ENV CACHE="$CACHE_GOMOD $CACHE_GOLANG $CACHE_GOBUILD"
+# RUN $CACHE go mod download
+# Since this failed, we copy paste 3 --mount commands explicitly on line 31 below.  Someday would be nice to generalize all of that into a interpolated call.
+
 WORKDIR /workspace/tkr
-RUN  go mod download
+
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
+
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +39,9 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-source/main.go
+
+# RUN --mount=type=cache,target=/root/.cache/go-build \
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-source/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

tkr/controller/tkr-status/Dockerfile

@@ -1,5 +1,6 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4 
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -18,7 +19,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +29,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-status/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager controller/tkr-status/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

tkr/webhook/cluster/tkr-resolver/Dockerfile

@@ -1,5 +1,7 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# Enable Buildkit
+# syntax=docker/dockerfile:1.4
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -18,7 +20,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +30,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/cluster/tkr-resolver/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/cluster/tkr-resolver/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

tkr/webhook/tkr-conversion/Dockerfile

@@ -1,5 +1,8 @@
 # Copyright 2022 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+# syntax=docker/dockerfile:1.4
+
+# Copyright 2021 VMware, Inc. All Rights Reserved.
 
 # Build from publicly reachable source by default, but allow people to re-build images on
 # top of their own trusted images.
@@ -18,7 +21,7 @@ COPY tkr/go.mod tkr/go.mod
 COPY tkr/go.sum tkr/go.sum
 
 WORKDIR /workspace/tkr
-RUN  go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod go mod download
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 
@@ -28,7 +31,7 @@ COPY tkr/ ./
 # Build
 ARG LD_FLAGS
 ENV LD_FLAGS="$LD_FLAGS "'-extldflags "-static"'
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/.local/share/golang --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -ldflags "$LD_FLAGS" -o manager webhook/tkr-conversion/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details