Merge pull request #1616 from buildpacks-community/blob-creds

Support authentication methods for blobs

api/openapi-spec/swagger.json

@@ -5721,6 +5721,9 @@
           "x-kubernetes-patch-merge-key": "type",
           "x-kubernetes-patch-strategy": "merge"
         },
+        "latestAttestationImage": {
+          "type": "string"
+        },
         "latestCacheImage": {
           "type": "string"
         },
@@ -6997,6 +7000,9 @@
         "url"
       ],
       "properties": {
+        "auth": {
+          "type": "string"
+        },
         "stripComponents": {
           "type": "integer",
           "format": "int64"

cmd/build-init/main.go

@@ -36,6 +36,7 @@ var (
 	gitRevision             = flag.String("git-revision", os.Getenv("GIT_REVISION"), "The Git revision to make the repository HEAD.")
 	gitInitializeSubmodules = flag.Bool("git-initialize-submodules", getenvBool("GIT_INITIALIZE_SUBMODULES"), "Initialize submodules during git clone")
 	blobURL                 = flag.String("blob-url", os.Getenv("BLOB_URL"), "The url of the source code blob.")
+	blobAuth                = flag.Bool("blob-auth", getenvBool("BLOB_AUTH"), "If authentication should be used for blobs")
 	stripComponents         = flag.Int("strip-components", getenvInt("BLOB_STRIP_COMPONENTS", 0), "The number of directory components to strip from the blobs content when extracting.")
 	registryImage           = flag.String("registry-image", os.Getenv("REGISTRY_IMAGE"), "The registry location of the source code image.")
 	hostName                = flag.String("dns-probe-hostname", os.Getenv("DNS_PROBE_HOSTNAME"), "hostname to dns poll")
@@ -49,6 +50,7 @@ var (
 
 	basicGitCredentials     flaghelpers.CredentialsFlags
 	sshGitCredentials       flaghelpers.CredentialsFlags
+	blobCredentials         flaghelpers.CredentialsFlags
 	basicDockerCredentials  flaghelpers.CredentialsFlags
 	dockerCfgCredentials    flaghelpers.CredentialsFlags
 	dockerConfigCredentials flaghelpers.CredentialsFlags
@@ -60,6 +62,7 @@ var (
 func init() {
 	flag.Var(&basicGitCredentials, "basic-git", "Basic authentication for git of the form 'secretname=git.domain.com'")
 	flag.Var(&sshGitCredentials, "ssh-git", "SSH authentication for git of the form 'secretname=git.domain.com'")
+	flag.Var(&blobCredentials, "blob", "Authentication for blob of the form 'secretname=git.domain.com'")
 	flag.Var(&basicDockerCredentials, "basic-docker", "Basic authentication for docker of the form 'secretname=git.domain.com'")
 	flag.Var(&dockerCfgCredentials, "dockercfg", "Docker Cfg credentials in the form of the path to the credential")
 	flag.Var(&dockerConfigCredentials, "dockerconfig", "Docker Config JSON credentials in the form of the path to the credential")
@@ -220,8 +223,27 @@ func fetchSource(logger *log.Logger, keychain authn.Keychain) error {
 		}
 		return fetcher.Fetch(appDir, *gitURL, *gitRevision, projectMetadataDir)
 	case *blobURL != "":
+		var (
+			blobKeychain blob.Keychain
+			err          error
+		)
+		if *blobAuth {
+			if len(blobCredentials) == 0 {
+				logger.Println("Loading blob credentials from helpers")
+				blobKeychain = blob.DefaultKeychain
+			} else {
+				logger.Println("Loading blob credentials from service account secrets")
+				logLoadingSecrets(logger, blobCredentials)
+				blobKeychain, err = blob.NewMountedSecretBlobKeychain(buildSecretsDir, blobCredentials)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
 		fetcher := blob.Fetcher{
-			Logger: logger,
+			Logger:   logger,
+			Keychain: blobKeychain,
 		}
 		return fetcher.Fetch(appDir, *blobURL, *stripComponents, projectMetadataDir)
 	case *registryImage != "":

cmd/completion/main.go

@@ -48,6 +48,7 @@ var (
 	cosignDockerMediaTypes  flaghelpers.CredentialsFlags
 	basicGitCredentials     flaghelpers.CredentialsFlags
 	sshGitCredentials       flaghelpers.CredentialsFlags
+	blobCredentials         flaghelpers.CredentialsFlags
 	logger                  *log.Logger
 )
 
@@ -60,6 +61,7 @@ func init() {
 	flag.Var(&dockerConfigCredentials, "dockerconfig", "Docker Config JSON credentials in the form of the path to the credential")
 	flag.Var(&basicGitCredentials, "basic-git", "Basic authentication for git of the form 'secretname=git.domain.com'")
 	flag.Var(&sshGitCredentials, "ssh-git", "SSH authentication for git of the form 'secretname=git.domain.com'")
+	flag.Var(&blobCredentials, "blob", "Authentication for blob of the form 'secretname=git.domain.com'")
 
 	flag.Var(&cosignAnnotations, "cosign-annotations", "Cosign custom signing annotations")
 	flag.Var(&cosignRepositories, "cosign-repositories", "Cosign signing repository of the form 'secretname=registry.example.com/project'")

docs/build.md

@@ -95,6 +95,10 @@ The `source` field is a composition of a source code location and a `subpath`. I
     - `git`: (Source Code is a git repository)
         - `url`: The git repository url. Both https and ssh formats are supported; with ssh format requiring a [ssh secret](secrets.md#git-secrets).
         - `revision`: The git revision to use. This value may be a commit sha, branch name, or tag.
+        - `auth`: Optional auth to use with blob source. Leave empty for no auth, "secret" for providing auth [via Secret](secrets.md#blob-secrets), or "helper" to use service account IAM (specific to each IaaS).
+             > Note: Only [Microsoft Azure](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview)
+             > and [Google Cloud Platform](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#kubernetes-sa-to-iam)
+             > helpers are currently implemented, contributions are welcome to `pkg/blob/<iaas>_keychain.go`.
     - `subPath`: A subdirectory within the source folder where application code resides. Can be ignored if the source code resides at the `root` level.
 
 * Blob

docs/image.md

@@ -99,11 +99,16 @@ The `source` field is a composition of a source code location and a `subpath`. I
       blob:
         url: ""
         stripComponents: 0
+        auth: "" | "secret" | "helper"
       subPath: ""
     ```
     - `blob`: (Source Code is a blob/jar in a blobstore)
         - `url`: The URL of the source code blob. This blob needs to either be publicly accessible or have the access token in the URL
         - `stripComponents`: Optional number of directory components to strip from the blobs content when extracting.
+        - `auth`: Optional auth to use with blob source. Leave empty for no auth, "secret" for providing auth [via Secret](secrets.md#blob-secrets), or "helper" to use service account IAM (specific to each IaaS).
+             > Note: Only [Microsoft Azure](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview)
+             > and [Google Cloud Platform](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#kubernetes-sa-to-iam)
+             > helpers are currently implemented, contributions are welcome to `pkg/blob/<iaas>_keychain.go`.
     - `subPath`: A subdirectory within the source folder where application code resides. Can be ignored if the source code resides at the `root` level.
 
 * Registry

docs/secrets.md

@@ -126,6 +126,26 @@ stringData:
   password: <generated-token>
 ```
 
+### Blob Secrets
+
+Secrets are used with a `kpack.io/blob` annotation that references a hostname for a blob location. Only one of username/password, bearer, or authorization is allowed.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: blob-secret
+  annotations:
+    kpack.io/blob: my-blob-store.com
+stringData:
+  username: <username>
+  password: <password>
+
+  bearer: <oauth2 token>
+
+  authorization: <third-party-auth-header>
+```
+
 ### Service Account
 
 To use these secrets with kpack create a service account and reference the service account in image and build resources. When configuring the image resource, reference the `name` of your registry credential and the `name` of your git credential.

go.mod

@@ -3,6 +3,9 @@ module github.com/pivotal/kpack
 go 1.21
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
 	github.com/BurntSushi/toml v1.3.2
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
@@ -30,6 +33,7 @@ require (
 	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.19.0
 	golang.org/x/net v0.21.0
+	golang.org/x/oauth2 v0.15.0
 	golang.org/x/sync v0.6.0
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0
@@ -50,6 +54,7 @@ require (
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0 // indirect
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
@@ -61,6 +66,7 @@ require (
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/OneOfOne/xxhash v1.2.8 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
@@ -158,6 +164,7 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
@@ -188,6 +195,7 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -216,6 +224,7 @@ require (
 	github.com/pborman/uuid v1.2.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.17.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
@@ -269,7 +278,6 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/oauth2 v0.15.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/term v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect

go.sum

@@ -77,6 +77,8 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1 h1:MyVTgWR
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1/go.mod h1:GpPjLhVR9dnUoJMyHWSPy71xY9/lcmpzIPZXmF0FCVY=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80=
 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
 github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
@@ -387,6 +389,8 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
+github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
 github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -1548,6 +1552,7 @@ golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/apis/build/v1alpha2/build_pod.go

@@ -43,6 +43,7 @@ const (
 	cosignRespositoryAnnotationPrefix      = "kpack.io/cosign.repository"
 	DOCKERSecretAnnotationPrefix           = "kpack.io/docker"
 	GITSecretAnnotationPrefix              = "kpack.io/git"
+	BlobSecretAnnotationPrefix             = "kpack.io/blob"
 	IstioInject                            = "sidecar.istio.io/inject"
 	BuildReadyAnnotation                   = "build.kpack.io/ready"
 
@@ -234,7 +235,9 @@ func (b *Build) BuildPod(images BuildPodImages, buildContext BuildContext) (*cor
 		buildEnv = append(buildEnv, envVar)
 	}
 
-	secretVolumes, secretVolumeMounts, secretArgs := b.setupSecretVolumesAndArgs(buildContext.Secrets, gitAndDockerSecrets)
+	blobAuthUseSecrets := b.Spec.Source.Blob != nil && b.Spec.Source.Blob.Auth == string(corev1alpha1.BlobAuthSecret)
+
+	secretVolumes, secretVolumeMounts, secretArgs := b.setupSecretVolumesAndArgs(buildContext.Secrets, buildSecrets(blobAuthUseSecrets))
 	cosignVolumes, cosignVolumeMounts, cosignSecretArgs := b.setupCosignVolumes(buildContext.Secrets)
 	imagePullVolumes, imagePullVolumeMounts, imagePullArgs := b.setupImagePullVolumes(buildContext.ImagePullSecrets)
 
@@ -978,8 +981,18 @@ func (b *Build) cacheVolume(os string) []corev1.Volume {
 	}}
 }
 
-func gitAndDockerSecrets(secret corev1.Secret) bool {
-	return secret.Annotations[GITSecretAnnotationPrefix] != "" || dockerSecrets(secret)
+func buildSecrets(includeBlobSecrets bool) func(corev1.Secret) bool {
+	return func(secret corev1.Secret) bool {
+		return gitSecrets(secret) || blobSecrets(includeBlobSecrets, secret) || dockerSecrets(secret)
+	}
+}
+
+func gitSecrets(secret corev1.Secret) bool {
+	return secret.Annotations[GITSecretAnnotationPrefix] != ""
+}
+
+func blobSecrets(includeBlobSecret bool, secret corev1.Secret) bool {
+	return includeBlobSecret && secret.Annotations[BlobSecretAnnotationPrefix] != ""
 }
 
 func dockerSecrets(secret corev1.Secret) bool {
@@ -1009,6 +1022,9 @@ func (b *Build) setupSecretVolumesAndArgs(secrets []corev1.Secret, filter func(s
 		case secret.Type == corev1.SecretTypeSSHAuth:
 			annotatedUrl := secret.Annotations[GITSecretAnnotationPrefix]
 			args = append(args, fmt.Sprintf("-ssh-%s=%s=%s", "git", secret.Name, annotatedUrl))
+		case secret.Annotations[BlobSecretAnnotationPrefix] != "":
+			annotatedUrl := secret.Annotations[BlobSecretAnnotationPrefix]
+			args = append(args, fmt.Sprintf("-blob=%s=%s", secret.Name, annotatedUrl))
 		default:
 			//ignoring secret
 			continue