From 5f478669a1cf490a4854555044f59276cd2ce05a Mon Sep 17 00:00:00 2001 From: Bohan Chen Date: Fri, 10 May 2024 15:20:31 -0400 Subject: [PATCH 1/5] stop using github.com/pkg/errors since it's been archived Signed-off-by: Bohan Chen --- pkg/blob/fetch.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/blob/fetch.go b/pkg/blob/fetch.go index c10feafb5..f1235b56a 100644 --- a/pkg/blob/fetch.go +++ b/pkg/blob/fetch.go @@ -12,12 +12,11 @@ import ( "path" "github.com/BurntSushi/toml" - "github.com/pkg/errors" "github.com/pivotal/kpack/pkg/archive" ) -var unexpectedBlobTypeError = errors.New("unexpected blob file type, must be one of .zip, .tar.gz, .tar, .jar") +var errUnexpectedBlobType = fmt.Errorf("unexpected blob file type, must be one of .zip, .tar.gz, .tar, .jar") type Fetcher struct { Logger *log.Logger @@ -58,11 +57,11 @@ func (f *Fetcher) Fetch(dir string, blobURL string, stripComponents int, metadat err = archive.ExtractTarGZ(file, dir, stripComponents) case "application/octet-stream": if !archive.IsTar(file.Name()) { - return unexpectedBlobTypeError + return errUnexpectedBlobType } err = archive.ExtractTar(file, dir, stripComponents) default: - return unexpectedBlobTypeError + return errUnexpectedBlobType } if err != nil { return err @@ -70,7 +69,7 @@ func (f *Fetcher) Fetch(dir string, blobURL string, stripComponents int, metadat projectMetadataFile, err := os.Create(path.Join(metadataDir, "project-metadata.toml")) if err != nil { - return errors.Wrapf(err, "invalid metadata destination '%s/project-metadata.toml' for blob: %s", metadataDir, blobURL) + return fmt.Errorf("invalid metadata destination '%s/project-metadata.toml' for blob '%s': %v", metadataDir, blobURL, err) } defer projectMetadataFile.Close() @@ -86,7 +85,7 @@ func (f *Fetcher) Fetch(dir string, blobURL string, stripComponents int, metadat }, } if err := toml.NewEncoder(projectMetadataFile).Encode(projectMd); err != nil { - return errors.Wrapf(err, "invalid metadata destination '%s/project-metadata.toml' for blob: %s", metadataDir, blobURL) + return fmt.Errorf("invalid metadata destination '%s/project-metadata.toml' for blob '%s': %v", metadataDir, blobURL, err) } f.Logger.Printf("Successfully downloaded %s%s in path %q", u.Host, u.Path, dir) @@ -102,7 +101,7 @@ func downloadBlob(blobURL string) (*os.File, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, errors.Errorf("failed to get blob %s", blobURL) + return nil, fmt.Errorf("failed to get blob %s", blobURL) } file, err := os.CreateTemp("", "") From ff457d26650ed5289ec2098c5ea169100039fd27 Mon Sep 17 00:00:00 2001 From: Bohan Chen Date: Thu, 16 May 2024 16:34:45 -0400 Subject: [PATCH 2/5] introduce keychains to blob fetcher similar to image keychains, the blob keychain is an interface to resolve a url to an auth string, and potentially other headers. while it's true the auth string can be embeded in the header, i felt separating them is more convenient as most keychains won't have to make use of the additional headers part. there's no aws keychain since i couldn't figure out how aws-sdk-go-v2 handles eks's oidc flow. And i don't have easy access to an aws environment to test this out on Signed-off-by: Bohan Chen --- go.mod | 10 ++- go.sum | 5 ++ pkg/blob/azure_keychain.go | 50 ++++++++++++++ pkg/blob/fetch.go | 30 +++++++-- pkg/blob/fetch_test.go | 51 ++++++++++++++ pkg/blob/file_keychain.go | 119 +++++++++++++++++++++++++++++++++ pkg/blob/file_keychain_test.go | 113 +++++++++++++++++++++++++++++++ pkg/blob/gcp_keychain.go | 28 ++++++++ pkg/blob/keychain.go | 30 +++++++++ pkg/blob/keychain_test.go | 71 ++++++++++++++++++++ 10 files changed, 502 insertions(+), 5 deletions(-) create mode 100644 pkg/blob/azure_keychain.go create mode 100644 pkg/blob/file_keychain.go create mode 100644 pkg/blob/file_keychain_test.go create mode 100644 pkg/blob/gcp_keychain.go create mode 100644 pkg/blob/keychain.go create mode 100644 pkg/blob/keychain_test.go diff --git a/go.mod b/go.mod index 0537073b8..fa05fafb2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/pivotal/kpack go 1.21 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/semver/v3 v3.2.1 github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a @@ -30,6 +33,7 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/crypto v0.19.0 golang.org/x/net v0.21.0 + golang.org/x/oauth2 v0.15.0 golang.org/x/sync v0.6.0 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 @@ -50,6 +54,7 @@ require ( filippo.io/edwards25519 v1.0.0 // indirect github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect @@ -61,6 +66,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect @@ -158,6 +164,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -188,6 +195,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.2 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -216,6 +224,7 @@ require ( github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -269,7 +278,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/oauth2 v0.15.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index ff722d5b6..80702e26a 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1 h1:MyVTgWR github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1/go.mod h1:GpPjLhVR9dnUoJMyHWSPy71xY9/lcmpzIPZXmF0FCVY= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= @@ -387,6 +389,8 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg= github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -1548,6 +1552,7 @@ golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/blob/azure_keychain.go b/pkg/blob/azure_keychain.go new file mode 100644 index 000000000..d685a409a --- /dev/null +++ b/pkg/blob/azure_keychain.go @@ -0,0 +1,50 @@ +package blob + +import ( + "context" + "fmt" + "regexp" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" +) + +var ( + azScope = "https://storage.azure.com/.default" + azApiVersion = sas.Version + azRegex = regexp.MustCompile(`.*[a-z0-9]+\.([a-z]+)\.core\.windows\.net\/.*`) +) + +type azKeychain struct{} + +func (a azKeychain) Resolve(url string) (string, map[string]string, error) { + submatches := azRegex.FindStringSubmatch(url) + if len(submatches) != 2 { + return "", nil, fmt.Errorf("not an azure url") + } + service := submatches[1] + + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return "", nil, err + } + + tk, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{Scopes: []string{azScope}}) + if err != nil { + return "", nil, err + } + + headers := map[string]string{ + "x-ms-version": azApiVersion, + "x-ms-date": time.Now().Format(time.RFC1123), + } + + // https://learn.microsoft.com/en-us/rest/api/storageservices/get-file + if service == "file" { + headers["x-ms-file-request-intent"] = "backup" + } + + return "Bearer " + tk.Token, headers, nil +} diff --git a/pkg/blob/fetch.go b/pkg/blob/fetch.go index f1235b56a..0772dba31 100644 --- a/pkg/blob/fetch.go +++ b/pkg/blob/fetch.go @@ -19,7 +19,8 @@ import ( var errUnexpectedBlobType = fmt.Errorf("unexpected blob file type, must be one of .zip, .tar.gz, .tar, .jar") type Fetcher struct { - Logger *log.Logger + Logger *log.Logger + Keychain Keychain } func (f *Fetcher) Fetch(dir string, blobURL string, stripComponents int, metadataDir string) error { @@ -27,9 +28,21 @@ func (f *Fetcher) Fetch(dir string, blobURL string, stripComponents int, metadat if err != nil { return err } + + var headers map[string]string + if f.Keychain != nil { + var auth string + auth, headers, err = f.Keychain.Resolve(blobURL) + if err != nil { + return fmt.Errorf("failed to resolve creds: %v", err) + } + + headers["Authorization"] = auth + } + f.Logger.Printf("Downloading %s%s...", u.Host, u.Path) - file, err := downloadBlob(blobURL) + file, err := downloadBlob(blobURL, headers) if err != nil { return err } @@ -93,8 +106,17 @@ func (f *Fetcher) Fetch(dir string, blobURL string, stripComponents int, metadat return nil } -func downloadBlob(blobURL string) (*os.File, error) { - resp, err := http.Get(blobURL) +func downloadBlob(blobURL string, headers map[string]string) (*os.File, error) { + req, err := http.NewRequest(http.MethodGet, blobURL, nil) + if err != nil { + return nil, err + } + + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } diff --git a/pkg/blob/fetch_test.go b/pkg/blob/fetch_test.go index bf4fa618f..9137bc965 100644 --- a/pkg/blob/fetch_test.go +++ b/pkg/blob/fetch_test.go @@ -168,4 +168,55 @@ func testBlobFetcher(t *testing.T, when spec.G, it spec.S) { err := fetcher.Fetch(dir, fmt.Sprintf("%s/%s", server.URL, "test.html"), 0, metadataDir) require.EqualError(t, err, "unexpected blob file type, must be one of .zip, .tar.gz, .tar, .jar") }) + + when("there's auth required", func() { + var ( + handler = &authHandler{http.FileServer(http.Dir("./testdata")), nil} + server = httptest.NewServer(handler) + ) + + it("doesn't send headers when there's no keychain", func() { + err := fetcher.Fetch(dir, fmt.Sprintf("%s/%s", server.URL, "test.zip"), 0, metadataDir) + require.NoError(t, err) + + require.NotContains(t, handler.headers, "Authorization") + }) + + it("uses the auth and headers from the keychain", func() { + fetcher = &blob.Fetcher{ + Logger: log.New(output, "", 0), + Keychain: &fakeKeychain{"some-auth", map[string]string{"Some-Header": "some-value"}, nil}, + } + + err := fetcher.Fetch(dir, fmt.Sprintf("%s/%s", server.URL, "test.zip"), 0, metadataDir) + require.NoError(t, err) + headers := handler.headers + + require.Contains(t, headers, "Authorization") + require.Equal(t, []string{"some-auth"}, headers["Authorization"]) + + require.Contains(t, headers, "Some-Header") + require.Equal(t, []string{"some-value"}, headers["Some-Header"]) + }) + + it("surfaces the error", func() { + fetcher = &blob.Fetcher{ + Logger: log.New(output, "", 0), + Keychain: &fakeKeychain{"", nil, fmt.Errorf("some-error")}, + } + + err := fetcher.Fetch(dir, fmt.Sprintf("%s/%s", server.URL, "test.zip"), 0, metadataDir) + require.EqualError(t, err, "failed to resolve creds: some-error") + }) + }) +} + +type authHandler struct { + h http.Handler + headers http.Header +} + +func (a *authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + a.headers = r.Header.Clone() + a.h.ServeHTTP(w, r) } diff --git a/pkg/blob/file_keychain.go b/pkg/blob/file_keychain.go new file mode 100644 index 000000000..ac76d6c91 --- /dev/null +++ b/pkg/blob/file_keychain.go @@ -0,0 +1,119 @@ +package blob + +import ( + "encoding/base64" + "errors" + "fmt" + "io/fs" + "net/url" + "os" + "path/filepath" + "strings" +) + +var errMultipleAuths = fmt.Errorf("only one of username/password, bearer, authorization is allowed") + +type fileCredential struct { + domain string + secretName string + + username string + password string + bearer string + authorization string +} + +type fileCreds struct { + creds []fileCredential +} + +func NewMountedSecretBlobKeychain(volumeName string, secrets []string) (*fileCreds, error) { + var creds []fileCredential + for _, s := range secrets { + splitSecret := strings.Split(s, "=") + if len(splitSecret) != 2 { + return nil, fmt.Errorf("could not parse blob secret argument %s", s) + } + + dir := os.DirFS(filepath.Join(volumeName, splitSecret[0])) + username, err := readFile(dir, "username") + if err != nil { + return nil, err + } + password, err := readFile(dir, "password") + if err != nil { + return nil, err + } + bearer, err := readFile(dir, "bearer") + if err != nil { + return nil, err + } + authorization, err := readFile(dir, "authorization") + if err != nil { + return nil, err + } + + creds = append(creds, fileCredential{ + domain: splitSecret[1], + secretName: splitSecret[0], + + username: username, + password: password, + bearer: bearer, + authorization: authorization, + }) + } + return &fileCreds{creds}, nil +} + +func readFile(dirFs fs.FS, filename string) (string, error) { + _, err := fs.Stat(dirFs, filename) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", nil + } else { + return "", err + } + } + + buf, err := fs.ReadFile(dirFs, filename) + return string(buf), err +} + +func (f *fileCreds) Resolve(blobUrl string) (string, map[string]string, error) { + u, err := url.Parse(blobUrl) + if err != nil { + return "", nil, fmt.Errorf("invalid url '%v': %v", blobUrl, u) + } + + for _, cred := range f.creds { + if u.Hostname() != cred.domain { + continue + } + + var authHeader []string + if cred.username != "" || cred.password != "" { + encoded := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v", cred.username, cred.password))) + authHeader = append(authHeader, "Basic "+encoded) + } + + if cred.bearer != "" { + authHeader = append(authHeader, "Bearer "+cred.bearer) + } + + if cred.authorization != "" { + authHeader = append(authHeader, cred.authorization) + } + + switch len(authHeader) { + case 0: + return "", nil, fmt.Errorf("no auths found for '%v'", cred.secretName) + case 1: + return authHeader[0], nil, nil + default: + return "", nil, fmt.Errorf("multiple auths found for '%v', only one of username/password, bearer, authorization is allowed", cred.secretName) + } + + } + return "", nil, fmt.Errorf("no secrets matched for '%v'", u.Hostname()) +} diff --git a/pkg/blob/file_keychain_test.go b/pkg/blob/file_keychain_test.go new file mode 100644 index 000000000..c23d8add6 --- /dev/null +++ b/pkg/blob/file_keychain_test.go @@ -0,0 +1,113 @@ +package blob_test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/pivotal/kpack/pkg/blob" + "github.com/sclevine/spec" + "github.com/stretchr/testify/require" +) + +func TestFileKeychain(t *testing.T) { + spec.Run(t, "testFileKeychain", testFileKeychain) +} + +func testFileKeychain(t *testing.T, when spec.G, it spec.S) { + var ( + testVolume string + testDir string + hostName = "some-blobstore.com" + secretName = "some-secret" + ) + + it.Before(func() { + var err error + testVolume, err = os.MkdirTemp("", "") + require.NoError(t, err) + + testDir = filepath.Join(testVolume, secretName) + require.NoError(t, os.Mkdir(testDir, 0777)) + }) + + it.After(func() { + require.NoError(t, os.RemoveAll(testDir)) + }) + + when("files are valid", func() { + it("reads username/password", func() { + os.WriteFile(filepath.Join(testDir, "username"), []byte("some-username"), 0777) + os.WriteFile(filepath.Join(testDir, "password"), []byte("some-password"), 0777) + + keychain, err := blob.NewMountedSecretBlobKeychain(testVolume, []string{fmt.Sprintf("%v=%v", secretName, hostName)}) + require.NoError(t, err) + + auth, header, err := keychain.Resolve("https://some-blobstore.com") + require.NoError(t, err) + + require.Equal(t, "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", auth) + require.Nil(t, header) + }) + + it("reads bearer token", func() { + os.WriteFile(filepath.Join(testDir, "bearer"), []byte("some-token"), 0777) + + keychain, err := blob.NewMountedSecretBlobKeychain(testVolume, []string{fmt.Sprintf("%v=%v", secretName, hostName)}) + require.NoError(t, err) + + auth, header, err := keychain.Resolve("https://some-blobstore.com") + require.NoError(t, err) + + require.Equal(t, "Bearer some-token", auth) + require.Nil(t, header) + }) + + it("reads authorization", func() { + os.WriteFile(filepath.Join(testDir, "authorization"), []byte("PRIVATE-METHOD some-auth"), 0777) + + keychain, err := blob.NewMountedSecretBlobKeychain(testVolume, []string{fmt.Sprintf("%v=%v", secretName, hostName)}) + require.NoError(t, err) + + auth, header, err := keychain.Resolve("https://some-blobstore.com") + require.NoError(t, err) + + require.Equal(t, "PRIVATE-METHOD some-auth", auth) + require.Nil(t, header) + }) + }) + + when("files are invalid", func() { + it("errors if no method is found", func() { + keychain, err := blob.NewMountedSecretBlobKeychain(testVolume, []string{fmt.Sprintf("%v=%v", secretName, hostName)}) + require.NoError(t, err) + + _, _, err = keychain.Resolve("https://some-blobstore.com") + require.EqualError(t, err, "no auths found for 'some-secret'") + }) + + it("errors if more than one method is found", func() { + os.WriteFile(filepath.Join(testDir, "username"), []byte("some-username"), 0777) + os.WriteFile(filepath.Join(testDir, "password"), []byte("some-password"), 0777) + os.WriteFile(filepath.Join(testDir, "bearer"), []byte("some-token"), 0777) + + keychain, err := blob.NewMountedSecretBlobKeychain(testVolume, []string{fmt.Sprintf("%v=%v", secretName, hostName)}) + require.NoError(t, err) + + _, _, err = keychain.Resolve("https://some-blobstore.com") + require.EqualError(t, err, "multiple auths found for 'some-secret', only one of username/password, bearer, authorization is allowed") + }) + + it("errors if the domain doesn't match", func() { + os.WriteFile(filepath.Join(testDir, "username"), []byte("some-username"), 0777) + os.WriteFile(filepath.Join(testDir, "password"), []byte("some-password"), 0777) + + keychain, err := blob.NewMountedSecretBlobKeychain(testVolume, []string{fmt.Sprintf("%v=%v", secretName, hostName)}) + require.NoError(t, err) + + _, _, err = keychain.Resolve("https://some-other-domain.com") + require.EqualError(t, err, "no secrets matched for 'some-other-domain.com'") + }) + }) +} diff --git a/pkg/blob/gcp_keychain.go b/pkg/blob/gcp_keychain.go new file mode 100644 index 000000000..db8a28b3c --- /dev/null +++ b/pkg/blob/gcp_keychain.go @@ -0,0 +1,28 @@ +package blob + +import ( + "context" + + "golang.org/x/oauth2/google" +) + +const ( + gcpScope = "https://www.googleapis.com/auth/devstorage.read_only" +) + +type gcpKeychain struct{} + +func (g gcpKeychain) Resolve(url string) (string, map[string]string, error) { + ctx := context.Background() + creds, err := google.FindDefaultCredentials(ctx, gcpScope) + if err != nil { + return "", nil, err + } + + tk, err := creds.TokenSource.Token() + if err != nil { + return "", nil, err + } + + return "Bearer " + tk.AccessToken, nil, nil +} diff --git a/pkg/blob/keychain.go b/pkg/blob/keychain.go new file mode 100644 index 000000000..fe094f70e --- /dev/null +++ b/pkg/blob/keychain.go @@ -0,0 +1,30 @@ +package blob + +import "fmt" + +type Keychain interface { + Resolve(url string) (authHeader string, headers map[string]string, err error) +} + +var DefaultKeychain = NewMultiKeychain( + azKeychain{}, + gcpKeychain{}, +) + +type multiKeychain struct { + keychains []Keychain +} + +func NewMultiKeychain(creds ...Keychain) Keychain { + return &multiKeychain{creds} +} + +func (m *multiKeychain) Resolve(url string) (string, map[string]string, error) { + for _, helper := range m.keychains { + t, h, err := helper.Resolve(url) + if t != "" { + return t, h, err + } + } + return "", nil, fmt.Errorf("no keychain matched for '%v'", url) +} diff --git a/pkg/blob/keychain_test.go b/pkg/blob/keychain_test.go new file mode 100644 index 000000000..95432352e --- /dev/null +++ b/pkg/blob/keychain_test.go @@ -0,0 +1,71 @@ +package blob_test + +import ( + "fmt" + "testing" + + "github.com/pivotal/kpack/pkg/blob" + "github.com/sclevine/spec" + "github.com/stretchr/testify/require" +) + +func TestKeychain(t *testing.T) { + spec.Run(t, "testKeychain", testKeychain) +} + +func testKeychain(t *testing.T, when spec.G, it spec.S) { + var ( + goodKeychain1 = &fakeKeychain{"some-auth", nil, nil} + goodKeychain2 = &fakeKeychain{"some-other-auth", map[string]string{"some-header": "some-value"}, nil} + badKeychain1 = &fakeKeychain{"", nil, fmt.Errorf("some-error")} + badKeychain2 = &fakeKeychain{"", nil, fmt.Errorf("some-other-error")} + ) + when("multi keychain", func() { + it("resolves them in order", func() { + keychain := blob.NewMultiKeychain( + goodKeychain1, + goodKeychain2, + ) + + auth, header, err := keychain.Resolve("https://some-url.com") + require.NoError(t, err) + + require.Equal(t, "some-auth", auth) + require.Nil(t, header) + }) + + it("returns the first one non-empty result", func() { + keychain := blob.NewMultiKeychain( + badKeychain1, + badKeychain2, + goodKeychain2, + ) + + auth, header, err := keychain.Resolve("https://some-url.com") + require.NoError(t, err) + + require.Equal(t, "some-other-auth", auth) + require.Contains(t, header, "some-header") + }) + + it("errors if no keychain matches", func() { + keychain := blob.NewMultiKeychain( + badKeychain1, + badKeychain2, + ) + + _, _, err := keychain.Resolve("https://some-url.com") + require.EqualError(t, err, "no keychain matched for 'https://some-url.com'") + }) + }) +} + +type fakeKeychain struct { + auth string + header map[string]string + err error +} + +func (f fakeKeychain) Resolve(_ string) (string, map[string]string, error) { + return f.auth, f.header, f.err +} From 717f709ca15d306471b83a3044b9acbce3afb4aa Mon Sep 17 00:00:00 2001 From: Bohan Chen Date: Thu, 16 May 2024 16:39:48 -0400 Subject: [PATCH 3/5] hook up blob auth to the build reconciler Signed-off-by: Bohan Chen --- cmd/build-init/main.go | 24 ++++- cmd/completion/main.go | 2 + pkg/apis/build/v1alpha2/build_pod.go | 22 ++++- pkg/apis/build/v1alpha2/build_pod_test.go | 98 +++++++++++++++++---- pkg/apis/core/v1alpha1/source_types.go | 15 ++++ pkg/apis/core/v1alpha1/source_validation.go | 4 + pkg/blob/fetch.go | 3 + pkg/blob/resolver.go | 1 + 8 files changed, 150 insertions(+), 19 deletions(-) diff --git a/cmd/build-init/main.go b/cmd/build-init/main.go index 809b6fe52..629f966c9 100644 --- a/cmd/build-init/main.go +++ b/cmd/build-init/main.go @@ -36,6 +36,7 @@ var ( gitRevision = flag.String("git-revision", os.Getenv("GIT_REVISION"), "The Git revision to make the repository HEAD.") gitInitializeSubmodules = flag.Bool("git-initialize-submodules", getenvBool("GIT_INITIALIZE_SUBMODULES"), "Initialize submodules during git clone") blobURL = flag.String("blob-url", os.Getenv("BLOB_URL"), "The url of the source code blob.") + blobAuth = flag.Bool("blob-auth", getenvBool("BLOB_AUTH"), "If authentication should be used for blobs") stripComponents = flag.Int("strip-components", getenvInt("BLOB_STRIP_COMPONENTS", 0), "The number of directory components to strip from the blobs content when extracting.") registryImage = flag.String("registry-image", os.Getenv("REGISTRY_IMAGE"), "The registry location of the source code image.") hostName = flag.String("dns-probe-hostname", os.Getenv("DNS_PROBE_HOSTNAME"), "hostname to dns poll") @@ -49,6 +50,7 @@ var ( basicGitCredentials flaghelpers.CredentialsFlags sshGitCredentials flaghelpers.CredentialsFlags + blobCredentials flaghelpers.CredentialsFlags basicDockerCredentials flaghelpers.CredentialsFlags dockerCfgCredentials flaghelpers.CredentialsFlags dockerConfigCredentials flaghelpers.CredentialsFlags @@ -60,6 +62,7 @@ var ( func init() { flag.Var(&basicGitCredentials, "basic-git", "Basic authentication for git of the form 'secretname=git.domain.com'") flag.Var(&sshGitCredentials, "ssh-git", "SSH authentication for git of the form 'secretname=git.domain.com'") + flag.Var(&blobCredentials, "blob", "Authentication for blob of the form 'secretname=git.domain.com'") flag.Var(&basicDockerCredentials, "basic-docker", "Basic authentication for docker of the form 'secretname=git.domain.com'") flag.Var(&dockerCfgCredentials, "dockercfg", "Docker Cfg credentials in the form of the path to the credential") flag.Var(&dockerConfigCredentials, "dockerconfig", "Docker Config JSON credentials in the form of the path to the credential") @@ -220,8 +223,27 @@ func fetchSource(logger *log.Logger, keychain authn.Keychain) error { } return fetcher.Fetch(appDir, *gitURL, *gitRevision, projectMetadataDir) case *blobURL != "": + var ( + blobKeychain blob.Keychain + err error + ) + if *blobAuth { + if len(blobCredentials) == 0 { + logger.Println("Loading blob credentials from helpers") + blobKeychain = blob.DefaultKeychain + } else { + logger.Println("Loading blob credentials from service account secrets") + logLoadingSecrets(logger, blobCredentials) + blobKeychain, err = blob.NewMountedSecretBlobKeychain(buildSecretsDir, blobCredentials) + if err != nil { + return err + } + } + } + fetcher := blob.Fetcher{ - Logger: logger, + Logger: logger, + Keychain: blobKeychain, } return fetcher.Fetch(appDir, *blobURL, *stripComponents, projectMetadataDir) case *registryImage != "": diff --git a/cmd/completion/main.go b/cmd/completion/main.go index f0feec6d2..358647bb9 100644 --- a/cmd/completion/main.go +++ b/cmd/completion/main.go @@ -48,6 +48,7 @@ var ( cosignDockerMediaTypes flaghelpers.CredentialsFlags basicGitCredentials flaghelpers.CredentialsFlags sshGitCredentials flaghelpers.CredentialsFlags + blobCredentials flaghelpers.CredentialsFlags logger *log.Logger ) @@ -60,6 +61,7 @@ func init() { flag.Var(&dockerConfigCredentials, "dockerconfig", "Docker Config JSON credentials in the form of the path to the credential") flag.Var(&basicGitCredentials, "basic-git", "Basic authentication for git of the form 'secretname=git.domain.com'") flag.Var(&sshGitCredentials, "ssh-git", "SSH authentication for git of the form 'secretname=git.domain.com'") + flag.Var(&blobCredentials, "blob", "Authentication for blob of the form 'secretname=git.domain.com'") flag.Var(&cosignAnnotations, "cosign-annotations", "Cosign custom signing annotations") flag.Var(&cosignRepositories, "cosign-repositories", "Cosign signing repository of the form 'secretname=registry.example.com/project'") diff --git a/pkg/apis/build/v1alpha2/build_pod.go b/pkg/apis/build/v1alpha2/build_pod.go index 28052489d..0d525daa6 100644 --- a/pkg/apis/build/v1alpha2/build_pod.go +++ b/pkg/apis/build/v1alpha2/build_pod.go @@ -43,6 +43,7 @@ const ( cosignRespositoryAnnotationPrefix = "kpack.io/cosign.repository" DOCKERSecretAnnotationPrefix = "kpack.io/docker" GITSecretAnnotationPrefix = "kpack.io/git" + BlobSecretAnnotationPrefix = "kpack.io/blob" IstioInject = "sidecar.istio.io/inject" BuildReadyAnnotation = "build.kpack.io/ready" @@ -234,7 +235,9 @@ func (b *Build) BuildPod(images BuildPodImages, buildContext BuildContext) (*cor buildEnv = append(buildEnv, envVar) } - secretVolumes, secretVolumeMounts, secretArgs := b.setupSecretVolumesAndArgs(buildContext.Secrets, gitAndDockerSecrets) + blobAuthUseSecrets := b.Spec.Source.Blob != nil && b.Spec.Source.Blob.Auth == string(corev1alpha1.BlobAuthSecret) + + secretVolumes, secretVolumeMounts, secretArgs := b.setupSecretVolumesAndArgs(buildContext.Secrets, buildSecrets(blobAuthUseSecrets)) cosignVolumes, cosignVolumeMounts, cosignSecretArgs := b.setupCosignVolumes(buildContext.Secrets) imagePullVolumes, imagePullVolumeMounts, imagePullArgs := b.setupImagePullVolumes(buildContext.ImagePullSecrets) @@ -978,8 +981,18 @@ func (b *Build) cacheVolume(os string) []corev1.Volume { }} } -func gitAndDockerSecrets(secret corev1.Secret) bool { - return secret.Annotations[GITSecretAnnotationPrefix] != "" || dockerSecrets(secret) +func buildSecrets(includeBlobSecrets bool) func(corev1.Secret) bool { + return func(secret corev1.Secret) bool { + return gitSecrets(secret) || blobSecrets(includeBlobSecrets, secret) || dockerSecrets(secret) + } +} + +func gitSecrets(secret corev1.Secret) bool { + return secret.Annotations[GITSecretAnnotationPrefix] != "" +} + +func blobSecrets(includeBlobSecret bool, secret corev1.Secret) bool { + return includeBlobSecret && secret.Annotations[BlobSecretAnnotationPrefix] != "" } func dockerSecrets(secret corev1.Secret) bool { @@ -1009,6 +1022,9 @@ func (b *Build) setupSecretVolumesAndArgs(secrets []corev1.Secret, filter func(s case secret.Type == corev1.SecretTypeSSHAuth: annotatedUrl := secret.Annotations[GITSecretAnnotationPrefix] args = append(args, fmt.Sprintf("-ssh-%s=%s=%s", "git", secret.Name, annotatedUrl)) + case secret.Annotations[BlobSecretAnnotationPrefix] != "": + annotatedUrl := secret.Annotations[BlobSecretAnnotationPrefix] + args = append(args, fmt.Sprintf("-blob=%s=%s", secret.Name, annotatedUrl)) default: //ignoring secret continue diff --git a/pkg/apis/build/v1alpha2/build_pod_test.go b/pkg/apis/build/v1alpha2/build_pod_test.go index 42e321bc0..7ade6f4e1 100644 --- a/pkg/apis/build/v1alpha2/build_pod_test.go +++ b/pkg/apis/build/v1alpha2/build_pod_test.go @@ -138,6 +138,15 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { }, Type: corev1.SecretTypeDockerConfigJson, }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "blob-secret", + Annotations: map[string]string{ + buildapi.BlobSecretAnnotationPrefix: "blobstore.com", + }, + }, + Type: corev1.SecretTypeOpaque, + }, { ObjectMeta: metav1.ObjectMeta{ Name: "secret-to-ignore", @@ -273,9 +282,9 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { ServiceAccountName: serviceAccount, Source: corev1alpha1.SourceConfig{ Git: &corev1alpha1.Git{ - URL: "giturl.com/git.git", - Revision: "gitrev1234", - InitializeSubmodules: true, + URL: "giturl.com/git.git", + Revision: "gitrev1234", + InitializeSubmodules: true, }, }, Cache: &buildapi.BuildCacheConfig{ @@ -580,6 +589,65 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { }) + it("configures prepare with blob credentials when using secret", func() { + build.Spec.Source = corev1alpha1.SourceConfig{ + Blob: &corev1alpha1.Blob{ + URL: "blobstore.com/source", + Auth: "secret", + }, + } + + pod, err := build.BuildPod(config, buildContext) + require.NoError(t, err) + + assert.Equal(t, pod.Spec.InitContainers[0].Name, "prepare") + assert.Equal(t, pod.Spec.InitContainers[0].Image, config.BuildInitImage) + + assert.Contains(t, pod.Spec.InitContainers[0].Args, "-blob=blob-secret=blobstore.com") + assert.Contains(t, pod.Spec.InitContainers[0].VolumeMounts, + corev1.VolumeMount{ + Name: "secret-volume-7", + MountPath: "/var/build-secrets/blob-secret", + }, + ) + assert.Contains(t, pod.Spec.InitContainers[0].Env, + corev1.EnvVar{ + Name: "BLOB_AUTH", + Value: "true", + }, + ) + }) + + it("configures prepare with blob credentials when using helper", func() { + build.Spec.Source = corev1alpha1.SourceConfig{ + Blob: &corev1alpha1.Blob{ + URL: "blobstore.com/source", + Auth: "helper", + }, + } + + pod, err := build.BuildPod(config, buildContext) + require.NoError(t, err) + + assert.Equal(t, pod.Spec.InitContainers[0].Name, "prepare") + assert.Equal(t, pod.Spec.InitContainers[0].Image, config.BuildInitImage) + + assert.NotContains(t, pod.Spec.InitContainers[0].Args, "-blob=blob-secret=blobstore.com") + assert.NotContains(t, pod.Spec.InitContainers[0].VolumeMounts, + corev1.VolumeMount{ + Name: "secret-volume-7", + MountPath: "/var/build-secrets/blob-secret", + }, + ) + + assert.Contains(t, pod.Spec.InitContainers[0].Env, + corev1.EnvVar{ + Name: "BLOB_AUTH", + Value: "true", + }, + ) + }) + it("configures prepare with the build configuration", func() { pod, err := build.BuildPod(config, buildContext) require.NoError(t, err) @@ -1464,15 +1532,15 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { assertSecretPresent(t, pod, secretName) } require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-8", + Name: "secret-volume-9", MountPath: "/var/build-secrets/cosign/cosign-secret-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-9", + Name: "secret-volume-10", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-10", + Name: "secret-volume-11", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-2", }) @@ -1674,15 +1742,15 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { assertSecretPresent(t, pod, secretName) } require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-8", + Name: "secret-volume-9", MountPath: "/var/build-secrets/cosign/cosign-secret-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-9", + Name: "secret-volume-10", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-10", + Name: "secret-volume-11", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-2", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ @@ -1797,15 +1865,15 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { assertSecretPresent(t, pod, secretName) } require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-8", + Name: "secret-volume-9", MountPath: "/var/build-secrets/cosign/cosign-secret-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-9", + Name: "secret-volume-10", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-10", + Name: "secret-volume-11", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-2", }) @@ -1964,15 +2032,15 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) { assertSecretPresent(t, pod, secretName) } require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-8", + Name: "secret-volume-9", MountPath: "/var/build-secrets/cosign/cosign-secret-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-9", + Name: "secret-volume-10", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-1", }) require.Contains(t, pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "secret-volume-10", + Name: "secret-volume-11", MountPath: "/var/build-secrets/cosign/cosign-secret-no-password-2", }) diff --git a/pkg/apis/core/v1alpha1/source_types.go b/pkg/apis/core/v1alpha1/source_types.go index e62fbc588..27b8bfaad 100644 --- a/pkg/apis/core/v1alpha1/source_types.go +++ b/pkg/apis/core/v1alpha1/source_types.go @@ -65,10 +65,19 @@ func (in *Git) ImagePullSecretsVolume(name string) corev1.Volume { } } +type BlobAuthKind string + +const ( + BlobAuthNone BlobAuthKind = "" + BlobAuthHelper BlobAuthKind = "helper" + BlobAuthSecret BlobAuthKind = "secret" +) + // +k8s:openapi-gen=true // +k8s:deepcopy-gen=true type Blob struct { URL string `json:"url"` + Auth string `json:"auth,omitempty"` StripComponents int64 `json:"stripComponents,omitempty"` } @@ -91,6 +100,10 @@ func (b *Blob) BuildEnvVars() []corev1.EnvVar { Name: "BLOB_STRIP_COMPONENTS", Value: strconv.FormatInt(b.StripComponents, 10), }, + { + Name: "BLOB_AUTH", + Value: strconv.FormatBool(b.Auth != string(BlobAuthNone)), + }, } } @@ -200,6 +213,7 @@ func (gs *ResolvedGitSource) IsPollable() bool { // +k8s:deepcopy-gen=true type ResolvedBlobSource struct { URL string `json:"url"` + Auth string `json:"auth,omitempty"` SubPath string `json:"subPath,omitempty"` StripComponents int64 `json:"stripComponents,omitempty"` } @@ -208,6 +222,7 @@ func (bs *ResolvedBlobSource) SourceConfig() SourceConfig { return SourceConfig{ Blob: &Blob{ URL: bs.URL, + Auth: bs.Auth, StripComponents: bs.StripComponents, }, SubPath: bs.SubPath, diff --git a/pkg/apis/core/v1alpha1/source_validation.go b/pkg/apis/core/v1alpha1/source_validation.go index afe231134..8e99ac2db 100644 --- a/pkg/apis/core/v1alpha1/source_validation.go +++ b/pkg/apis/core/v1alpha1/source_validation.go @@ -47,6 +47,10 @@ func (b *Blob) Validate(ctx context.Context) *apis.FieldError { return nil } + if b.Auth != "" && b.Auth != "helper" && b.Auth != "secret" { + return apis.ErrInvalidValue(b.Auth, "auth", "must be one of '', 'helper', or 'secret'") + } + fieldError := validate.FieldNotEmpty(b.URL, "url"). Also(validate.StripComponents(b.StripComponents)) diff --git a/pkg/blob/fetch.go b/pkg/blob/fetch.go index 0772dba31..dc4e6ea29 100644 --- a/pkg/blob/fetch.go +++ b/pkg/blob/fetch.go @@ -37,6 +37,9 @@ func (f *Fetcher) Fetch(dir string, blobURL string, stripComponents int, metadat return fmt.Errorf("failed to resolve creds: %v", err) } + if headers == nil { + headers = make(map[string]string) + } headers["Authorization"] = auth } diff --git a/pkg/blob/resolver.go b/pkg/blob/resolver.go index 0cd8719c3..717188a32 100644 --- a/pkg/blob/resolver.go +++ b/pkg/blob/resolver.go @@ -14,6 +14,7 @@ func (*Resolver) Resolve(ctx context.Context, sourceResolver *buildapi.SourceRes return corev1alpha1.ResolvedSourceConfig{ Blob: &corev1alpha1.ResolvedBlobSource{ URL: sourceResolver.Spec.Source.Blob.URL, + Auth: sourceResolver.Spec.Source.Blob.Auth, SubPath: sourceResolver.Spec.Source.SubPath, }, }, nil From 2a513895bbab83212308840da5bec284ed910d09 Mon Sep 17 00:00:00 2001 From: Bohan Chen Date: Fri, 17 May 2024 14:48:47 -0400 Subject: [PATCH 4/5] run codegen Signed-off-by: Bohan Chen --- api/openapi-spec/swagger.json | 6 ++++++ pkg/apis/build/v1alpha2/zz_generated.deepcopy.go | 7 +++++++ pkg/openapi/openapi_generated.go | 12 ++++++++++++ 3 files changed, 25 insertions(+) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 3ae8ccd54..f00bbfd64 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -5721,6 +5721,9 @@ "x-kubernetes-patch-merge-key": "type", "x-kubernetes-patch-strategy": "merge" }, + "latestAttestationImage": { + "type": "string" + }, "latestCacheImage": { "type": "string" }, @@ -6997,6 +7000,9 @@ "url" ], "properties": { + "auth": { + "type": "string" + }, "stripComponents": { "type": "integer", "format": "int64" diff --git a/pkg/apis/build/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/build/v1alpha2/zz_generated.deepcopy.go index 1411d8cf9..b7584c1bb 100644 --- a/pkg/apis/build/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/build/v1alpha2/zz_generated.deepcopy.go @@ -498,6 +498,13 @@ func (in *BuilderSpec) DeepCopyInto(out *BuilderSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.AdditionalLabels != nil { + in, out := &in.AdditionalLabels, &out.AdditionalLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/pkg/openapi/openapi_generated.go b/pkg/openapi/openapi_generated.go index 84c8d8f76..900f181e2 100644 --- a/pkg/openapi/openapi_generated.go +++ b/pkg/openapi/openapi_generated.go @@ -2396,6 +2396,12 @@ func schema_pkg_apis_build_v1alpha2_BuildStatus(ref common.ReferenceCallback) co Format: "", }, }, + "latestAttestationImage": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, "podName": { SchemaProps: spec.SchemaProps{ Type: []string{"string"}, @@ -4783,6 +4789,12 @@ func schema_pkg_apis_core_v1alpha1_Blob(ref common.ReferenceCallback) common.Ope Format: "int64", }, }, + "auth": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"url"}, }, From 9d9bc14e52be32f409ee00fd3e292f781b60cea4 Mon Sep 17 00:00:00 2001 From: Bohan Chen Date: Fri, 17 May 2024 15:22:30 -0400 Subject: [PATCH 5/5] add docs for blob auth Signed-off-by: Bohan Chen --- docs/build.md | 4 ++++ docs/image.md | 5 +++++ docs/secrets.md | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/docs/build.md b/docs/build.md index 845cd4bd7..389c28aae 100644 --- a/docs/build.md +++ b/docs/build.md @@ -95,6 +95,10 @@ The `source` field is a composition of a source code location and a `subpath`. I - `git`: (Source Code is a git repository) - `url`: The git repository url. Both https and ssh formats are supported; with ssh format requiring a [ssh secret](secrets.md#git-secrets). - `revision`: The git revision to use. This value may be a commit sha, branch name, or tag. + - `auth`: Optional auth to use with blob source. Leave empty for no auth, "secret" for providing auth [via Secret](secrets.md#blob-secrets), or "helper" to use service account IAM (specific to each IaaS). + > Note: Only [Microsoft Azure](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview) + > and [Google Cloud Platform](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#kubernetes-sa-to-iam) + > helpers are currently implemented, contributions are welcome to `pkg/blob/_keychain.go`. - `subPath`: A subdirectory within the source folder where application code resides. Can be ignored if the source code resides at the `root` level. * Blob diff --git a/docs/image.md b/docs/image.md index a4e9e78e0..5e77c0725 100644 --- a/docs/image.md +++ b/docs/image.md @@ -99,11 +99,16 @@ The `source` field is a composition of a source code location and a `subpath`. I blob: url: "" stripComponents: 0 + auth: "" | "secret" | "helper" subPath: "" ``` - `blob`: (Source Code is a blob/jar in a blobstore) - `url`: The URL of the source code blob. This blob needs to either be publicly accessible or have the access token in the URL - `stripComponents`: Optional number of directory components to strip from the blobs content when extracting. + - `auth`: Optional auth to use with blob source. Leave empty for no auth, "secret" for providing auth [via Secret](secrets.md#blob-secrets), or "helper" to use service account IAM (specific to each IaaS). + > Note: Only [Microsoft Azure](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview) + > and [Google Cloud Platform](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#kubernetes-sa-to-iam) + > helpers are currently implemented, contributions are welcome to `pkg/blob/_keychain.go`. - `subPath`: A subdirectory within the source folder where application code resides. Can be ignored if the source code resides at the `root` level. * Registry diff --git a/docs/secrets.md b/docs/secrets.md index 151e9a51d..637eea63b 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -126,6 +126,26 @@ stringData: password: ``` +### Blob Secrets + +Secrets are used with a `kpack.io/blob` annotation that references a hostname for a blob location. Only one of username/password, bearer, or authorization is allowed. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: blob-secret + annotations: + kpack.io/blob: my-blob-store.com +stringData: + username: + password: + + bearer: + + authorization: +``` + ### Service Account To use these secrets with kpack create a service account and reference the service account in image and build resources. When configuring the image resource, reference the `name` of your registry credential and the `name` of your git credential.