Merge pull request #46 from ahouene/minio-k8s-credentials

S3 Backend: load keys from file with automatic reload

Author: wojas

backends/s3/credentials.go

@@ -0,0 +1,51 @@
+package s3
+
+import (
+	"os"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// FileSecretsCredentials is an implementation of Minio's credentials.Provider,
+// allowing to read credentials from Kubernetes or Docker secrets, as described in
+// https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure
+// and https://docs.docker.com/engine/swarm/secrets.
+type FileSecretsCredentials struct {
+	credentials.Expiry
+
+	// Path to the file containing the access key,
+	// e.g. /etc/s3-secrets/access-key.
+	AccessKeyFile string
+
+	// Path to the file containing the secret key,
+	// e.g. /etc/s3-secrets/secret-key.
+	SecretKeyFile string
+
+	// Time between each secrets retrieval.
+	RefreshInterval time.Duration
+}
+
+// Retrieve implements credentials.Provider.
+// It reads files pointed to by p.AccessKeyFilename and p.SecretKeyFilename.
+func (c *FileSecretsCredentials) Retrieve() (credentials.Value, error) {
+	keyId, err := os.ReadFile(c.AccessKeyFile)
+	if err != nil {
+		return credentials.Value{}, err
+	}
+	secretKey, err := os.ReadFile(c.SecretKeyFile)
+	if err != nil {
+		return credentials.Value{}, err
+	}
+
+	creds := credentials.Value{
+		AccessKeyID:     string(keyId),
+		SecretAccessKey: string(secretKey),
+	}
+
+	c.SetExpiration(time.Now().Add(c.RefreshInterval), -1)
+
+	return creds, err
+}
+
+var _ credentials.Provider = new(FileSecretsCredentials)

backends/s3/credentials_test.go

@@ -0,0 +1,167 @@
+package s3_test
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/PowerDNS/simpleblob/backends/s3"
+	"github.com/PowerDNS/simpleblob/backends/s3/s3testing"
+	"github.com/PowerDNS/simpleblob/tester"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+func TestFileSecretsCredentials(t *testing.T) {
+	tempDir := t.TempDir()
+
+	access, secret := secretsPaths(tempDir)
+
+	// Instanciate provider (what we're testing).
+	provider := &s3.FileSecretsCredentials{
+		AccessKeyFile: access,
+		SecretKeyFile: secret,
+	}
+
+	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	// Create server
+	addr, stop, err := s3testing.ServeMinio(ctx, tempDir)
+	if errors.Is(err, s3testing.ErrMinioNotFound) {
+		t.Skip("minio binary not found locally, make sure it is in PATH")
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = stop() }()
+
+	// Create minio client, using our provider.
+	creds := credentials.New(provider)
+	clt, err := minio.New(addr, &minio.Options{
+		Creds:  creds,
+		Region: "us-east-1",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertClientSuccess := func(want bool, when string) {
+		_, err = clt.BucketExists(ctx, "doesnotmatter")
+		s := "fail"
+		if want {
+			s = "succeed"
+		}
+		ok := (err == nil) == want
+		if !ok {
+			t.Fatalf("expected call to %s %s", s, when)
+		}
+	}
+
+	// First credential files creation.
+	// Keep them empty for now,
+	// so that calls to the server will fail.
+	writeSecrets(t, tempDir, "")
+
+	// The files do not hold the right values,
+	// so a call to the server should fail.
+	assertClientSuccess(false, "just after init")
+
+	// Write the right keys to the files.
+	// We're not testing expiry here,
+	// and forcing credentials cache to update.
+	writeSecrets(t, tempDir, s3testing.AdminUserOrPassword)
+	creds.Expire()
+	assertClientSuccess(true, "after changing files content")
+
+	// Change content of the files.
+	writeSecrets(t, tempDir, "badcredentials")
+	creds.Expire()
+	assertClientSuccess(false, "after changing again, to bad credentials")
+}
+
+func TestBackendWithSecrets(t *testing.T) {
+	tempDir := t.TempDir()
+
+	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	addr, stop, err := s3testing.ServeMinio(ctx, tempDir)
+	if errors.Is(err, s3testing.ErrMinioNotFound) {
+		t.Skip("minio binary not found locally, make sure it is in PATH")
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = stop() }()
+
+	// Prepare backend options to reuse.
+	// These will not change.
+	access, secret := secretsPaths(tempDir)
+	opt := s3.Options{
+		AccessKeyFile: access,
+		SecretKeyFile: secret,
+		Region:        "us-east-1",
+		Bucket:        "test-bucket",
+		CreateBucket:  true,
+		EndpointURL:   "http://" + addr,
+	}
+
+	// Backend should not start if secrets files do not exist.
+	_, err = s3.New(ctx, opt)
+	if !errors.Is(err, os.ErrNotExist) {
+		t.Fatal("backend should not start without credentials")
+	}
+
+	// Now write files, but with bad content.
+	writeSecrets(t, tempDir, "")
+	_, err = s3.New(ctx, opt)
+	if err == nil || err.Error() != "Access Denied." {
+		t.Fatal("backend should not start with bad credentials")
+	}
+
+	// Write the good content.
+	// Now the backend should start and be able to perform a request.
+	writeSecrets(t, tempDir, s3testing.AdminUserOrPassword)
+
+	backend, err := s3.New(ctx, opt)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = backend.List(ctx, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Finally, the whole test suite should succeed.
+	tester.DoBackendTests(t, backend)
+}
+
+// secretsPaths returns the file paths for the access key
+// and the secret key, respectively.
+// For a same dir, the returned values will always be the same.
+func secretsPaths(dir string) (access, secret string) {
+	access = filepath.Join(dir, "access-key")
+	secret = filepath.Join(dir, "secret-key")
+	return
+}
+
+// writeSecrets writes content to files called "access-key" and "secret-key"
+// in dir.
+// It returns
+func writeSecrets(t testing.TB, dir, content string) {
+	access, secret := secretsPaths(dir)
+	err := os.WriteFile(access, []byte(content), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.WriteFile(secret, []byte(content), 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

backends/s3/s3.go

@@ -37,6 +37,9 @@ const (
 	// DefaultUpdateMarkerForceListInterval is the default value for
 	// UpdateMarkerForceListInterval.
 	DefaultUpdateMarkerForceListInterval = 5 * time.Minute
+	// DefaultSecretsRefreshInterval is the default value for RefreshSecrets.
+	// It should not be too high so as to retrieve secrets regularly.
+	DefaultSecretsRefreshInterval = 15 * time.Second
 )
 
 // Options describes the storage options for the S3 backend
@@ -45,6 +48,22 @@ type Options struct {
 	AccessKey string `yaml:"access_key"`
 	SecretKey string `yaml:"secret_key"`
 
+	// Path to the file containing the access key
+	// as an alternative to AccessKey and SecretKey,
+	// e.g. /etc/s3-secrets/access-key.
+	AccessKeyFile string `yaml:"access_key_file"`
+
+	// Path to the file containing the secret key
+	// as an alternative to AccessKey and SecretKey,
+	// e.g. /etc/s3-secrets/secret-key.
+	SecretKeyFile string `yaml:"secret_key_file"`
+
+	// Time between each secrets retrieval.
+	// Minimum is 1s, lower values are considered an error.
+	// It defaults to DefaultSecretsRefreshInterval,
+	// which is currently 15s.
+	SecretsRefreshInterval time.Duration `yaml:"secrets_refresh_interval"`
+
 	// Region defaults to "us-east-1", which also works for Minio
 	Region string `yaml:"region"`
 	Bucket string `yaml:"bucket"`
@@ -93,11 +112,13 @@ type Options struct {
 }
 
 func (o Options) Check() error {
-	if o.AccessKey == "" {
-		return fmt.Errorf("s3 storage.options: access_key is required")
+	hasSecretsCreds := o.AccessKeyFile != "" && o.SecretKeyFile != ""
+	hasStaticCreds := o.AccessKey != "" && o.SecretKey != ""
+	if !hasSecretsCreds && !hasStaticCreds {
+		return fmt.Errorf("s3 storage.options: credentials are required, fill either (access_key and secret_key) or (access_key_filename and secret_key_filename)")
 	}
-	if o.SecretKey == "" {
-		return fmt.Errorf("s3 storage.options: secret_key is required")
+	if hasSecretsCreds && o.SecretsRefreshInterval < time.Second {
+		return fmt.Errorf("s3 storage.options: field secrets_refresh_interval must be at least 1s")
 	}
 	if o.Bucket == "" {
 		return fmt.Errorf("s3 storage.options: bucket is required")
@@ -289,6 +310,9 @@ func New(ctx context.Context, opt Options) (*Backend, error) {
 	if opt.EndpointURL == "" {
 		opt.EndpointURL = DefaultEndpointURL
 	}
+	if opt.SecretsRefreshInterval == 0 {
+		opt.SecretsRefreshInterval = DefaultSecretsRefreshInterval
+	}
 	if err := opt.Check(); err != nil {
 		return nil, err
 	}
@@ -342,8 +366,17 @@ func New(ctx context.Context, opt Options) (*Backend, error) {
 		return nil, fmt.Errorf("unsupported scheme for S3: '%s', use http or https.", u.Scheme)
 	}
 
+	creds := credentials.NewStaticV4(opt.AccessKey, opt.SecretKey, "")
+	if opt.AccessKeyFile != "" {
+		creds = credentials.New(&FileSecretsCredentials{
+			AccessKeyFile:   opt.AccessKeyFile,
+			SecretKeyFile:   opt.SecretKeyFile,
+			RefreshInterval: opt.SecretsRefreshInterval,
+		})
+	}
+
 	cfg := &minio.Options{
-		Creds:     credentials.NewStaticV4(opt.AccessKey, opt.SecretKey, ""),
+		Creds:     creds,
 		Secure:    useSSL,
 		Transport: hc.Transport,
 		Region:    opt.Region,
@@ -404,7 +437,7 @@ func convertMinioError(err error, isList bool) error {
 	}
 	errRes := minio.ToErrorResponse(err)
 	if !isList && errRes.StatusCode == 404 {
-    		return fmt.Errorf("%w: %s", os.ErrNotExist, err.Error())
+		return fmt.Errorf("%w: %s", os.ErrNotExist, err.Error())
 	}
 	if errRes.Code == "BucketAlreadyOwnedByYou" {
 		return nil

backends/s3/s3testing/minio.go

@@ -0,0 +1,73 @@
+package s3testing
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"time"
+)
+
+const (
+	AdminUserOrPassword = "simpleblob"
+)
+
+var ErrMinioNotFound = exec.ErrNotFound
+
+// ServeMinio starts a Minio server in the background,
+// and waits for it to be ready.
+// It returns its address,
+// and a function to stop the server gracefully.
+//
+// The admin username and password for the server are both "simpleblob".
+//
+// If the minio binary cannot be found in PATH,
+// ErrMinioNotFound will be returned.
+func ServeMinio(ctx context.Context, dir string) (string, func() error, error) {
+	cmdname, err := exec.LookPath("minio")
+	if err != nil {
+		return "", nil, err
+	}
+
+	port, err := FreePort()
+	if err != nil {
+		return "", nil, err
+	}
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
+
+	cmd := exec.CommandContext(ctx, cmdname, "server", "--quiet", "--address", addr, dir)
+	cmd.Env = append([]string{}, os.Environ()...)
+	cmd.Env = append(cmd.Env, "MINIO_BROWSER=off")
+	cmd.Env = append(cmd.Env, "MINIO_ROOT_USER="+AdminUserOrPassword, "MINIO_ROOT_PASSWORD="+AdminUserOrPassword)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		return "", nil, err
+	}
+
+	// Wait for server to accept requests.
+	readyURL := "http://" + addr + "/minio/health/ready"
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	ticker := time.NewTicker(30 * time.Millisecond)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return "", nil, ctx.Err()
+		case <-ticker.C:
+		}
+
+		resp, err := http.Get(readyURL)
+		if err == nil && resp.StatusCode == 200 {
+			break
+		}
+	}
+
+	stop := func() error {
+		_ = cmd.Process.Signal(os.Interrupt)
+		return cmd.Wait()
+	}
+	return addr, stop, nil
+}

backends/s3/s3testing/port.go

@@ -0,0 +1,14 @@
+package s3testing
+
+import "net"
+
+// FreePort returns a port number free for use.
+func FreePort() (int, error) {
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return 0, err
+	}
+	defer l.Close()
+	addr := l.Addr().(*net.TCPAddr)
+	return addr.Port, nil
+}