From a7ff829773aa6f9e5abcdb4b46a9aa7c15b1a122 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Fri, 22 Dec 2017 11:04:58 +0000 Subject: [PATCH 1/4] s3.go: Added options to use paths with S3 and the ability to disable SSL (for testing or other endpoints) - fixes #3728. --- physical/s3/s3.go | 14 ++++++++++++-- .../source/docs/configuration/storage/s3.html.md | 8 +++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/physical/s3/s3.go b/physical/s3/s3.go index 7118e7da14bd..052a683d5950 100644 --- a/physical/s3/s3.go +++ b/physical/s3/s3.go @@ -72,6 +72,14 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, } } } + s3ForceStylePath, ok := conf['s3_force_style_path'] + if !ok { + s3ForceStylePath = false + } + disableSSL, ok := conf['disable_ssl'] + if !ok { + disableSSL = false + } credsConfig := &awsutil.CredentialsConfig{ AccessKey: accessKey, @@ -91,8 +99,10 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, HTTPClient: &http.Client{ Transport: pooledTransport, }, - Endpoint: aws.String(endpoint), - Region: aws.String(region), + Endpoint: aws.String(endpoint), + Region: aws.String(region), + S3ForcePathStyle: aws.Bool(s3ForceStylePath), + DisableSSL: aws.Bool(disableSSL), })) _, err = s3conn.ListObjects(&s3.ListObjectsInput{Bucket: &bucket}) diff --git a/website/source/docs/configuration/storage/s3.html.md b/website/source/docs/configuration/storage/s3.html.md index d18507e35af3..ca02cd58ef05 100644 --- a/website/source/docs/configuration/storage/s3.html.md +++ b/website/source/docs/configuration/storage/s3.html.md @@ -58,9 +58,15 @@ cause Vault to attempt to retrieve credentials from the AWS metadata service. - `session_token` `(string: "")` – Specifies the AWS session token. This can also be provided via the environment variable `AWS_SESSION_TOKEN`. -- `max_parallel` `(string: "128")` – Specifies The maximum number of concurrent +- `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent requests to S3. +- `s3_force_style_path` `(bool: false)` - Specifies whether to use host bucket + style domains with the configured endpoint. + +- `disable_ssl` `(bool: false)` - Specifies if SSL should be used for the + endpoint connection (highly recommended not to disable for production). + ## `s3` Examples ### Default Example From f8a3a7791d233d001936a95333c9611e8e425a01 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Fri, 22 Dec 2017 11:38:19 +0000 Subject: [PATCH 2/4] s3.go: Convert new string options to booleans for AWS config. --- physical/s3/s3.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/physical/s3/s3.go b/physical/s3/s3.go index 052a683d5950..07ef43cee8c1 100644 --- a/physical/s3/s3.go +++ b/physical/s3/s3.go @@ -72,13 +72,21 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, } } } - s3ForceStylePath, ok := conf['s3_force_style_path'] + s3ForceStylePathStr, ok := conf["s3_force_style_path"] if !ok { - s3ForceStylePath = false + s3ForceStylePathStr = "false" } - disableSSL, ok := conf['disable_ssl'] + s3ForceStylePathBool, err := strconv.ParseBool(s3ForceStylePathStr) + if err != nil { + return nil, fmt.Errorf("invalid boolean set for s3_force_style_path: '%s'", s3ForceStylePathStr) + } + disableSSLStr, ok := conf["disable_ssl"] if !ok { - disableSSL = false + disableSSLStr = "false" + } + disableSSLBool, err := strconv.ParseBool(disableSSLStr) + if err != nil { + return nil, fmt.Errorf("invalid boolean set for disable_ssl: '%s'", disableSSLStr) } credsConfig := &awsutil.CredentialsConfig{ @@ -101,8 +109,8 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, }, Endpoint: aws.String(endpoint), Region: aws.String(region), - S3ForcePathStyle: aws.Bool(s3ForceStylePath), - DisableSSL: aws.Bool(disableSSL), + S3ForcePathStyle: aws.Bool(s3ForceStylePathBool), + DisableSSL: aws.Bool(disableSSLBool), })) _, err = s3conn.ListObjects(&s3.ListObjectsInput{Bucket: &bucket}) From fe7422c1509d580a57710c132fdf56251bb18f9e Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Fri, 22 Dec 2017 14:03:32 +0000 Subject: [PATCH 3/4] s3 docs: Updated to match new string-based options. --- website/source/docs/configuration/storage/s3.html.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/source/docs/configuration/storage/s3.html.md b/website/source/docs/configuration/storage/s3.html.md index ca02cd58ef05..3d9fda2ba6b6 100644 --- a/website/source/docs/configuration/storage/s3.html.md +++ b/website/source/docs/configuration/storage/s3.html.md @@ -61,10 +61,10 @@ cause Vault to attempt to retrieve credentials from the AWS metadata service. - `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent requests to S3. -- `s3_force_style_path` `(bool: false)` - Specifies whether to use host bucket - style domains with the configured endpoint. +- `s3_force_style_path` `(string: "false")` - Specifies whether to use host + bucket style domains with the configured endpoint. -- `disable_ssl` `(bool: false)` - Specifies if SSL should be used for the +- `disable_ssl` `(string: "false")` - Specifies if SSL should be used for the endpoint connection (highly recommended not to disable for production). ## `s3` Examples From 4ead2f56e3632d2bc00117f469993607ad94961d Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 26 Dec 2017 18:56:12 +0000 Subject: [PATCH 4/4] s3.go: Corrections per PR review. --- physical/s3/s3.go | 13 +++++++------ .../source/docs/configuration/storage/s3.html.md | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/physical/s3/s3.go b/physical/s3/s3.go index 07ef43cee8c1..5adae1aca21f 100644 --- a/physical/s3/s3.go +++ b/physical/s3/s3.go @@ -22,6 +22,7 @@ import ( cleanhttp "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/vault/helper/awsutil" "github.com/hashicorp/vault/helper/consts" + "github.com/hashicorp/vault/helper/parseutil" "github.com/hashicorp/vault/physical" ) @@ -72,19 +73,19 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, } } } - s3ForceStylePathStr, ok := conf["s3_force_style_path"] + s3ForcePathStyleStr, ok := conf["s3_force_path_style"] if !ok { - s3ForceStylePathStr = "false" + s3ForcePathStyleStr = "false" } - s3ForceStylePathBool, err := strconv.ParseBool(s3ForceStylePathStr) + s3ForcePathStyleBool, err := parseutil.ParseBool(s3ForcePathStyleStr) if err != nil { - return nil, fmt.Errorf("invalid boolean set for s3_force_style_path: '%s'", s3ForceStylePathStr) + return nil, fmt.Errorf("invalid boolean set for s3_force_path_style: '%s'", s3ForcePathStyleStr) } disableSSLStr, ok := conf["disable_ssl"] if !ok { disableSSLStr = "false" } - disableSSLBool, err := strconv.ParseBool(disableSSLStr) + disableSSLBool, err := parseutil.ParseBool(disableSSLStr) if err != nil { return nil, fmt.Errorf("invalid boolean set for disable_ssl: '%s'", disableSSLStr) } @@ -109,7 +110,7 @@ func NewS3Backend(conf map[string]string, logger log.Logger) (physical.Backend, }, Endpoint: aws.String(endpoint), Region: aws.String(region), - S3ForcePathStyle: aws.Bool(s3ForceStylePathBool), + S3ForcePathStyle: aws.Bool(s3ForcePathStyleBool), DisableSSL: aws.Bool(disableSSLBool), })) diff --git a/website/source/docs/configuration/storage/s3.html.md b/website/source/docs/configuration/storage/s3.html.md index 3d9fda2ba6b6..247b1fe973d1 100644 --- a/website/source/docs/configuration/storage/s3.html.md +++ b/website/source/docs/configuration/storage/s3.html.md @@ -61,7 +61,7 @@ cause Vault to attempt to retrieve credentials from the AWS metadata service. - `max_parallel` `(string: "128")` – Specifies the maximum number of concurrent requests to S3. -- `s3_force_style_path` `(string: "false")` - Specifies whether to use host +- `s3_force_path_style` `(string: "false")` - Specifies whether to use host bucket style domains with the configured endpoint. - `disable_ssl` `(string: "false")` - Specifies if SSL should be used for the