From 7fe1f2cc0ad77d5cd2ee45dc808ba2094baf15c5 Mon Sep 17 00:00:00 2001 From: Havrileck Alexandre Date: Tue, 18 May 2021 22:38:57 +0200 Subject: [PATCH] feat(bucket): Add templating on PUT metadata and storage class --- conf/config-example.yaml | 4 +- docs/configuration/example.md | 4 +- docs/configuration/structure.md | 10 +- docs/feature-guide/templates.md | 37 ++- pkg/s3-proxy/bucket/client.go | 8 + pkg/s3-proxy/bucket/requestContext.go | 58 ++++- pkg/s3-proxy/bucket/requestContext_test.go | 271 +++++++++++++++++++++ pkg/s3-proxy/response-handler/utils.go | 6 +- pkg/s3-proxy/utils/constants.go | 6 + 9 files changed, 389 insertions(+), 15 deletions(-) create mode 100644 pkg/s3-proxy/utils/constants.go diff --git a/conf/config-example.yaml b/conf/config-example.yaml index a35c4cda..5ab95004 100644 --- a/conf/config-example.yaml +++ b/conf/config-example.yaml @@ -232,11 +232,13 @@ targets: # enabled: true # # Configuration for PUT requests # config: - # # Metadata key/values that will be put on S3 objects + # # Metadata key/values that will be put on S3 objects. + # # Values can be templated. Empty values will be flushed. # metadata: # key: value # # Storage class that will be used for uploaded objects # # See storage class here: https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html + # # Values can be templated. Empty values will be flushed. # storageClass: STANDARD # GLACIER, ... # # Will allow override objects if enabled # allowOverride: false diff --git a/docs/configuration/example.md b/docs/configuration/example.md index b242046b..0e767adf 100644 --- a/docs/configuration/example.md +++ b/docs/configuration/example.md @@ -242,11 +242,13 @@ targets: # enabled: true # # Configuration for PUT requests # config: - # # Metadata key/values that will be put on S3 objects + # # Metadata key/values that will be put on S3 objects. + # # Values can be templated. Empty values will be flushed. # metadata: # key: value # # Storage class that will be used for uploaded objects # # See storage class here: https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html + # # Values can be templated. Empty values will be flushed. # storageClass: STANDARD # GLACIER, ... # # Will allow override objects if enabled # allowOverride: false diff --git a/docs/configuration/structure.md b/docs/configuration/structure.md index 6b3d2815..c9f92c97 100644 --- a/docs/configuration/structure.md +++ b/docs/configuration/structure.md @@ -178,11 +178,11 @@ See more information [here](../feature-guide/key-rewrite.md). ## PutActionConfigConfiguration -| Key | Type | Required | Default | Description | -| ------------- | ----------------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| metadata | Map[String]String | No | None | Metadata key/values that will be put on S3 objects | -| storageClass | String | No | `""` | Storage class that will be used for uploaded objects. See storage class here: [https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) | -| allowOverride | Boolean | No | `false` | Will allow override objects if enabled | +| Key | Type | Required | Default | Description | +| ------------- | ----------------- | -------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| metadata | Map[String]String | No | None | Metadata key/values that will be put on S3 objects. Map Values can be templated. Empty values will be flushed. See [here](../feature-guide/templates.md#put-metadata) | +| storageClass | String | No | `""` | Storage class that will be used for uploaded objects. See storage class here: [https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html). Value can be templated. Empty values will be flushed. See [here](../feature-guide/templates.md#put-storage-class) | +| allowOverride | Boolean | No | `false` | Will allow override objects if enabled | ## DeleteActionConfiguration diff --git a/docs/feature-guide/templates.md b/docs/feature-guide/templates.md index f25da050..62c0ffab 100644 --- a/docs/feature-guide/templates.md +++ b/docs/feature-guide/templates.md @@ -123,11 +123,37 @@ Available data: | Request | [http.Request](https://golang.org/pkg/net/http/#Request) | HTTP Request object from golang | | Error | Error | Error raised and caught | +## PUT Metadata and Storage class + +### PUT Metadata + +This case will be used for all PUT metadata templates. + +Available data: + +| Name | Type | Description | +| ----- | --------------------------- | ------------------------------------------------- | +| User | [GenericUser](#genericuser) | Authenticated user if present in incoming request | +| Input | [PutInput](#putinput) | PutInput structure data | +| Key | String | The final S3 key generated for upload request | + +### PUT Storage class + +This case will be used for all PUT storage class templates. + +Available data: + +| Name | Type | Description | +| ----- | --------------------------- | ------------------------------------------------- | +| User | [GenericUser](#genericuser) | Authenticated user if present in incoming request | +| Input | [PutInput](#putinput) | PutInput structure data | +| Key | String | The final S3 key generated for upload request | + ## Headers templates and structures ### Generic case -This case is the main one and used for all templates rendered explained before. +This case is the main one and used for all header templates rendered explained before. The following table will show the data structure available for the header template rendering: @@ -195,3 +221,12 @@ These are the properties available: | ETag | String | ETag value from S3 | | LastModified | [Time](https://golang.org/pkg/time/#Time) | Last modified value from S3 | | Metadata  | Map[String]String | Metadata value from S3 | + +### PutInput + +| Name | Type | Description | +| ----------- | ------- | ---------------------------- | +| RequestPath | String | Request path | +| Filename | String | Filename used for upload | +| ContentType | String | File content type for upload | +| ContentSize | Integer | File content size for upload | diff --git a/pkg/s3-proxy/bucket/client.go b/pkg/s3-proxy/bucket/client.go index 8aeff1b5..8145aee6 100644 --- a/pkg/s3-proxy/bucket/client.go +++ b/pkg/s3-proxy/bucket/client.go @@ -6,6 +6,7 @@ import ( "io" "time" + "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/authx/models" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/s3client" ) @@ -45,6 +46,13 @@ type PutInput struct { ContentSize int64 } +// PutData Put Data represents a put data structure used in put templates rendering. +type PutData struct { + User models.GenericUser + Input *PutInput + Key string +} + // NewClient will generate a new client to do GET,PUT or DELETE actions. func NewClient( tgt *config.TargetConfig, diff --git a/pkg/s3-proxy/bucket/requestContext.go b/pkg/s3-proxy/bucket/requestContext.go index f767bd5d..b0e63b4e 100644 --- a/pkg/s3-proxy/bucket/requestContext.go +++ b/pkg/s3-proxy/bucket/requestContext.go @@ -8,9 +8,11 @@ import ( "path" "strings" + "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/authx/models" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/config" responsehandler "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/response-handler" "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/s3client" + "github.com/oxyno-zeta/s3-proxy/pkg/s3-proxy/utils" ) // requestContext Bucket request context. @@ -218,12 +220,64 @@ func (rctx *requestContext) Put(ctx context.Context, inp *PutInput) { rctx.targetCfg.Actions.PUT.Config != nil { // Check if metadata is configured in target configuration if rctx.targetCfg.Actions.PUT.Config.Metadata != nil { - input.Metadata = rctx.targetCfg.Actions.PUT.Config.Metadata + // Store templated data + metadata := map[string]string{} + + // Render templates + for k, v := range rctx.targetCfg.Actions.PUT.Config.Metadata { + // Execute template + buf, err := utils.ExecuteTemplate(v, &PutData{ + User: models.GetAuthenticatedUserFromContext(ctx), + Input: inp, + Key: key, + }) + // Check error + if err != nil { + resHan.InternalServerError(rctx.LoadFileContent, err) + + return + } + + // Store value + val := buf.String() + // Remove all new lines + val = utils.NewLineMatcherRegex.ReplaceAllString(val, "") + // Check if value is empty or not + if val != "" { + // Store + metadata[k] = val + } + } + + // Store all metadata + input.Metadata = metadata } // Check if storage class is present in target configuration if rctx.targetCfg.Actions.PUT.Config.StorageClass != "" { - input.StorageClass = rctx.targetCfg.Actions.PUT.Config.StorageClass + // Execute template + buf, err := utils.ExecuteTemplate(rctx.targetCfg.Actions.PUT.Config.StorageClass, &PutData{ + User: models.GetAuthenticatedUserFromContext(ctx), + Input: inp, + Key: key, + }) + + // Check error + if err != nil { + resHan.InternalServerError(rctx.LoadFileContent, err) + + return + } + + // Store value + val := buf.String() + // Remove all new lines + val = utils.NewLineMatcherRegex.ReplaceAllString(val, "") + // Check if value is empty or not + if val != "" { + // Store + input.StorageClass = val + } } // Check if allow override is enabled diff --git a/pkg/s3-proxy/bucket/requestContext_test.go b/pkg/s3-proxy/bucket/requestContext_test.go index ab0ce9bd..42581381 100644 --- a/pkg/s3-proxy/bucket/requestContext_test.go +++ b/pkg/s3-proxy/bucket/requestContext_test.go @@ -688,6 +688,277 @@ func Test_requestContext_Put(t *testing.T) { }, responseHandlerNoContentMockResultTimes: 1, }, + { + name: "should be ok to do templating on metadata", + fields: fields{ + targetCfg: &config.TargetConfig{ + Bucket: &config.BucketConfig{Prefix: "/"}, + Actions: &config.ActionsConfig{ + PUT: &config.PutActionConfig{ + Config: &config.PutActionConfigConfig{ + Metadata: map[string]string{ + "fixed": "fixed", + "testkey": "{{ .Key }}", + "tpl": "{{ .Key }} - {{ .Input.ContentType }}", + }, + StorageClass: "storage-class", + AllowOverride: true, + }, + }, + }, + }, + mountPath: "/mount", + }, + args: args{ + inp: &PutInput{ + RequestPath: "/test", + Filename: "file", + Body: nil, + ContentType: "content-type", + }, + }, + s3ClientHeadObjectMockResult: s3ClientHeadObjectMockResult{ + times: 0, + }, + s3ClientPutObjectMockResult: s3ClientPutObjectMockResult{ + input2: &s3client.PutInput{ + Key: "/test/file", + ContentType: "content-type", + Metadata: map[string]string{ + "fixed": "fixed", + "testkey": "/test/file", + "tpl": "/test/file - content-type", + }, + StorageClass: "storage-class", + }, + times: 1, + }, + responseHandlerNoContentMockResultTimes: 1, + }, + { + name: "should be ok to flush empty metadata", + fields: fields{ + targetCfg: &config.TargetConfig{ + Bucket: &config.BucketConfig{Prefix: "/"}, + Actions: &config.ActionsConfig{ + PUT: &config.PutActionConfig{ + Config: &config.PutActionConfigConfig{ + Metadata: map[string]string{ + "fixed": "fixed", + "testkey": "", + }, + StorageClass: "storage-class", + AllowOverride: true, + }, + }, + }, + }, + mountPath: "/mount", + }, + args: args{ + inp: &PutInput{ + RequestPath: "/test", + Filename: "file", + Body: nil, + ContentType: "content-type", + }, + }, + s3ClientHeadObjectMockResult: s3ClientHeadObjectMockResult{ + times: 0, + }, + s3ClientPutObjectMockResult: s3ClientPutObjectMockResult{ + input2: &s3client.PutInput{ + Key: "/test/file", + ContentType: "content-type", + Metadata: map[string]string{ + "fixed": "fixed", + }, + StorageClass: "storage-class", + }, + times: 1, + }, + responseHandlerNoContentMockResultTimes: 1, + }, + { + name: "should be ok to do templating on metadata and remove new lines", + fields: fields{ + targetCfg: &config.TargetConfig{ + Bucket: &config.BucketConfig{Prefix: "/"}, + Actions: &config.ActionsConfig{ + PUT: &config.PutActionConfig{ + Config: &config.PutActionConfigConfig{ + Metadata: map[string]string{ + "fixed": "fixed", + "testkey": "{{ .Key }}", + "tpl": ` +{{ .Key }} - {{ .Input.ContentType }} +`, + }, + StorageClass: "storage-class", + AllowOverride: true, + }, + }, + }, + }, + mountPath: "/mount", + }, + args: args{ + inp: &PutInput{ + RequestPath: "/test", + Filename: "file", + Body: nil, + ContentType: "content-type", + }, + }, + s3ClientHeadObjectMockResult: s3ClientHeadObjectMockResult{ + times: 0, + }, + s3ClientPutObjectMockResult: s3ClientPutObjectMockResult{ + input2: &s3client.PutInput{ + Key: "/test/file", + ContentType: "content-type", + Metadata: map[string]string{ + "fixed": "fixed", + "testkey": "/test/file", + "tpl": "/test/file - content-type", + }, + StorageClass: "storage-class", + }, + times: 1, + }, + responseHandlerNoContentMockResultTimes: 1, + }, + { + name: "should be ok to do templating on storage class", + fields: fields{ + targetCfg: &config.TargetConfig{ + Bucket: &config.BucketConfig{Prefix: "/"}, + Actions: &config.ActionsConfig{ + PUT: &config.PutActionConfig{ + Config: &config.PutActionConfigConfig{ + Metadata: map[string]string{ + "fixed": "fixed", + }, + StorageClass: "{{ .Key }} - {{ .Input.ContentType }}", + AllowOverride: true, + }, + }, + }, + }, + mountPath: "/mount", + }, + args: args{ + inp: &PutInput{ + RequestPath: "/test", + Filename: "file", + Body: nil, + ContentType: "content-type", + }, + }, + s3ClientHeadObjectMockResult: s3ClientHeadObjectMockResult{ + times: 0, + }, + s3ClientPutObjectMockResult: s3ClientPutObjectMockResult{ + input2: &s3client.PutInput{ + Key: "/test/file", + ContentType: "content-type", + Metadata: map[string]string{ + "fixed": "fixed", + }, + StorageClass: "/test/file - content-type", + }, + times: 1, + }, + responseHandlerNoContentMockResultTimes: 1, + }, + { + name: "should be ok to flush storage class", + fields: fields{ + targetCfg: &config.TargetConfig{ + Bucket: &config.BucketConfig{Prefix: "/"}, + Actions: &config.ActionsConfig{ + PUT: &config.PutActionConfig{ + Config: &config.PutActionConfigConfig{ + Metadata: map[string]string{ + "fixed": "fixed", + }, + StorageClass: "", + AllowOverride: true, + }, + }, + }, + }, + mountPath: "/mount", + }, + args: args{ + inp: &PutInput{ + RequestPath: "/test", + Filename: "file", + Body: nil, + ContentType: "content-type", + }, + }, + s3ClientHeadObjectMockResult: s3ClientHeadObjectMockResult{ + times: 0, + }, + s3ClientPutObjectMockResult: s3ClientPutObjectMockResult{ + input2: &s3client.PutInput{ + Key: "/test/file", + ContentType: "content-type", + Metadata: map[string]string{ + "fixed": "fixed", + }, + StorageClass: "", + }, + times: 1, + }, + responseHandlerNoContentMockResultTimes: 1, + }, + { + name: "should be ok to do templating on storage class and remove new lines", + fields: fields{ + targetCfg: &config.TargetConfig{ + Bucket: &config.BucketConfig{Prefix: "/"}, + Actions: &config.ActionsConfig{ + PUT: &config.PutActionConfig{ + Config: &config.PutActionConfigConfig{ + Metadata: map[string]string{ + "fixed": "fixed", + }, + StorageClass: ` +{{ .Key }} - {{ .Input.ContentType }} +`, + AllowOverride: true, + }, + }, + }, + }, + mountPath: "/mount", + }, + args: args{ + inp: &PutInput{ + RequestPath: "/test", + Filename: "file", + Body: nil, + ContentType: "content-type", + }, + }, + s3ClientHeadObjectMockResult: s3ClientHeadObjectMockResult{ + times: 0, + }, + s3ClientPutObjectMockResult: s3ClientPutObjectMockResult{ + input2: &s3client.PutInput{ + Key: "/test/file", + ContentType: "content-type", + Metadata: map[string]string{ + "fixed": "fixed", + }, + StorageClass: "/test/file - content-type", + }, + times: 1, + }, + responseHandlerNoContentMockResultTimes: 1, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/s3-proxy/response-handler/utils.go b/pkg/s3-proxy/response-handler/utils.go index 6ea5e773..63725138 100644 --- a/pkg/s3-proxy/response-handler/utils.go +++ b/pkg/s3-proxy/response-handler/utils.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "net/http" "reflect" - "regexp" "strconv" "strings" "time" @@ -16,9 +15,6 @@ import ( ) func (h *handler) manageHeaders(helpersContent string, headersTpl map[string]string, hData interface{}) (map[string]string, error) { - // Create regex to remove all new lines - reg := regexp.MustCompile(`\r?\n`) - // Store result res := map[string]string{} @@ -35,7 +31,7 @@ func (h *handler) manageHeaders(helpersContent string, headersTpl map[string]str // Get string from buffer str := buf.String() // Remove all new lines - str = reg.ReplaceAllString(str, "") + str = utils.NewLineMatcherRegex.ReplaceAllString(str, "") // Save data only if the header isn't empty if str != "" { // Save diff --git a/pkg/s3-proxy/utils/constants.go b/pkg/s3-proxy/utils/constants.go new file mode 100644 index 00000000..e3e2f9dc --- /dev/null +++ b/pkg/s3-proxy/utils/constants.go @@ -0,0 +1,6 @@ +package utils + +import "regexp" + +// NewLineMatcherRegex Regex to remove all new lines. +var NewLineMatcherRegex = regexp.MustCompile(`\r?\n`)