From 48958e032470e9d972fbbaefdaa8da8eae1732ac Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 6 Mar 2018 23:12:23 +0100 Subject: [PATCH 1/2] start working on AWS v4 presign url validation --- package-lock.json | 168 +++++++++++++++++++++++++++++++++++++------ pkg/middleware/s3.go | 65 +++++++++++++++++ 2 files changed, 210 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index b7c2efe..66e93cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2,6 +2,7 @@ "name": "mort-integrations-tests", "version": "1.0.0", "lockfileVersion": 1, + "requires": true, "dependencies": { "assertion-error": { "version": "1.0.2", @@ -16,7 +17,19 @@ "aws-sdk": { "version": "2.151.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.151.0.tgz", - "integrity": "sha1-XnPXeLdlFPNhg7kUAEd3eKteWNg=" + "integrity": "sha1-XnPXeLdlFPNhg7kUAEd3eKteWNg=", + "requires": { + "buffer": "4.9.1", + "crypto-browserify": "1.0.9", + "events": "1.1.1", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.1.0", + "xml2js": "0.4.17", + "xmlbuilder": "4.2.1" + } }, "balanced-match": { "version": "1.0.0", @@ -31,7 +44,11 @@ "brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", - "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=" + "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", + "requires": { + "balanced-match": "1.0.0", + "concat-map": "0.0.1" + } }, "browser-stdout": { "version": "1.3.0", @@ -41,12 +58,25 @@ "buffer": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", - "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=" + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "1.2.1", + "ieee754": "1.1.8", + "isarray": "1.0.0" + } }, "chai": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", - "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=" + "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=", + "requires": { + "assertion-error": "1.0.2", + "check-error": "1.0.2", + "deep-eql": "3.0.1", + "get-func-name": "2.0.0", + "pathval": "1.1.0", + "type-detect": "4.0.5" + } }, "check-error": { "version": "1.0.2", @@ -56,7 +86,10 @@ "combined-stream": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", - "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=" + "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=", + "requires": { + "delayed-stream": "1.0.0" + } }, "commander": { "version": "2.11.0", @@ -91,12 +124,18 @@ "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==" + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } }, "deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", - "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==" + "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", + "requires": { + "type-detect": "4.0.5" + } }, "delayed-stream": { "version": "1.0.0", @@ -126,7 +165,12 @@ "form-data": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz", - "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=" + "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=", + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.5", + "mime-types": "2.1.17" + } }, "formidable": { "version": "1.1.1", @@ -146,7 +190,15 @@ "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==" + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "requires": { + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" + } }, "growl": { "version": "1.10.3", @@ -171,7 +223,11 @@ "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=" + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "1.4.0", + "wrappy": "1.0.2" + } }, "inherits": { "version": "2.0.3", @@ -211,12 +267,18 @@ "mime-types": { "version": "2.1.17", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz", - "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=" + "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=", + "requires": { + "mime-db": "1.30.0" + } }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==" + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "1.1.8" + } }, "minimist": { "version": "0.0.8", @@ -226,12 +288,27 @@ "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=" + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + } }, "mocha": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/mocha/-/mocha-4.0.1.tgz", - "integrity": "sha512-evDmhkoA+cBNiQQQdSKZa2b9+W2mpLoj50367lhy+Klnx9OV8XlCIhigUnn1gaTFLQCa0kdNhEGDr0hCXOQFDw==" + "integrity": "sha512-evDmhkoA+cBNiQQQdSKZa2b9+W2mpLoj50367lhy+Klnx9OV8XlCIhigUnn1gaTFLQCa0kdNhEGDr0hCXOQFDw==", + "requires": { + "browser-stdout": "1.3.0", + "commander": "2.11.0", + "debug": "3.1.0", + "diff": "3.3.1", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.3", + "he": "1.1.1", + "mkdirp": "0.5.1", + "supports-color": "4.4.0" + } }, "moment": { "version": "2.20.1", @@ -247,7 +324,10 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=" + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1.0.2" + } }, "path-is-absolute": { "version": "1.0.1", @@ -282,7 +362,16 @@ "readable-stream": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", - "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==" + "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "1.0.7", + "safe-buffer": "5.1.1", + "string_decoder": "1.0.3", + "util-deprecate": "1.0.2" + } }, "safe-buffer": { "version": "5.1.1", @@ -297,12 +386,27 @@ "string_decoder": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", - "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==" + "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "requires": { + "safe-buffer": "5.1.1" + } }, "superagent": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.1.tgz", - "integrity": "sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==" + "integrity": "sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==", + "requires": { + "component-emitter": "1.2.1", + "cookiejar": "2.1.1", + "debug": "3.1.0", + "extend": "3.0.1", + "form-data": "2.3.1", + "formidable": "1.1.1", + "methods": "1.1.2", + "mime": "1.4.1", + "qs": "6.5.1", + "readable-stream": "2.3.3" + } }, "superagent-binary-parser": { "version": "1.0.1", @@ -313,12 +417,19 @@ "supertest": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/supertest/-/supertest-3.0.0.tgz", - "integrity": "sha1-jUu2j9GDDuBwM7HFpamkAhyWUpY=" + "integrity": "sha1-jUu2j9GDDuBwM7HFpamkAhyWUpY=", + "requires": { + "methods": "1.1.2", + "superagent": "3.8.1" + } }, "supports-color": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", - "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==" + "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", + "requires": { + "has-flag": "2.0.0" + } }, "type-detect": { "version": "4.0.5", @@ -328,7 +439,11 @@ "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=" + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } }, "util-deprecate": { "version": "1.0.2", @@ -348,12 +463,19 @@ "xml2js": { "version": "0.4.17", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.17.tgz", - "integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=" + "integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=", + "requires": { + "sax": "1.2.1", + "xmlbuilder": "4.2.1" + } }, "xmlbuilder": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz", - "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=" + "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=", + "requires": { + "lodash": "4.17.4" + } } } } diff --git a/pkg/middleware/s3.go b/pkg/middleware/s3.go index b38efc1..c978f6c 100644 --- a/pkg/middleware/s3.go +++ b/pkg/middleware/s3.go @@ -103,6 +103,11 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { accessKey = matches[1] } + if req.URL.Query().Get("X-Amz-Signature") != "" { + s.authByQuery(resWriter, req, bucketName, next) + return + } + bucket, ok := mortConfig.Buckets[bucketName] if !ok { buckets := mortConfig.BucketsByAccessKey(accessKey) @@ -226,3 +231,63 @@ func (s *S3Auth) listAllMyBuckets(resWriter http.ResponseWriter, accessKey strin res.SetContentType("application/xml") res.Send(resWriter) } + +func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, bucketName string, next http.Handler) { + validationReq := *r + mortConfig := s.mortConfig + + validationReq.URL.Query().Del("X-Amz-Signature") + var credential awsauth.Credentials + accessKey := strings.Split(validationReq.URL.Query().Get("X-Amz-Credential"), "/")[0] + + bucket, ok := mortConfig.Buckets[bucketName] + if !ok { + buckets := mortConfig.BucketsByAccessKey(accessKey) + if len(buckets) == 0 { + monitoring.Log().Warn("S3Auth no bucket for access key") + res := response.NewString(403, "") + res.Send(resWriter) + return + } + + bucket = buckets[0] + } + + if r.URL.Query().Get("X-Amz-Credential") == "" { + res := response.NewString(401, "") + monitoring.Log().Warn("S3Auth invalid request no x-amz-credential in query string", zap.String("bucket", bucketName)) + res.Send(resWriter) + return + } + + + + keys := bucket.Keys + for _, key := range keys { + if accessKey == key.AccessKey { + credential.AccessKeyID = accessKey + credential.SecretAccessKey = key.SecretAccessKey + break + } + + } + + if credential.AccessKeyID == "" { + res := response.NewString(401, "") + monitoring.Log().Warn("S3Auth invalid bucket config no access key or invalid", zap.String("bucket", bucketName)) + res.Send(resWriter) + return + } + + awsauth.PreSign(&validationReq, "mort", "s3", strings.Split(validationReq.URL.Query().Get("X-Amz-SignedHeaders"), ","), credential) + + if validationReq.URL.Query().Get("X-Amz-Signature") == r.URL.Query().Get("X-Amz-Signature") { + next.ServeHTTP(resWriter, r) + return + } + + monitoring.Log().Warn("S3Auth signature mismatch", zap.String("req.path", r.URL.Path)) + response.NewNoContent(403).Send(resWriter) + return + +} \ No newline at end of file From 719dbbfbe3fc41163ac716d95fa403f78014b79d Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 7 Mar 2018 20:04:08 +0100 Subject: [PATCH 2/2] Added support for presign AWS v4 --- CHANGELOG.md | 4 + cmd/mort/mort.go | 2 +- package-lock.json | 350 ++++++++++++++++++++++++++++++++++++ package.json | 1 + pkg/middleware/s3.go | 23 ++- pkg/processor/processor.go | 11 +- scripts/run-tests.sh | 3 +- tests-int/S3Objects.Spec.js | 71 ++++++++ 8 files changed, 454 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2bc1ac..50a8cd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +* 0.8.0 + * Feature: Added support for AWS presigned url + * Bugfix: Fixed reporting collaped reqeuest to promethues + * Feature: Update golang to 1.10 update libvips 8.6.2 * 0.7.0 * Feature: Remove mutex from time monitoring * 0.6.1 diff --git a/cmd/mort/mort.go b/cmd/mort/mort.go index 3efb183..eaa1b39 100644 --- a/cmd/mort/mort.go +++ b/cmd/mort/mort.go @@ -30,7 +30,7 @@ import ( const ( // Version of mort - Version = "0.7.0" + Version = "0.8.0" // BANNER just fancy command line banner BANNER = ` /\/\ ___ _ __| |_ diff --git a/package-lock.json b/package-lock.json index 66e93cd..7f043fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,30 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "dev": true, + "requires": { + "co": "4.6.0", + "fast-deep-equal": "1.1.0", + "fast-json-stable-stringify": "2.0.0", + "json-schema-traverse": "0.3.1" + } + }, + "asn1": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=", + "dev": true + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true + }, "assertion-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.2.tgz", @@ -31,6 +55,18 @@ "xmlbuilder": "4.2.1" } }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true + }, + "aws4": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", + "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=", + "dev": true + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -41,6 +77,25 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.1.tgz", "integrity": "sha512-dwVUVIXsBZXwTuwnXI9RK8sBmgq09NDHzyR9SAph9eqk76gKK2JSQmZARC2zRC81JC2QTtxD0ARU5qTS25gIGw==" }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", + "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", + "dev": true, + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "boom": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", + "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", + "dev": true, + "requires": { + "hoek": "4.2.1" + } + }, "brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", @@ -65,6 +120,12 @@ "isarray": "1.0.0" } }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, "chai": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", @@ -83,6 +144,12 @@ "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=" }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", + "dev": true + }, "combined-stream": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", @@ -116,11 +183,40 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cryptiles": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", + "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", + "dev": true, + "requires": { + "boom": "5.2.0" + }, + "dependencies": { + "boom": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", + "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", + "dev": true, + "requires": { + "hoek": "4.2.1" + } + } + } + }, "crypto-browserify": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-1.0.9.tgz", "integrity": "sha1-zFRJaF37hesRyYKKzHy4erW7/MA=" }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "requires": { + "assert-plus": "1.0.0" + } + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -147,6 +243,16 @@ "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==" }, + "ecc-jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", + "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", + "dev": true, + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -162,6 +268,30 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=", + "dev": true + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true + }, "form-data": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz", @@ -187,6 +317,15 @@ "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "requires": { + "assert-plus": "1.0.0" + } + }, "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", @@ -205,16 +344,61 @@ "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==" }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true + }, + "har-validator": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", + "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", + "dev": true, + "requires": { + "ajv": "5.5.2", + "har-schema": "2.0.0" + } + }, "has-flag": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=" }, + "hawk": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", + "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", + "dev": true, + "requires": { + "boom": "4.3.1", + "cryptiles": "3.1.2", + "hoek": "4.2.1", + "sntp": "2.1.0" + } + }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=" }, + "hoek": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", + "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==", + "dev": true + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "jsprim": "1.4.1", + "sshpk": "1.13.1" + } + }, "ieee754": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", @@ -234,16 +418,65 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true, + "optional": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=", + "dev": true + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, "lodash": { "version": "4.17.4", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", @@ -321,6 +554,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=", + "dev": true + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -339,6 +578,12 @@ "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=" }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, "process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", @@ -373,6 +618,36 @@ "util-deprecate": "1.0.2" } }, + "request": { + "version": "2.83.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz", + "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", + "dev": true, + "requires": { + "aws-sign2": "0.7.0", + "aws4": "1.6.0", + "caseless": "0.12.0", + "combined-stream": "1.0.5", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.3.1", + "har-validator": "5.0.3", + "hawk": "6.0.2", + "http-signature": "1.2.0", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.17", + "oauth-sign": "0.8.2", + "performance-now": "2.1.0", + "qs": "6.5.1", + "safe-buffer": "5.1.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.4", + "tunnel-agent": "0.6.0", + "uuid": "3.1.0" + } + }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -383,6 +658,31 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "sntp": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", + "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", + "dev": true, + "requires": { + "hoek": "4.2.1" + } + }, + "sshpk": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", + "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", + "dev": true, + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + } + }, "string_decoder": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", @@ -391,6 +691,12 @@ "safe-buffer": "5.1.1" } }, + "stringstream": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", + "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=", + "dev": true + }, "superagent": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.1.tgz", @@ -431,6 +737,39 @@ "has-flag": "2.0.0" } }, + "tough-cookie": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", + "dev": true, + "requires": { + "punycode": "1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + } + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true, + "optional": true + }, "type-detect": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.5.tgz", @@ -455,6 +794,17 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "1.3.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index fc6c7bf..a0d3b82 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "devDependencies": { "mocha": "^4.0.1", "moment": "^2.20.1", + "request": "^2.83.0", "superagent-binary-parser": "^1.0.1" }, "scripts": { diff --git a/pkg/middleware/s3.go b/pkg/middleware/s3.go index c978f6c..f4e647d 100644 --- a/pkg/middleware/s3.go +++ b/pkg/middleware/s3.go @@ -13,6 +13,7 @@ import ( "github.com/aldor007/go-aws-auth" "go.uber.org/zap" + "golang.org/x/net/context" ) // authHeaderRegexpv4 regular expression for AWS Auth v4 header mode @@ -21,7 +22,8 @@ var autHeaderRegexpv4 = regexp.MustCompile("^(:?[A-Za-z0-9-]+) Credential=(:?.+) // authHeaderRegexpv2 regular expression for Aws Auth v2 header mode var authHeaderRegexpv2 = regexp.MustCompile("^AWS ([A-Za-z0-9-]+):(.+)$") -func isAuthRequired(method string, auth string, path string) bool { +func isAuthRequired(req *http.Request, auth string, path string) bool { + method := req.Method switch method { case "GET", "HEAD", "OPTIONS": if path == "/" { @@ -32,6 +34,10 @@ func isAuthRequired(method string, auth string, path string) bool { return true } + if req.URL.Query().Get("X-Amz-Signature") != "" { + return true + } + return false case "POST", "PUT", "DELETE", "PATCH": return true @@ -59,7 +65,7 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { fn := func(resWriter http.ResponseWriter, req *http.Request) { path := req.URL.Path auth := req.Header.Get("Authorization") - if !isAuthRequired(req.Method, auth, path) { + if !isAuthRequired(req, auth, path) { next.ServeHTTP(resWriter, req) return } @@ -132,7 +138,7 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { } if credential.AccessKeyID == "" { res := response.NewString(401, "") - monitoring.Log().Warn("S3Auth invalid bucket config no access key or invalid", zap.String("bucket", bucketName)) + monitoring.Log().Warn("S3Auth invalid bucket config no access key or invalid", zap.String("bucket", bucketName), zap.String("req.method", req.Method), zap.String("req.path", req.URL.Path)) res.Send(resWriter) return } @@ -179,12 +185,15 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { s.listAllMyBuckets(resWriter, accessKey) return } - next.ServeHTTP(resWriter, req) + + ctx := context.WithValue(req.Context(), "auth", true) + + next.ServeHTTP(resWriter, req.WithContext(ctx)) return } - monitoring.Log().Warn("S3Auth signature mismatch", zap.String("req.path", req.URL.Path)) + monitoring.Log().Warn("S3Auth signature mismatch", zap.String("req.path", req.URL.Path), zap.String("req.method", req.Method)) response.NewNoContent(403).Send(resWriter) return } @@ -282,7 +291,9 @@ func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, buc awsauth.PreSign(&validationReq, "mort", "s3", strings.Split(validationReq.URL.Query().Get("X-Amz-SignedHeaders"), ","), credential) if validationReq.URL.Query().Get("X-Amz-Signature") == r.URL.Query().Get("X-Amz-Signature") { - next.ServeHTTP(resWriter, r) + ctx := context.WithValue(r.Context(), "auth", true) + + next.ServeHTTP(resWriter, r.WithContext(ctx)) return } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index dfa17bb..ec9dff9 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -90,10 +90,10 @@ func (r *RequestProcessor) process(req *http.Request, obj *object.FileObject) *r } if obj.HasTransform() { - return updateHeaders(r.collapseGET(req, obj)) + return updateHeaders(req, r.collapseGET(req, obj)) } - return updateHeaders(r.handleGET(req, obj)) + return updateHeaders(req, r.handleGET(req, obj)) case "PUT": return handlePUT(req, obj) case "DELETE": @@ -347,7 +347,12 @@ func (r *RequestProcessor) processImage(ctx context.Context, obj *object.FileObj } -func updateHeaders(res *response.Response) *response.Response { +func updateHeaders(req *http.Request, res *response.Response) *response.Response { + ctx := req.Context() + if ctx.Value("auth") != nil { + return res; + } + headers := config.GetInstance().Headers for _, headerPred := range headers { for _, status := range headerPred.StatusCodes { diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 9396864..1913c4f 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -27,4 +27,5 @@ if [[ $TEST_RESULT -eq 0 ]]; then rm -rf /tmp/mort-tests fi unset MORT_PORT -exit ${TEST_RESULT} \ No newline at end of file +exit ${TEST_RESULT} +pkill -f 'tests-int/config.yml' \ No newline at end of file diff --git a/tests-int/S3Objects.Spec.js b/tests-int/S3Objects.Spec.js index baa27cc..8d3f714 100644 --- a/tests-int/S3Objects.Spec.js +++ b/tests-int/S3Objects.Spec.js @@ -2,6 +2,7 @@ const chai = require('chai'); const expect = chai.expect; const AWS = require('aws-sdk'); const supertest = require('supertest'); +const makeRequest = require('request'); const host = 'localhost:' + process.env.MORT_PORT; const request = supertest(`http://${host}`); @@ -238,6 +239,45 @@ describe('S3 features', function () { }); }); + it('should allow to upload when presinged', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + this.s3.getSignedUrl('putObject', params, function (err, url) { + makeRequest({ + url: url, + method: 'PUT', + body: 'aa' + }, function (err, response) { + expect(response.statusCode).to.eql(200); + done(); + }); + }); + }); + + it('should return error when invalid access key', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + const s3Opts = JSON.parse(JSON.stringify(this.s3opts)); + s3Opts.accessKeyId = 'invalid'; + const s3 = new AWS.S3(s3Opts); + s3.getSignedUrl('putObject', params, function (err, url) { + makeRequest({ + url: url, + method: 'PUT', + body: 'aa' + }, function (err, response) { + expect(response.statusCode).to.eql(401); + done(); + }); + }); + }); + it('should return valid metadata for uploaded file', function (done) { request.get('/local/file.jpg') .expect(200) @@ -247,6 +287,37 @@ describe('S3 features', function () { }); }); + it('should return error when invalid access key - presign', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + const s3Opts = JSON.parse(JSON.stringify(this.s3opts)); + s3Opts.accessKeyId = 'invalid'; + const s3 = new AWS.S3(s3Opts); + s3.getSignedUrl('getObject', params, function (err, url) { + makeRequest(url, function (err, response) { + expect(response.statusCode).to.eql(401); + done(); + }); + }); + }); + + it('should allow to get url with presign', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + this.s3.getSignedUrl('getObject', params, function (err, url) { + makeRequest(url, function (err, response) { + expect(response.statusCode).to.eql(200); + done(); + }); + }); + }); + it('should delete file', function (done) { const params = { Bucket: 'local',