From 48958e032470e9d972fbbaefdaa8da8eae1732ac Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 6 Mar 2018 23:12:23 +0100 Subject: [PATCH 1/9] start working on AWS v4 presign url validation --- package-lock.json | 168 +++++++++++++++++++++++++++++++++++++------ pkg/middleware/s3.go | 65 +++++++++++++++++ 2 files changed, 210 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index b7c2efe..66e93cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2,6 +2,7 @@ "name": "mort-integrations-tests", "version": "1.0.0", "lockfileVersion": 1, + "requires": true, "dependencies": { "assertion-error": { "version": "1.0.2", @@ -16,7 +17,19 @@ "aws-sdk": { "version": "2.151.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.151.0.tgz", - "integrity": "sha1-XnPXeLdlFPNhg7kUAEd3eKteWNg=" + "integrity": "sha1-XnPXeLdlFPNhg7kUAEd3eKteWNg=", + "requires": { + "buffer": "4.9.1", + "crypto-browserify": "1.0.9", + "events": "1.1.1", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.1.0", + "xml2js": "0.4.17", + "xmlbuilder": "4.2.1" + } }, "balanced-match": { "version": "1.0.0", @@ -31,7 +44,11 @@ "brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", - "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=" + "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", + "requires": { + "balanced-match": "1.0.0", + "concat-map": "0.0.1" + } }, "browser-stdout": { "version": "1.3.0", @@ -41,12 +58,25 @@ "buffer": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", - "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=" + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "1.2.1", + "ieee754": "1.1.8", + "isarray": "1.0.0" + } }, "chai": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", - "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=" + "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=", + "requires": { + "assertion-error": "1.0.2", + "check-error": "1.0.2", + "deep-eql": "3.0.1", + "get-func-name": "2.0.0", + "pathval": "1.1.0", + "type-detect": "4.0.5" + } }, "check-error": { "version": "1.0.2", @@ -56,7 +86,10 @@ "combined-stream": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", - "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=" + "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=", + "requires": { + "delayed-stream": "1.0.0" + } }, "commander": { "version": "2.11.0", @@ -91,12 +124,18 @@ "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==" + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } }, "deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", - "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==" + "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", + "requires": { + "type-detect": "4.0.5" + } }, "delayed-stream": { "version": "1.0.0", @@ -126,7 +165,12 @@ "form-data": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz", - "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=" + "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=", + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.5", + "mime-types": "2.1.17" + } }, "formidable": { "version": "1.1.1", @@ -146,7 +190,15 @@ "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==" + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "requires": { + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" + } }, "growl": { "version": "1.10.3", @@ -171,7 +223,11 @@ "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=" + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "1.4.0", + "wrappy": "1.0.2" + } }, "inherits": { "version": "2.0.3", @@ -211,12 +267,18 @@ "mime-types": { "version": "2.1.17", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz", - "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=" + "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=", + "requires": { + "mime-db": "1.30.0" + } }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==" + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "1.1.8" + } }, "minimist": { "version": "0.0.8", @@ -226,12 +288,27 @@ "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=" + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + } }, "mocha": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/mocha/-/mocha-4.0.1.tgz", - "integrity": "sha512-evDmhkoA+cBNiQQQdSKZa2b9+W2mpLoj50367lhy+Klnx9OV8XlCIhigUnn1gaTFLQCa0kdNhEGDr0hCXOQFDw==" + "integrity": "sha512-evDmhkoA+cBNiQQQdSKZa2b9+W2mpLoj50367lhy+Klnx9OV8XlCIhigUnn1gaTFLQCa0kdNhEGDr0hCXOQFDw==", + "requires": { + "browser-stdout": "1.3.0", + "commander": "2.11.0", + "debug": "3.1.0", + "diff": "3.3.1", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.3", + "he": "1.1.1", + "mkdirp": "0.5.1", + "supports-color": "4.4.0" + } }, "moment": { "version": "2.20.1", @@ -247,7 +324,10 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=" + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1.0.2" + } }, "path-is-absolute": { "version": "1.0.1", @@ -282,7 +362,16 @@ "readable-stream": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", - "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==" + "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "1.0.7", + "safe-buffer": "5.1.1", + "string_decoder": "1.0.3", + "util-deprecate": "1.0.2" + } }, "safe-buffer": { "version": "5.1.1", @@ -297,12 +386,27 @@ "string_decoder": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", - "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==" + "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "requires": { + "safe-buffer": "5.1.1" + } }, "superagent": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.1.tgz", - "integrity": "sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==" + "integrity": "sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==", + "requires": { + "component-emitter": "1.2.1", + "cookiejar": "2.1.1", + "debug": "3.1.0", + "extend": "3.0.1", + "form-data": "2.3.1", + "formidable": "1.1.1", + "methods": "1.1.2", + "mime": "1.4.1", + "qs": "6.5.1", + "readable-stream": "2.3.3" + } }, "superagent-binary-parser": { "version": "1.0.1", @@ -313,12 +417,19 @@ "supertest": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/supertest/-/supertest-3.0.0.tgz", - "integrity": "sha1-jUu2j9GDDuBwM7HFpamkAhyWUpY=" + "integrity": "sha1-jUu2j9GDDuBwM7HFpamkAhyWUpY=", + "requires": { + "methods": "1.1.2", + "superagent": "3.8.1" + } }, "supports-color": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", - "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==" + "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", + "requires": { + "has-flag": "2.0.0" + } }, "type-detect": { "version": "4.0.5", @@ -328,7 +439,11 @@ "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=" + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } }, "util-deprecate": { "version": "1.0.2", @@ -348,12 +463,19 @@ "xml2js": { "version": "0.4.17", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.17.tgz", - "integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=" + "integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=", + "requires": { + "sax": "1.2.1", + "xmlbuilder": "4.2.1" + } }, "xmlbuilder": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz", - "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=" + "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=", + "requires": { + "lodash": "4.17.4" + } } } } diff --git a/pkg/middleware/s3.go b/pkg/middleware/s3.go index b38efc1..c978f6c 100644 --- a/pkg/middleware/s3.go +++ b/pkg/middleware/s3.go @@ -103,6 +103,11 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { accessKey = matches[1] } + if req.URL.Query().Get("X-Amz-Signature") != "" { + s.authByQuery(resWriter, req, bucketName, next) + return + } + bucket, ok := mortConfig.Buckets[bucketName] if !ok { buckets := mortConfig.BucketsByAccessKey(accessKey) @@ -226,3 +231,63 @@ func (s *S3Auth) listAllMyBuckets(resWriter http.ResponseWriter, accessKey strin res.SetContentType("application/xml") res.Send(resWriter) } + +func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, bucketName string, next http.Handler) { + validationReq := *r + mortConfig := s.mortConfig + + validationReq.URL.Query().Del("X-Amz-Signature") + var credential awsauth.Credentials + accessKey := strings.Split(validationReq.URL.Query().Get("X-Amz-Credential"), "/")[0] + + bucket, ok := mortConfig.Buckets[bucketName] + if !ok { + buckets := mortConfig.BucketsByAccessKey(accessKey) + if len(buckets) == 0 { + monitoring.Log().Warn("S3Auth no bucket for access key") + res := response.NewString(403, "") + res.Send(resWriter) + return + } + + bucket = buckets[0] + } + + if r.URL.Query().Get("X-Amz-Credential") == "" { + res := response.NewString(401, "") + monitoring.Log().Warn("S3Auth invalid request no x-amz-credential in query string", zap.String("bucket", bucketName)) + res.Send(resWriter) + return + } + + + + keys := bucket.Keys + for _, key := range keys { + if accessKey == key.AccessKey { + credential.AccessKeyID = accessKey + credential.SecretAccessKey = key.SecretAccessKey + break + } + + } + + if credential.AccessKeyID == "" { + res := response.NewString(401, "") + monitoring.Log().Warn("S3Auth invalid bucket config no access key or invalid", zap.String("bucket", bucketName)) + res.Send(resWriter) + return + } + + awsauth.PreSign(&validationReq, "mort", "s3", strings.Split(validationReq.URL.Query().Get("X-Amz-SignedHeaders"), ","), credential) + + if validationReq.URL.Query().Get("X-Amz-Signature") == r.URL.Query().Get("X-Amz-Signature") { + next.ServeHTTP(resWriter, r) + return + } + + monitoring.Log().Warn("S3Auth signature mismatch", zap.String("req.path", r.URL.Path)) + response.NewNoContent(403).Send(resWriter) + return + +} \ No newline at end of file From 163f9979f8e83334c60232fa4da89b029bd27e68 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 7 Mar 2018 01:13:27 +0100 Subject: [PATCH 2/9] update golang version --- Dockerfile | 8 ++++---- Makefile | 7 ++----- cmd/mort/mort.go | 6 ++++++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1725de..4233fd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,8 @@ FROM ubuntu:16.04 as builder - -ENV LIBVIPS_VERSION 8.5.9 -ENV DEP_VERSION v0.3.2 -ENV GOLANG_VERSION 1.9.2 +ENV LIBVIPS_VERSION 8.6.2 +ENV DEP_VERSION v0.4.1 +ENV GOLANG_VERSION 1.10 # Installs libvips + required libraries @@ -78,6 +77,7 @@ COPY --from=builder /usr/local/lib /usr/local/lib RUN ldconfig COPY --from=builder /go/mort /go/mort COPY --from=builder /go/src/github.com/aldor007/mort/configuration/config.yml /etc/mort/mort.yml +RUN /go/mort -version # add mime types ADD http://svn.apache.org/viewvc/httpd/httpd/branches/2.2.x/docs/conf/mime.types?view=co /etc/mime.types diff --git a/Makefile b/Makefile index d734a9e..216e016 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,8 @@ format: tests: unit integrations -docker-build: - docker build -t aldo007/mort -f Dockerfile . - -docker-push: docker-build - docker push aldor007/mort:latest +docker-push: + docker build -t aldor007/mort -f Dockerfile . -t aldor007/mort:latest; docker push aldor007/mort:latest run-server: mkdir -p /tmp/mort diff --git a/cmd/mort/mort.go b/cmd/mort/mort.go index 3efb183..a5f5b70 100644 --- a/cmd/mort/mort.go +++ b/cmd/mort/mort.go @@ -150,8 +150,14 @@ func startServer(s *http.Server, ln net.Listener) { func main() { configPath := flag.String("config", "/etc/mort/mort.yml", "Path to configuration") debug := flag.Bool("debug", false, "enable debug mode") + version := flag.Bool("version", false,"get mort version") flag.Parse() + if version != nil && *version == true { + fmt.Println(Version) + return + } + router := chi.NewRouter() imgConfig := config.GetInstance() err := imgConfig.Load(*configPath) From 719dbbfbe3fc41163ac716d95fa403f78014b79d Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 7 Mar 2018 20:04:08 +0100 Subject: [PATCH 3/9] Added support for presign AWS v4 --- CHANGELOG.md | 4 + cmd/mort/mort.go | 2 +- package-lock.json | 350 ++++++++++++++++++++++++++++++++++++ package.json | 1 + pkg/middleware/s3.go | 23 ++- pkg/processor/processor.go | 11 +- scripts/run-tests.sh | 3 +- tests-int/S3Objects.Spec.js | 71 ++++++++ 8 files changed, 454 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2bc1ac..50a8cd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +* 0.8.0 + * Feature: Added support for AWS presigned url + * Bugfix: Fixed reporting collaped reqeuest to promethues + * Feature: Update golang to 1.10 update libvips 8.6.2 * 0.7.0 * Feature: Remove mutex from time monitoring * 0.6.1 diff --git a/cmd/mort/mort.go b/cmd/mort/mort.go index 3efb183..eaa1b39 100644 --- a/cmd/mort/mort.go +++ b/cmd/mort/mort.go @@ -30,7 +30,7 @@ import ( const ( // Version of mort - Version = "0.7.0" + Version = "0.8.0" // BANNER just fancy command line banner BANNER = ` /\/\ ___ _ __| |_ diff --git a/package-lock.json b/package-lock.json index 66e93cd..7f043fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,30 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "dev": true, + "requires": { + "co": "4.6.0", + "fast-deep-equal": "1.1.0", + "fast-json-stable-stringify": "2.0.0", + "json-schema-traverse": "0.3.1" + } + }, + "asn1": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=", + "dev": true + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true + }, "assertion-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.2.tgz", @@ -31,6 +55,18 @@ "xmlbuilder": "4.2.1" } }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true + }, + "aws4": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", + "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=", + "dev": true + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -41,6 +77,25 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.1.tgz", "integrity": "sha512-dwVUVIXsBZXwTuwnXI9RK8sBmgq09NDHzyR9SAph9eqk76gKK2JSQmZARC2zRC81JC2QTtxD0ARU5qTS25gIGw==" }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", + "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", + "dev": true, + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "boom": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", + "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", + "dev": true, + "requires": { + "hoek": "4.2.1" + } + }, "brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", @@ -65,6 +120,12 @@ "isarray": "1.0.0" } }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, "chai": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", @@ -83,6 +144,12 @@ "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=" }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", + "dev": true + }, "combined-stream": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", @@ -116,11 +183,40 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cryptiles": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", + "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", + "dev": true, + "requires": { + "boom": "5.2.0" + }, + "dependencies": { + "boom": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", + "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", + "dev": true, + "requires": { + "hoek": "4.2.1" + } + } + } + }, "crypto-browserify": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-1.0.9.tgz", "integrity": "sha1-zFRJaF37hesRyYKKzHy4erW7/MA=" }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "requires": { + "assert-plus": "1.0.0" + } + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -147,6 +243,16 @@ "resolved": "https://registry.npmjs.org/diff/-/diff-3.3.1.tgz", "integrity": "sha512-MKPHZDMB0o6yHyDryUOScqZibp914ksXwAMYMTHj6KO8UeKsRYNJD3oNCKjTqZon+V488P7N/HzXF8t7ZR95ww==" }, + "ecc-jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", + "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", + "dev": true, + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -162,6 +268,30 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=", + "dev": true + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true + }, "form-data": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz", @@ -187,6 +317,15 @@ "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "requires": { + "assert-plus": "1.0.0" + } + }, "glob": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", @@ -205,16 +344,61 @@ "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.3.tgz", "integrity": "sha512-hKlsbA5Vu3xsh1Cg3J7jSmX/WaW6A5oBeqzM88oNbCRQFz+zUaXm6yxS4RVytp1scBoJzSYl4YAEOQIt6O8V1Q==" }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true + }, + "har-validator": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", + "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", + "dev": true, + "requires": { + "ajv": "5.5.2", + "har-schema": "2.0.0" + } + }, "has-flag": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=" }, + "hawk": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", + "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", + "dev": true, + "requires": { + "boom": "4.3.1", + "cryptiles": "3.1.2", + "hoek": "4.2.1", + "sntp": "2.1.0" + } + }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=" }, + "hoek": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", + "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==", + "dev": true + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "jsprim": "1.4.1", + "sshpk": "1.13.1" + } + }, "ieee754": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", @@ -234,16 +418,65 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true, + "optional": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=", + "dev": true + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, "lodash": { "version": "4.17.4", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", @@ -321,6 +554,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=", + "dev": true + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -339,6 +578,12 @@ "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=" }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, "process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", @@ -373,6 +618,36 @@ "util-deprecate": "1.0.2" } }, + "request": { + "version": "2.83.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz", + "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", + "dev": true, + "requires": { + "aws-sign2": "0.7.0", + "aws4": "1.6.0", + "caseless": "0.12.0", + "combined-stream": "1.0.5", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.3.1", + "har-validator": "5.0.3", + "hawk": "6.0.2", + "http-signature": "1.2.0", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.17", + "oauth-sign": "0.8.2", + "performance-now": "2.1.0", + "qs": "6.5.1", + "safe-buffer": "5.1.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.4", + "tunnel-agent": "0.6.0", + "uuid": "3.1.0" + } + }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -383,6 +658,31 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "sntp": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", + "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", + "dev": true, + "requires": { + "hoek": "4.2.1" + } + }, + "sshpk": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", + "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", + "dev": true, + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + } + }, "string_decoder": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", @@ -391,6 +691,12 @@ "safe-buffer": "5.1.1" } }, + "stringstream": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", + "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=", + "dev": true + }, "superagent": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.1.tgz", @@ -431,6 +737,39 @@ "has-flag": "2.0.0" } }, + "tough-cookie": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", + "dev": true, + "requires": { + "punycode": "1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + } + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "requires": { + "safe-buffer": "5.1.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true, + "optional": true + }, "type-detect": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.5.tgz", @@ -455,6 +794,17 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "requires": { + "assert-plus": "1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "1.3.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/package.json b/package.json index fc6c7bf..a0d3b82 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "devDependencies": { "mocha": "^4.0.1", "moment": "^2.20.1", + "request": "^2.83.0", "superagent-binary-parser": "^1.0.1" }, "scripts": { diff --git a/pkg/middleware/s3.go b/pkg/middleware/s3.go index c978f6c..f4e647d 100644 --- a/pkg/middleware/s3.go +++ b/pkg/middleware/s3.go @@ -13,6 +13,7 @@ import ( "github.com/aldor007/go-aws-auth" "go.uber.org/zap" + "golang.org/x/net/context" ) // authHeaderRegexpv4 regular expression for AWS Auth v4 header mode @@ -21,7 +22,8 @@ var autHeaderRegexpv4 = regexp.MustCompile("^(:?[A-Za-z0-9-]+) Credential=(:?.+) // authHeaderRegexpv2 regular expression for Aws Auth v2 header mode var authHeaderRegexpv2 = regexp.MustCompile("^AWS ([A-Za-z0-9-]+):(.+)$") -func isAuthRequired(method string, auth string, path string) bool { +func isAuthRequired(req *http.Request, auth string, path string) bool { + method := req.Method switch method { case "GET", "HEAD", "OPTIONS": if path == "/" { @@ -32,6 +34,10 @@ func isAuthRequired(method string, auth string, path string) bool { return true } + if req.URL.Query().Get("X-Amz-Signature") != "" { + return true + } + return false case "POST", "PUT", "DELETE", "PATCH": return true @@ -59,7 +65,7 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { fn := func(resWriter http.ResponseWriter, req *http.Request) { path := req.URL.Path auth := req.Header.Get("Authorization") - if !isAuthRequired(req.Method, auth, path) { + if !isAuthRequired(req, auth, path) { next.ServeHTTP(resWriter, req) return } @@ -132,7 +138,7 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { } if credential.AccessKeyID == "" { res := response.NewString(401, "") - monitoring.Log().Warn("S3Auth invalid bucket config no access key or invalid", zap.String("bucket", bucketName)) + monitoring.Log().Warn("S3Auth invalid bucket config no access key or invalid", zap.String("bucket", bucketName), zap.String("req.method", req.Method), zap.String("req.path", req.URL.Path)) res.Send(resWriter) return } @@ -179,12 +185,15 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { s.listAllMyBuckets(resWriter, accessKey) return } - next.ServeHTTP(resWriter, req) + + ctx := context.WithValue(req.Context(), "auth", true) + + next.ServeHTTP(resWriter, req.WithContext(ctx)) return } - monitoring.Log().Warn("S3Auth signature mismatch", zap.String("req.path", req.URL.Path)) + monitoring.Log().Warn("S3Auth signature mismatch", zap.String("req.path", req.URL.Path), zap.String("req.method", req.Method)) response.NewNoContent(403).Send(resWriter) return } @@ -282,7 +291,9 @@ func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, buc awsauth.PreSign(&validationReq, "mort", "s3", strings.Split(validationReq.URL.Query().Get("X-Amz-SignedHeaders"), ","), credential) if validationReq.URL.Query().Get("X-Amz-Signature") == r.URL.Query().Get("X-Amz-Signature") { - next.ServeHTTP(resWriter, r) + ctx := context.WithValue(r.Context(), "auth", true) + + next.ServeHTTP(resWriter, r.WithContext(ctx)) return } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index dfa17bb..ec9dff9 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -90,10 +90,10 @@ func (r *RequestProcessor) process(req *http.Request, obj *object.FileObject) *r } if obj.HasTransform() { - return updateHeaders(r.collapseGET(req, obj)) + return updateHeaders(req, r.collapseGET(req, obj)) } - return updateHeaders(r.handleGET(req, obj)) + return updateHeaders(req, r.handleGET(req, obj)) case "PUT": return handlePUT(req, obj) case "DELETE": @@ -347,7 +347,12 @@ func (r *RequestProcessor) processImage(ctx context.Context, obj *object.FileObj } -func updateHeaders(res *response.Response) *response.Response { +func updateHeaders(req *http.Request, res *response.Response) *response.Response { + ctx := req.Context() + if ctx.Value("auth") != nil { + return res; + } + headers := config.GetInstance().Headers for _, headerPred := range headers { for _, status := range headerPred.StatusCodes { diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 9396864..1913c4f 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -27,4 +27,5 @@ if [[ $TEST_RESULT -eq 0 ]]; then rm -rf /tmp/mort-tests fi unset MORT_PORT -exit ${TEST_RESULT} \ No newline at end of file +exit ${TEST_RESULT} +pkill -f 'tests-int/config.yml' \ No newline at end of file diff --git a/tests-int/S3Objects.Spec.js b/tests-int/S3Objects.Spec.js index baa27cc..8d3f714 100644 --- a/tests-int/S3Objects.Spec.js +++ b/tests-int/S3Objects.Spec.js @@ -2,6 +2,7 @@ const chai = require('chai'); const expect = chai.expect; const AWS = require('aws-sdk'); const supertest = require('supertest'); +const makeRequest = require('request'); const host = 'localhost:' + process.env.MORT_PORT; const request = supertest(`http://${host}`); @@ -238,6 +239,45 @@ describe('S3 features', function () { }); }); + it('should allow to upload when presinged', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + this.s3.getSignedUrl('putObject', params, function (err, url) { + makeRequest({ + url: url, + method: 'PUT', + body: 'aa' + }, function (err, response) { + expect(response.statusCode).to.eql(200); + done(); + }); + }); + }); + + it('should return error when invalid access key', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + const s3Opts = JSON.parse(JSON.stringify(this.s3opts)); + s3Opts.accessKeyId = 'invalid'; + const s3 = new AWS.S3(s3Opts); + s3.getSignedUrl('putObject', params, function (err, url) { + makeRequest({ + url: url, + method: 'PUT', + body: 'aa' + }, function (err, response) { + expect(response.statusCode).to.eql(401); + done(); + }); + }); + }); + it('should return valid metadata for uploaded file', function (done) { request.get('/local/file.jpg') .expect(200) @@ -247,6 +287,37 @@ describe('S3 features', function () { }); }); + it('should return error when invalid access key - presign', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + const s3Opts = JSON.parse(JSON.stringify(this.s3opts)); + s3Opts.accessKeyId = 'invalid'; + const s3 = new AWS.S3(s3Opts); + s3.getSignedUrl('getObject', params, function (err, url) { + makeRequest(url, function (err, response) { + expect(response.statusCode).to.eql(401); + done(); + }); + }); + }); + + it('should allow to get url with presign', function (done) { + const params = { + Bucket: 'local', + Key: 'pre-sign', + }; + + this.s3.getSignedUrl('getObject', params, function (err, url) { + makeRequest(url, function (err, response) { + expect(response.statusCode).to.eql(200); + done(); + }); + }); + }); + it('should delete file', function (done) { const params = { Bucket: 'local', From dcc0cb87f63b4f1568c78ffaeb0583090b270503 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 7 Mar 2018 20:10:03 +0100 Subject: [PATCH 4/9] update dependencies --- Gopkg.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 2e0d377..0946353 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -5,7 +5,7 @@ branch = "master" name = "github.com/aldor007/go-aws-auth" packages = ["."] - revision = "479c2853d1f8aec2969767928ce68b116e537cbc" + revision = "5223cf5e46e666780a3fcdd4e13a0f6b3b00ae49" [[projects]] branch = "develop" @@ -171,6 +171,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "299c127c9fd9da73e9ba96de823ab388a29390759da0e56c628948fdcc459182" + inputs-digest = "009e220a3cd6593aa45a5708b0655cbac7321410363b26bd8c9fdb75989bcc7a" solver-name = "gps-cdcl" solver-version = 1 From 62c2519a6ed1bc5046e9f02b3fa69c2854241146 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 8 Mar 2018 22:44:28 +0100 Subject: [PATCH 5/9] allow to define placeholder image for error response --- CHANGELOG.md | 2 + cmd/mort/mort.go | 2 +- pkg/config/types.go | 1 + pkg/helpers/helpers.go | 53 +++++++++++++++++++++++ pkg/processor/processor.go | 81 +++++++++++++++++++++++++++--------- pkg/response/response.go | 8 ++-- pkg/storage/storage.go | 3 +- pkg/transforms/transforms.go | 47 +-------------------- 8 files changed, 125 insertions(+), 72 deletions(-) create mode 100644 pkg/helpers/helpers.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a8cd5..e669604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* 0.9.0 + * Feature: Allow to define placeholder for error response * 0.8.0 * Feature: Added support for AWS presigned url * Bugfix: Fixed reporting collaped reqeuest to promethues diff --git a/cmd/mort/mort.go b/cmd/mort/mort.go index c5a0c96..f60def6 100644 --- a/cmd/mort/mort.go +++ b/cmd/mort/mort.go @@ -30,7 +30,7 @@ import ( const ( // Version of mort - Version = "0.8.0" + Version = "0.9.0" // BANNER just fancy command line banner BANNER = ` /\/\ ___ _ __| |_ diff --git a/pkg/config/types.go b/pkg/config/types.go index 8015291..ecb725d 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -111,4 +111,5 @@ type Server struct { QueueLen int `yaml:"queueLen"` Listen []string `yaml:"listens"` Monitoring string `yaml:"monitoring"` + Placeholder string `yaml:"placeholder"` } diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go new file mode 100644 index 0000000..e1f7021 --- /dev/null +++ b/pkg/helpers/helpers.go @@ -0,0 +1,53 @@ +package helpers + +import ( + "strings" + "net/http" + "net" + "time" + "io/ioutil" + "os" +) + +func FetchObject(uri string) ([]byte, error) { + if strings.HasPrefix(uri, "http") { + client := &http.Client{ + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } + + req, err := http.NewRequest("GET", uri, nil) + if err != nil { + return nil, err + } + + response, err := client.Do(req) + if err != nil { + return nil, err + } + + defer response.Body.Close() + buf, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + + return buf, nil + } + + f, err := os.Open(uri) + if err != nil { + return nil, err + } + + defer f.Close() + + return ioutil.ReadAll(f) +} diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index ec9dff9..9648f82 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -17,12 +17,21 @@ import ( "github.com/aldor007/mort/pkg/storage" "github.com/aldor007/mort/pkg/throttler" "github.com/aldor007/mort/pkg/transforms" + "github.com/aldor007/mort/pkg/helpers" "github.com/karlseguin/ccache" "go.uber.org/zap" ) const s3LocationStr = "EU" +var ( + ErrTimeout = errors.New("timeout") + ErrContextCancel = errors.New("context timeout") + ErrThrottled = errors.New("throttled") +) + + + // NewRequestProcessor create instance of request processor // It main component of mort it handle all of requests func NewRequestProcessor(serverConfig config.Server, l lock.Lock, throttler throttler.Throttler) RequestProcessor { @@ -33,6 +42,7 @@ func NewRequestProcessor(serverConfig config.Server, l lock.Lock, throttler thro rp.cache = ccache.New(ccache.Configure().MaxSize(serverConfig.CacheSize)) rp.processTimeout = time.Duration(serverConfig.RequestTimeout) * time.Second rp.lockTimeout = time.Duration(serverConfig.RequestTimeout-1) * time.Second + rp.serverConfig = serverConfig return rp } @@ -44,6 +54,7 @@ type RequestProcessor struct { cache *ccache.Cache // cache for created image transformations processTimeout time.Duration // request processing timeout lockTimeout time.Duration // lock timeout for collapsed request it equal processTimeout - 1 s + serverConfig config.Server } type requestMessage struct { @@ -66,12 +77,12 @@ func (r *RequestProcessor) Process(req *http.Request, obj *object.FileObject) *r select { case <-ctx.Done(): monitoring.Log().Warn("Process timeout", zap.String("obj.Key", obj.Key), zap.String("error", "Context.timeout")) - return response.NewNoContent(499) + return r.replyWithError(obj, 499, ErrContextCancel) case res := <-msg.responseChan: return res case <-timer.C: monitoring.Log().Warn("Process timeout", zap.String("obj.Key", obj.Key), zap.String("error", "timeout")) - return response.NewString(504, "timeout") + return r.replyWithError(obj, 504, ErrTimeout) } } @@ -82,6 +93,36 @@ func (r *RequestProcessor) processChan() { msg.responseChan <- res } +func (r *RequestProcessor) replyWithError(obj *object.FileObject, sc int, err error) *response.Response { + if !obj.HasTransform() || r.serverConfig.Placeholder == "" { + return response.NewError(sc, err) + } + + key := r.serverConfig.Placeholder + strconv.FormatUint(obj.Transforms.Hash().Sum64(), 16) + strconv.FormatInt(int64(sc), 10) + if cacheRes := r.fetchResponseFromCache(key); cacheRes != nil { + return cacheRes + } + + buf, err := helpers.FetchObject(r.serverConfig.Placeholder) + if err != nil { + return response.NewError(sc, err) + } + + parent := response.NewBuf(sc, buf) + transformsTab := []transforms.Transforms{obj.Transforms} + + eng := engine.NewImageEngine(parent) + res, err := eng.Process(obj, transformsTab) + res.StatusCode = sc + resCpy, errCpy := res.Copy() + if errCpy != nil { + r.cache.Set(key, resCpy, time.Minute * 10) + } + + return res +} + + func (r *RequestProcessor) process(req *http.Request, obj *object.FileObject) *response.Response { switch req.Method { case "GET", "HEAD": @@ -128,7 +169,7 @@ func (r *RequestProcessor) collapseGET(req *http.Request, obj *object.FileObject select { case <-ctx.Done(): lockResult.Cancel <- true - return response.NewNoContent(499) + return r.replyWithError(obj, 499, ErrContextCancel) case res, ok := <-lockResult.ResponseChan: if ok { return res @@ -137,7 +178,7 @@ func (r *RequestProcessor) collapseGET(req *http.Request, obj *object.FileObject return r.handleGET(req, obj) case <-timer.C: lockResult.Cancel <- true - return response.NewString(504, "timeout") + return r.replyWithError(obj, 504, ErrTimeout) default: if cacheRes := r.fetchResponseFromCache(obj.Key); cacheRes != nil { lockResult.Cancel <- true @@ -180,7 +221,7 @@ func (r *RequestProcessor) handleGET(req *http.Request, obj *object.FileObject) var currObj *object.FileObject = obj var parentObj *object.FileObject - var transforms []transforms.Transforms + var transformsTab []transforms.Transforms var res *response.Response var parentRes *response.Response ctx := req.Context() @@ -188,7 +229,7 @@ func (r *RequestProcessor) handleGET(req *http.Request, obj *object.FileObject) // search for last parent for currObj.HasParent() { if currObj.HasTransform() { - transforms = append(transforms, currObj.Transforms) + transformsTab = append(transformsTab, currObj.Transforms) } currObj = currObj.Parent @@ -215,7 +256,7 @@ resLoop: for { select { case <-ctx.Done(): - return response.NewNoContent(499) + return r.replyWithError(obj, 499, ErrContextCancel) case res = <-resChan: if obj.CheckParent && parentObj != nil && (parentRes == nil || parentRes.StatusCode == 0) { go func() { @@ -258,18 +299,18 @@ resLoop: defer parentRes.Close() - transLen := len(transforms) + transLen := len(transformsTab) if transLen > 1 { // revers order of transforms - for i := 0; i < len(transforms)/2; i++ { - j := len(transforms) - i - 1 - transforms[i], transforms[j] = transforms[j], transforms[i] + for i := 0; i < len(transformsTab)/2; i++ { + j := len(transformsTab) - i - 1 + transformsTab[i], transformsTab[j] = transformsTab[j], transformsTab[i] } } - monitoring.Log().Info("Performing transforms", zap.String("obj.Bucket", obj.Bucket), zap.String("obj.Key", obj.Key), zap.Int("transformsLen", len(transforms))) - return r.processImage(ctx, obj, parentRes, transforms) + monitoring.Log().Info("Performing transforms", zap.String("obj.Bucket", obj.Bucket), zap.String("obj.Key", obj.Key), zap.Int("transformsLen", len(transformsTab))) + return r.processImage(ctx, obj, parentRes, transformsTab) } else if obj.HasTransform() { parentRes.Close() monitoring.Log().Warn("Not performing transforms", zap.String("obj.Bucket", obj.Bucket), zap.String("obj.Key", obj.Key), @@ -321,12 +362,12 @@ func (r *RequestProcessor) processImage(ctx context.Context, obj *object.FileObj if !taked { monitoring.Log().Warn("Processor/processImage", zap.String("obj.Key", obj.Key), zap.String("error", "throttled")) monitoring.Report().Inc("throttled_count") - return response.NewNoContent(503) + return r.replyWithError(obj, 503, ErrThrottled) } defer r.throttler.Release() - engine := engine.NewImageEngine(parent) - res, err := engine.Process(obj, transforms) + eng := engine.NewImageEngine(parent) + res, err := eng.Process(obj, transforms) if err != nil { return response.NewError(400, err) } @@ -349,9 +390,6 @@ func (r *RequestProcessor) processImage(ctx context.Context, obj *object.FileObj func updateHeaders(req *http.Request, res *response.Response) *response.Response { ctx := req.Context() - if ctx.Value("auth") != nil { - return res; - } headers := config.GetInstance().Headers for _, headerPred := range headers { @@ -364,5 +402,10 @@ func updateHeaders(req *http.Request, res *response.Response) *response.Response } } } + + if ctx.Value("auth") != nil { + res.Set("Cache-Control", "no-cache") + } + return res } diff --git a/pkg/response/response.go b/pkg/response/response.go index 8e13e1e..d44ed33 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -4,10 +4,8 @@ import ( "bytes" "encoding/json" "errors" - "github.com/aldor007/mort/pkg/monitoring" "github.com/aldor007/mort/pkg/object" "github.com/djherbis/stream" - "go.uber.org/zap" "io" "io/ioutil" "net/http" @@ -239,7 +237,7 @@ func (r *Response) Send(w http.ResponseWriter) error { // SendContent use http.ServeContent to return response to client // It can handle range and condition requests func (r *Response) SendContent(req *http.Request, w http.ResponseWriter) error { - if r.StatusCode != 200 || r.bodySeeker == nil || isRangeOrCondition(req) == false { + if r.bodySeeker == nil || isRangeOrCondition(req) == false { return r.Send(w) } @@ -250,7 +248,6 @@ func (r *Response) SendContent(req *http.Request, w http.ResponseWriter) error { lastMod, err := time.Parse(http.TimeFormat, r.Headers.Get("Last-Modified")) if err != nil { - monitoring.Log().Error("Unable to parse last-modified", zap.Error(err)) lastMod = time.Now() } @@ -288,6 +285,7 @@ func (r *Response) Copy() (*Response, error) { c.ContentLength = int64(len(r.body)) c.body = r.body c.bodySeeker = bytes.NewReader(c.body) + c.reader = ioutil.NopCloser(c.bodySeeker) } else if r.reader != nil { buf, err := r.CopyBody() if err != nil { @@ -353,7 +351,7 @@ func (r *Response) Stream() io.ReadCloser { return nil } -// IsBuffered check if response has access to original buffor +// IsBuffered check if response has access to original buffer func (r *Response) IsBuffered() bool { return r.body != nil } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 1f15f44..bc30685 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -55,9 +55,8 @@ func Get(obj *object.FileObject) *response.Response { return response.NewError(500, err) } - var reader io.ReadCloser if isDir(item) == false { - reader, err = item.Open() + reader, err := item.Open() if err != nil { monitoring.Logs().Warnw("Storage/Get open item", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) diff --git a/pkg/transforms/transforms.go b/pkg/transforms/transforms.go index ffaccb9..b613b55 100644 --- a/pkg/transforms/transforms.go +++ b/pkg/transforms/transforms.go @@ -4,15 +4,11 @@ import ( "encoding/binary" "errors" "hash" - "io/ioutil" - "net" - "net/http" - "os" "strings" - "time" "github.com/spaolacci/murmur3" "gopkg.in/h2non/bimg.v1" + "github.com/aldor007/mort/pkg/helpers" ) var watermarkPosX = map[string]float32{ @@ -57,46 +53,7 @@ var angleMap = map[int]bimg.Angle{ var prime64 = 1099511628211 func (w watermark) fetchImage() ([]byte, error) { - if strings.HasPrefix(w.image, "http") { - client := &http.Client{ - Transport: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } - - req, err := http.NewRequest("GET", w.image, nil) - if err != nil { - return nil, err - } - - response, err := client.Do(req) - if err != nil { - return nil, err - } - - defer response.Body.Close() - buf, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - - return buf, nil - } - - f, err := os.Open(w.image) - if err != nil { - return nil, err - } - - defer f.Close() - - return ioutil.ReadAll(f) + return helpers.FetchObject(w.image) } func (w watermark) calculatePostion(width, height int) (top int, left int) { From 3dc7f53d70ec14f6310fa22753eca933baa83a65 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Fri, 9 Mar 2018 00:08:27 +0100 Subject: [PATCH 6/9] restore debug option for request --- cmd/mort/mort.go | 15 +++++---------- pkg/helpers/helpers.go | 8 ++++---- pkg/middleware/s3.go | 8 +++----- pkg/object/file_object.go | 1 + pkg/processor/processor.go | 24 ++++++++++++------------ pkg/response/response.go | 30 ++++++++++++++---------------- pkg/response/response_test.go | 3 ++- pkg/transforms/transforms.go | 2 +- 8 files changed, 42 insertions(+), 49 deletions(-) diff --git a/cmd/mort/mort.go b/cmd/mort/mort.go index f60def6..b3d72b9 100644 --- a/cmd/mort/mort.go +++ b/cmd/mort/mort.go @@ -74,13 +74,7 @@ func handleSignals(servers []*http.Server, socketPaths []string, wg *sync.WaitGr for { sig := <-signalChan switch sig { - case syscall.SIGTERM: - fallthrough - case syscall.SIGKILL: - fallthrough - case syscall.SIGINT: - fallthrough - case os.Kill: + case os.Kill, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT: for _, s := range servers { s.Close() wg.Done() @@ -150,7 +144,7 @@ func startServer(s *http.Server, ln net.Listener) { func main() { configPath := flag.String("config", "/etc/mort/mort.yml", "Path to configuration") debug := flag.Bool("debug", false, "enable debug mode") - version := flag.Bool("version", false,"get mort version") + version := flag.Bool("version", false, "get mort version") flag.Parse() if version != nil && *version == true { @@ -184,12 +178,13 @@ func main() { obj, err := object.NewFileObject(req.URL, imgConfig) if err != nil { monitoring.Logs().Errorf("Unable to create file object err = %s", err) - response.NewError(400, err).SetDebug(debug, nil).Send(resWriter) + response.NewError(400, err).SetDebug(&object.FileObject{Debug: debug}).Send(resWriter) return } + obj.Debug = debug res := rp.Process(req, obj) - res.SetDebug(debug, obj) + res.SetDebug(obj) if debug { res.Set("X-Mort-Version", Version) } diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index e1f7021..5789e98 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -1,12 +1,12 @@ package helpers import ( - "strings" - "net/http" - "net" - "time" "io/ioutil" + "net" + "net/http" "os" + "strings" + "time" ) func FetchObject(uri string) ([]byte, error) { diff --git a/pkg/middleware/s3.go b/pkg/middleware/s3.go index f4e647d..c4ce633 100644 --- a/pkg/middleware/s3.go +++ b/pkg/middleware/s3.go @@ -65,7 +65,7 @@ func (s *S3Auth) Handler(next http.Handler) http.Handler { fn := func(resWriter http.ResponseWriter, req *http.Request) { path := req.URL.Path auth := req.Header.Get("Authorization") - if !isAuthRequired(req, auth, path) { + if !isAuthRequired(req, auth, path) { next.ServeHTTP(resWriter, req) return } @@ -241,7 +241,7 @@ func (s *S3Auth) listAllMyBuckets(resWriter http.ResponseWriter, accessKey strin res.Send(resWriter) } -func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, bucketName string, next http.Handler) { +func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, bucketName string, next http.Handler) { validationReq := *r mortConfig := s.mortConfig @@ -269,8 +269,6 @@ func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, buc return } - - keys := bucket.Keys for _, key := range keys { if accessKey == key.AccessKey { @@ -301,4 +299,4 @@ func (s *S3Auth) authByQuery(resWriter http.ResponseWriter, r *http.Request, buc response.NewNoContent(403).Send(resWriter) return -} \ No newline at end of file +} diff --git a/pkg/object/file_object.go b/pkg/object/file_object.go index a80933a..88dbcb9 100644 --- a/pkg/object/file_object.go +++ b/pkg/object/file_object.go @@ -20,6 +20,7 @@ type FileObject struct { Parent *FileObject // original image for transformed image CheckParent bool // boolean if we should always check if parent exists allowChangeKey bool // parser can allow or not changing key by this flag + Debug bool // flag for debug requests } // NewFileObjectFromPath create new instance of FileObject diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 9648f82..94b5bd0 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -10,6 +10,7 @@ import ( "github.com/aldor007/mort/pkg/config" "github.com/aldor007/mort/pkg/engine" + "github.com/aldor007/mort/pkg/helpers" "github.com/aldor007/mort/pkg/lock" "github.com/aldor007/mort/pkg/monitoring" "github.com/aldor007/mort/pkg/object" @@ -17,7 +18,6 @@ import ( "github.com/aldor007/mort/pkg/storage" "github.com/aldor007/mort/pkg/throttler" "github.com/aldor007/mort/pkg/transforms" - "github.com/aldor007/mort/pkg/helpers" "github.com/karlseguin/ccache" "go.uber.org/zap" ) @@ -25,13 +25,11 @@ import ( const s3LocationStr = "EU" var ( - ErrTimeout = errors.New("timeout") + ErrTimeout = errors.New("timeout") ErrContextCancel = errors.New("context timeout") - ErrThrottled = errors.New("throttled") + ErrThrottled = errors.New("throttled") ) - - // NewRequestProcessor create instance of request processor // It main component of mort it handle all of requests func NewRequestProcessor(serverConfig config.Server, l lock.Lock, throttler throttler.Throttler) RequestProcessor { @@ -54,7 +52,7 @@ type RequestProcessor struct { cache *ccache.Cache // cache for created image transformations processTimeout time.Duration // request processing timeout lockTimeout time.Duration // lock timeout for collapsed request it equal processTimeout - 1 s - serverConfig config.Server + serverConfig config.Server } type requestMessage struct { @@ -94,7 +92,7 @@ func (r *RequestProcessor) processChan() { } func (r *RequestProcessor) replyWithError(obj *object.FileObject, sc int, err error) *response.Response { - if !obj.HasTransform() || r.serverConfig.Placeholder == "" { + if !obj.HasTransform() || obj.Debug || r.serverConfig.Placeholder == "" { return response.NewError(sc, err) } @@ -116,13 +114,12 @@ func (r *RequestProcessor) replyWithError(obj *object.FileObject, sc int, err er res.StatusCode = sc resCpy, errCpy := res.Copy() if errCpy != nil { - r.cache.Set(key, resCpy, time.Minute * 10) + r.cache.Set(key, resCpy, time.Minute*10) } return res } - func (r *RequestProcessor) process(req *http.Request, obj *object.FileObject) *response.Response { switch req.Method { case "GET", "HEAD": @@ -292,6 +289,12 @@ resLoop: parentRes = storage.Head(parentObj) } + if parentRes.HasError() { + return r.replyWithError(obj, parentRes.StatusCode, parentRes.Error()) + } else if parentRes.StatusCode == 404 { + return parentRes + } + if obj.HasTransform() && parentRes.StatusCode == 200 && strings.Contains(parentRes.Headers.Get(response.HeaderContentType), "image/") { defer res.Close() parentRes.Close() @@ -317,9 +320,6 @@ resLoop: zap.String("parent.Key", parentObj.Key), zap.Int("parent.sc", parentRes.StatusCode), zap.String("parent.ContentType", parentRes.Headers.Get(response.HeaderContentType)), zap.Error(parentRes.Error())) } - if parentRes.StatusCode != 200 && parentRes.StatusCode != 404 { - return parentRes - } } return res diff --git a/pkg/response/response.go b/pkg/response/response.go index d44ed33..27995d1 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -174,23 +174,21 @@ func (r *Response) Close() { } // SetDebug set flag indicating that response can including debug information -func (r *Response) SetDebug(debug bool, obj *object.FileObject) *Response { - if debug == true { +func (r *Response) SetDebug(obj *object.FileObject) *Response { + if obj.Debug == true { r.debug = true - if obj != nil { - r.Headers.Set("Cache-Control", "no-cache") - r.Headers.Set("x-mort-key", obj.Key) - r.Headers.Set("x-mort-storage", obj.Storage.Kind) - - if obj.HasTransform() { - r.Headers.Set("x-mort-transform", "true") - } - - if obj.HasParent() { - r.Headers.Set("x-mort-parent-key", obj.Parent.Key) - r.Headers.Set("x-mort-parent-bucket", obj.Parent.Bucket) - r.Headers.Set("x-mort-parent-storage", obj.Parent.Storage.Kind) - } + r.Headers.Set("Cache-Control", "no-cache") + r.Headers.Set("x-mort-key", obj.Key) + r.Headers.Set("x-mort-storage", obj.Storage.Kind) + + if obj.HasTransform() { + r.Headers.Set("x-mort-transform", "true") + } + + if obj.HasParent() { + r.Headers.Set("x-mort-parent-key", obj.Parent.Key) + r.Headers.Set("x-mort-parent-bucket", obj.Parent.Bucket) + r.Headers.Set("x-mort-parent-storage", obj.Parent.Storage.Kind) } r.writeDebug() return r diff --git a/pkg/response/response_test.go b/pkg/response/response_test.go index c57537c..fba8c0c 100644 --- a/pkg/response/response_test.go +++ b/pkg/response/response_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "testing" + "github.com/aldor007/mort/pkg/object" ) func TestResponse_Copy(t *testing.T) { @@ -88,7 +89,7 @@ func TestNewError(t *testing.T) { assert.NotNil(t, err, "Should return error when reading body") assert.Nil(t, buf) - res.SetDebug(true, nil) + res.SetDebug(&object.FileObject{Debug: true}) buf, err = res.ReadBody() assert.Nil(t, err) assert.NotNil(t, buf, "Should return error when reading body") diff --git a/pkg/transforms/transforms.go b/pkg/transforms/transforms.go index b613b55..21105c2 100644 --- a/pkg/transforms/transforms.go +++ b/pkg/transforms/transforms.go @@ -6,9 +6,9 @@ import ( "hash" "strings" + "github.com/aldor007/mort/pkg/helpers" "github.com/spaolacci/murmur3" "gopkg.in/h2non/bimg.v1" - "github.com/aldor007/mort/pkg/helpers" ) var watermarkPosX = map[string]float32{ From 271c5d7d00926a51ff74de7a28d141da39bab425 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Fri, 9 Mar 2018 00:43:57 +0100 Subject: [PATCH 7/9] change hashing algorithm - add new folder level --- favicon.png | Bin 0 -> 21435 bytes pkg/object/uri.go | 13 +++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 favicon.png diff --git a/favicon.png b/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..4b8db29ac9a534d14f4d3ef487cb8ddff0ea70c5 GIT binary patch literal 21435 zcmeI4c{Egg{P?fPz80a7eajfjSVorY`%cJGV`eZihM^G25}~pr*&<8GlFAy&91RU?KyEDZgNrpAkr>fK;Z-mH!Uj<0H8X&_aOq3?y>*?r6EdH)yT*R$rkVXQW zYlGWQ0!ARObrn~apP65EUr+@d32)OuYz)8{o^iBrfB8ih7RoaF1>kdV9 zr%HjexwjT8ZhLIh)1xZ}2dAf|-sm){Si@Q^F7G(jpRKjncKPZrzqP*hu6lwuP{KA) zlVsyvefz3j7F*XXimUd+Gf|pbWO0Y9+!A`h z*RzY57!%nfRveQ89y==%`Nn`A*usgw=nBwVde%;?1Hf_>rtY;MIdOnf@OYQk?vm2B zW*Q$5;GliW9RTdqcty=xijky=u=x`V@YR9okyud47i)~nE~ zP$B}@u;+vdf>Z=UJ07zOp98->#SW=8b_$JkqZO?;e@d(1M)wUyo?0jPERcdkxrO{N zjO%R}k$r?EPb4{WV#o$pofgTV>%5#V!x)XZbz`Ko#7!d1IL)=#mld!Fpx10PDq~Kz z01`o4q0cnMZwD3`E4-#IRjqg+bM&f42GuKsr+8A-k+kGj*pj0PH>A^NU*$W|y^T^- zZ5FG2&q5~I=prWC)S&W2R)iuJ-u8sMfI7eF=;PA`#8u&I@UACbB!>dkYbm5jyi_=0 z9LidjdX@%nv>tG>@j@8hQpivcUp1~5;EsKuTf$$-@U@Y}UPAnurIrX!|DmYEV0zy$ z)kYcKL*c3t0u|8*w4dBJ>p9e;(c@~uGbfUp4ADBu6Z`Tl;(QSq^tK4+iI$M5l&OX( z$*E&gT+7y!DfXw_?{An+2%5Ay>#$DKPeZ0jkqHvY=9x(*BM;Ke7^I@ks60$BGkIiu z?ZIKlg(PT4@DXj@tcMwKy+@sxoz_v0-g{OujKpym#J{rWX`9o0AOD_s)r*=YlqtA5 z%u$dvnT3qSk=2XkQG#AJYwvZ|W-~Dv@r8uhM7l(bxutNKupd)QD`lKc9Dm#~VX#?l zUUuGS9+mlud7D{OuAS-SLJf0C)5%g%t^_9u3lxe8Az+#tRr9}py?RZc`#+xbkKAphcf4gzJlCZp=jRutYhg_YF;(2 zIg1IY)ph}?dYGJ1pXamR*e-*&&9>3sa@I7dElG>1c}a(;J6YskNV#L=*}=_2pN&;* z8&0Wa!Qx=?<5y!Uy5%$bG;V6p-9R}{G|L`{JQ${`in^>`W4wi*(dc* z>NR(?cVu+*CrTwNh!052CeJ1hB#)J!JL7#ur0hZ2Q>1p$)R}Cn#jPD>raa{9uXyZXRl`8X(perb92{fuaNu!ymRH_7 zY>8R%iz4>kq~fH!%&5eu)!BBL+VYn2Hp2!?>e-Mn)hj%^h}}!O-9SzNACV(*c%X8i z^?}WeOMYB6kxFty$S*~MAC+&URVXtlM+H3$ny!1u<<#;a`XSf)Y(i&DCyF-2$p3x+ zD>B#&?0SPgwLr+V5Q|WyNVhP{qut{1QeNG|svcu%-_-7?jV3#a8$us}bfkn8^F*EH z-^x@%DwUc<9jw~>EK@ACU3A%?qQZsJt}fMMTOU`hO{7dvP5Lq?fLmCxzoB+I29NxI01mQav;E*^XbmNU-_oS{YX?mk15lS~Isgq|iXkbfFcCXDVg%>QT+ zU+Djw|C0XsK)brupt@3`^9xkwl+z)X7lsEu+8SDdt;R}W-$GN4DKdpe6fzDm$1`1d zBRK3mee4>hwI*@+c1do2V+Eqip~qovW)zfaO?xLVekPvQm1gj53C-B)l8liOlu;9Qu%(+MC7eN`>Kha~pRy(Y6l_(C#0biF=5*$#`hX1}a|nvG*vLnMC!U zSkR&6r2-tt4Z(*RbJE1#0emOq22JM%%Jzu-P z`e>-BzKNsyrMING-RkMhk9A(dUhGSmV|0d7ewL{LB`OVqi%H9H%0NZMrDS9vG9sK3V&YO@ zF-fqb7)U}=4k9K8k>LDe<5r}^9m!)64ss@H8h?btNebLfo}O-UV6c~$m#CMdCgaioJ1LWc3>Iw4(xq9&YaPp@gHKYd|i*oZsVO%-){KD)p7d#cXx%UG7 zwf!+JwA){STs{6^hocDghPi>oMa97XVx+H6*bI&SM@tV+^^3SO`D0}N7|_Ga#|;TK zL3&^=VBtvhi%3^bo_{(Cw?CgG{@LAM>VF*t0{)k!+Xbx4cfbfZ80msU<6?T??8N_Z z44x>5pOf?35cV|x8V>1=`WLf3%@4EhsqkZl_ zL17RFr~?EJ5{E#LAczzc0gp@$w#)6c=jRh%#1VJGZP^7q+jEub)^v@9g;O1|k zv``+neDe7v&v4oHz4n+Q-G6WWQ{jU8UPIhqSP$gh(p2F7b8Y@rbpEO-dp6&fxEu_= zSA7)WdkKt0$b)}t{drn{S^i;$`ulYLHv~WY|6=~{F}$3Ru7uM6hpX?s{@sZO#=+AI zhD9nn;xgpFt<4{Pe>cPrAP?TF*Dk34g{km>UHt!>i2t^({*O$=&rF6p!CW1Y2u1Lp zN&IKSf1A4>Gy41L`g6tpeKpGetS*0617%!w+gr3AzvsmFMXLgH{JnMWwExJUf38_# z_DD&j6s|5xOF|LK-hMk?ZlhTnwW+X&^2bg@)J;Vy$7d*xgb7v;MR zzTzJZt$s0t{A&1%;t#_g$@R;9>-V(&(Tck(#@&#E|GFdpTgLx&5&54y_&K8gNw0Xm z_JiO7AuR6WBFqO*yN?SG2w`y_7hyhl+I?JjKnRQbxCryX)9&NK143Bb$3>VAo^~G> z9uUIfJ}$z1@U;85@PH5&_i+*CgQwlcg$IPNxQ~l4A3W_oE<7NF#eH0a`QT~yap3_W zEbikX%m+`qj|&e7VR0W9VLo`;eO!1z2#fo;2=l?y?&HD(LRj3#MVJqsb{`iW5W?a< zF2a29wEMX5fDjhBJt^a0=!9{>ad1Hji^+iCI{f0XjMwVCUP8z+onaj{qARD{>dN<@Xow?RF1*+uTgvd% zKs7PFu*%BG0r*%807Q(8hX4~3v)!XdkAClg``X?8?w$ze4cJ)!3XqcF{xt@`ozw6E}*Y2)Joj4VdJ6rR3SIs@@dDm%bZg9Q*3is2xld?VsPs2VPx~=Xotd9m}`&IB;pxvdPW(_4sGM+|%0<)+wK`9OavFVlFYf6iVOEL_YF~?g4i|qh@U6 zf-|##0iSj(q=YmQ-Tmry^bD4sXnu!%nQ{jtq!g%(4s#;uR?m7YC(^hM$s1X$g1pwyua); zXsN<>B=Z1KLiXV0DdWuc6HALVL?V-hTDQ#04dBQ-yorh0z3Z2St3tT7CYqPtxi3%? zKjZ*vmFMVH!tOrZdJ3Z_aZ|GOtKhiB)8PH&UV3vzg3gU&*r6{g29r#>y(D!PVr~$NHkX_!*3OU0e&4qvi!uWu8PQH&jioQnVGl_u+VLL>}YDl9ZjU z`|$M8E;86pC5iGKu%(cktm<3ZJ<3s5=6uBd3XIX&GSo81@WVC2*UON_}G`%v|*5j%57c;)o7+>N~V_V#>|uUPRt%YSazUrYR{` z1jdd&|+JazER;hpV#GP+>?`r zjR*&;jh&ok`41dT#c%cJ+ca(T?MhM%s%f$**yP$>xL@?XMTxR0XxjGCemH@CmS~6t zd*+Oq)z+p%-@LMH7_RDKQ4bD3&ybyJ_{2y|aOVbnil0VG?=_2j7_Xeh`)}%giyEPu z*S-kWoij3LoUR(Qe{t)2ckczZ{=N)Bj?R0Ex^pwmCXy7`mlS6CiF9XYUv2?4V}|^( ztVW);nORxpM{cTT#6EHwweGUy4O+xXt}GwCn0%AT`5+BVm5`9o_?3yzZ$`#k;?R0= z7g(CE?9R&5EzPEpanO>L^1<9WTP<5>S~|?<6Q}a%SkqLCwXj)V0@ZA31|ZhfM{^K^zW?e=)~7>qa}fHa-?f8kEhSXKs~S77A_bO_Tz2@?l(+YE5 zo#Wky1SYRUWeQSuYfWj~&LBN6J^(j%S7wZR9qxLQnq_#Da|+TsnL77)|w}8UOT3y5o$~JM)7cT9w@E| zUSukLp}eK3@Dbu`YR7J{)9%huRPmhdgNK)Q`7<($vS(6Ib)h3&ZP|-Q1{Zg%PA^?T zU$U!4#oo}!EO%{{&wBic3n)YTgclYTqOMb8SguR4s++nEPbQ=umvcK;+11)waKCWz zJsNuDYYS}!!aZ>Kj49vQf;WEFG>=WJYWy3x99~xzmD=p?2)D(9?7Q*=HhSed&l2PR#+$a23uO~U=V#2l)k zF}UKZ&GUZUMq1F46qT{rH%rW5;c`_0N=QrSvyXrnN*LvVe!V2$vOF81Rg&=8hyRsX z+|x#q`an*Fnw`4pW#{xH`%=rAAx(@XUE%pm`lN{}(~s0NwD7)Os80cn?z8ZTB8g6$ z>Km!hW3z9W?zii|2s;`Uu}q0+3`>Y@*Y*}T;D>z{{^71m(F)sb*MfILlbWcVXW?MQ zH>6y|`k|<*Ta8oqqXN_Y(^**Zw57V=ygK!{0d+Mr^7C}Hz{|(k65cy#>|$H<(t5Ao z1Z1f{<^Izmj$9Vrgu`wXHV;(D(OlU$%aWX&$)k zw$!8Y!Br+QNLwYH{&aramoL(0j|L{ilR;A;o5qFXJCX2lp7yF&QF$ou;ow#A@YJOG zPoKonjozQtpw#0(>@3CtN$j|jOxKWBQ*+O}Gmck4;4nGKs|)N6Vh$pzdN)GHj=wvu z(XJpnKOZr9Z`0T1OL=lfb(<10Bxl%opqx10y z$L8942EgPnx375eG2LHZce;a zl)jcFbHndLOSS4qtNWrT`B%(>foJouUDsh(42KRhjl@JpulLjpqlzG8gyrd6##326 z3&$Rr%XRHh1M<6v$2PA;t_&xw4JGaN`NaoXy4m(6TobM_Qt4-TP6OU z6j1}JqVcLuM|U0{PxLTqRfa}*>)tqwscv3mzPN~TmM=C5Q1E)W6}#@|H5lF7eDv!` zluTi#Lvh#BWg8Zi_navuLs(zs;$wf(c#7;Gi=ARnkW;{|`JOXg)qU849a}r!FXqqE{T6L1NUrc&j`a8u zjfgSCXmP0=K70Op`LWXCnyn9AS3cj!v3Hqfn!Qk#o_H2*Y1rQTvh|+-wMENWV9-&B zi#+b?;Od4($%<9K-|jWDeluH`$6~uxs-Hc2eMV_#{7HR%wC2D~{d&kvYiDQYFYI!w zv;*9_@nB?so)05w{ZtVYpzckpdUS1Ipt^mACYgdxc&kRXd{j=eZ{WC1`t+=?K!Npo zTOjUY*1>*#iI<*|j<o)^-$?Jrx6?5tAm-a(c#K_>K?8&5q>Xwe94)axUO=> zXi>i}ck$6CIdG6Ksc@lbHhg*5b*Bh@67yj+Knafbq$Kp@lmZ7`jK9?#hDW=Gb z4tz7s&Hr%r@>ixv4u7&{cR`V@cBt(@H>88)74t~-r=UvG@Ox3Yn-)rG99A`F=F*xh z42_Vgnt2h%JbhhuK019M>j$*Q%^5D!k_Z*YIY1)Tp3`g;?wT9AO7iV z%-H5zj_UPb&AFqgb<_SGj}WCv zR2gN-i8|U3gI%v#O=Z7~KFM$@k>%5axK^W2H@rnJ2!FWUF|;|x?avizk~TYc+uWx4 zrp!&A-gtfnLwag5cTGR}-A|-5=AHEmYgou&?Zma3)(X0$=t$Kxv9Lu6S)VCceF4MR zCz}i^VxFsRM?lZ7*DNWv%paudJIGyU8`@aM{OK#!fcE5Ib7Qgq7bLJ<&$JYC5a^W2k^tEHTsfu@a&^cO=3(<30Pn;LGMV?{Akpoc~lO`wcbg8aAbYc^d*-sJSyGH{n*&`qU8;WZH^N{BiY4kVxy)C zCtl)E2txX>nA#Z#%7rCocOi7Xk|as)A_j)4qD?+eOeTLm7@_Tl4Xg_}!p8+hW|9_9 zzfdqUNY+G;P*-N!+1v?Vl;%!BvxhOAIK2@)V!g z*L6SJ6}@Cbn#)WuZU}y2Ha;h~{QQXT{H`+RHrGiySF%gv*juBa#hz=c_VW>7QQxEx zuQAQqS^n5gBQvwr&7gqY1=4fGLbW{?QmEqJYL{MV4DDNCQLq=xoYf1_JN{4~B>(AQ z4(jE!?v8&1H$%n?h6rw0_2-Y8S~;6B zCWqJ=r)vW*UHst9&J*L*6{0eJ6_o@N;>&Cc{XEcgSpI7=`zpfLlkXmUH36HHJSuRA z!c^Q@)QaKXp5 zoC>})D)ZoB%9xDH8!?xm>`b1z4})jxA^N!0;?8Cx%ABcleXtyJe7j8FE|sbcdzhLe zH!m-v@S6|}fm*xA#=5KpAp3yFJ)FyoIk z4mhOpz(qRW)7D~Tw#y>mmWls233Jyrlb47?LRkmcPN9;rcO=U<#@fozIy#|{{{H^u bB_hC)=F!0**tgidM Date: Fri, 9 Mar 2018 01:14:45 +0100 Subject: [PATCH 8/9] prefetch placeholder --- pkg/config/config.go | 9 +++++++++ pkg/config/types.go | 1 + pkg/processor/processor.go | 11 +++-------- pkg/response/response_test.go | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index bd2a678..ec040c4 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/aldor007/mort/pkg/helpers" "github.com/aldor007/mort/pkg/monitoring" ) @@ -241,6 +242,14 @@ func (c *Config) validateServer() error { c.Server.QueueLen = 5 } + if c.Server.Placeholder != "" { + var err error + c.Server.PlaceholderBuf, err = helpers.FetchObject(c.Server.Placeholder) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/config/types.go b/pkg/config/types.go index ecb725d..21e4a3e 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -112,4 +112,5 @@ type Server struct { Listen []string `yaml:"listens"` Monitoring string `yaml:"monitoring"` Placeholder string `yaml:"placeholder"` + PlaceholderBuf []byte } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 94b5bd0..bcaf7f5 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -10,7 +10,6 @@ import ( "github.com/aldor007/mort/pkg/config" "github.com/aldor007/mort/pkg/engine" - "github.com/aldor007/mort/pkg/helpers" "github.com/aldor007/mort/pkg/lock" "github.com/aldor007/mort/pkg/monitoring" "github.com/aldor007/mort/pkg/object" @@ -96,17 +95,13 @@ func (r *RequestProcessor) replyWithError(obj *object.FileObject, sc int, err er return response.NewError(sc, err) } - key := r.serverConfig.Placeholder + strconv.FormatUint(obj.Transforms.Hash().Sum64(), 16) + strconv.FormatInt(int64(sc), 10) + key := r.serverConfig.Placeholder + strconv.FormatUint(obj.Transforms.Hash().Sum64(), 16) if cacheRes := r.fetchResponseFromCache(key); cacheRes != nil { + cacheRes.StatusCode = sc return cacheRes } - buf, err := helpers.FetchObject(r.serverConfig.Placeholder) - if err != nil { - return response.NewError(sc, err) - } - - parent := response.NewBuf(sc, buf) + parent := response.NewBuf(sc, r.serverConfig.PlaceholderBuf) transformsTab := []transforms.Transforms{obj.Transforms} eng := engine.NewImageEngine(parent) diff --git a/pkg/response/response_test.go b/pkg/response/response_test.go index fba8c0c..14bde46 100644 --- a/pkg/response/response_test.go +++ b/pkg/response/response_test.go @@ -3,13 +3,13 @@ package response import ( "bytes" "errors" + "github.com/aldor007/mort/pkg/object" "github.com/stretchr/testify/assert" "io" "io/ioutil" "net/http" "net/http/httptest" "testing" - "github.com/aldor007/mort/pkg/object" ) func TestResponse_Copy(t *testing.T) { From e47c1786fd8ed25d28b48208717284a2d0d32735 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Fri, 9 Mar 2018 01:35:52 +0100 Subject: [PATCH 9/9] add collapse on error --- pkg/processor/processor.go | 43 ++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index bcaf7f5..c119cb3 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -101,18 +101,43 @@ func (r *RequestProcessor) replyWithError(obj *object.FileObject, sc int, err er return cacheRes } - parent := response.NewBuf(sc, r.serverConfig.PlaceholderBuf) - transformsTab := []transforms.Transforms{obj.Transforms} + lockResult, locked := r.collapse.Lock(key) + if locked { + defer r.collapse.Release(key) + monitoring.Log().Info("Lock acquired", zap.String("obj.Key", obj.Key)) + parent := response.NewBuf(sc, r.serverConfig.PlaceholderBuf) + transformsTab := []transforms.Transforms{obj.Transforms} - eng := engine.NewImageEngine(parent) - res, err := eng.Process(obj, transformsTab) - res.StatusCode = sc - resCpy, errCpy := res.Copy() - if errCpy != nil { - r.cache.Set(key, resCpy, time.Minute*10) + eng := engine.NewImageEngine(parent) + res, errProcess := eng.Process(obj, transformsTab) + + if errProcess != nil { + return response.NewError(sc, err) + } + + res.StatusCode = sc + resCpy, errCpy := res.Copy() + if errCpy != nil { + r.cache.Set(key, resCpy, time.Minute*10) + } + return res + } + + timer := time.NewTimer(r.lockTimeout) + + for { + + select { + case <-timer.C: + return response.NewError(sc, err) + default: + if cacheRes := r.fetchResponseFromCache(key); cacheRes != nil { + lockResult.Cancel <- true + return cacheRes + } + } } - return res } func (r *RequestProcessor) process(req *http.Request, obj *object.FileObject) *response.Response {