diff --git a/cmd/mort/mort.go b/cmd/mort/mort.go index 00d62ab..17868d4 100644 --- a/cmd/mort/mort.go +++ b/cmd/mort/mort.go @@ -221,7 +221,7 @@ func main() { fmt.Printf(BANNER, version, commit, date) fmt.Printf("Config file %s listen addr %s montoring: and debug listen %s pid: %d \n", *configPath, imgConfig.Server.Listen, imgConfig.Server.InternalListen, os.Getpid()) - rp := processor.NewRequestProcessor(imgConfig.Server, lock.NewMemoryLock(), throttler.NewBucketThrottler(10)) + rp := processor.NewRequestProcessor(imgConfig.Server, lock.Create(imgConfig.Server.Lock), throttler.NewBucketThrottler(10)) cloudinaryUploadInterceptor := cloudinary.NewUploadInterceptorMiddleware(imgConfig) router.Use(cloudinaryUploadInterceptor.Handler) diff --git a/configuration/config.yml b/configuration/config.yml index 18c6209..7a33fb1 100644 --- a/configuration/config.yml +++ b/configuration/config.yml @@ -6,6 +6,11 @@ server: - "localhost:6379" maxCacheItemSizeMB: 50 clientConfig: + lock: + type: "redis" + address: + - "localhost:6379" + clientConfig: listens: - ":8084" diff --git a/go.mod b/go.mod index 26362d1..8c77f57 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,12 @@ require ( github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 github.com/aldor007/stow v1.1.0 github.com/aws/aws-sdk-go v1.38.57 // indirect + github.com/bsm/redislock v0.7.2 github.com/d5/tengo/v2 v2.10.1 github.com/djherbis/stream v1.3.1 github.com/go-chi/chi v1.5.2 github.com/go-redis/cache/v8 v8.3.1-0.20210129151214-1cdfea0b552b - github.com/go-redis/redis/v8 v8.8.0 + github.com/go-redis/redis/v8 v8.11.4 github.com/go-redis/redismock/v8 v8.0.6 github.com/google/brotli/go/cbrotli v0.0.0-20210127140805-63be8a994019 github.com/h2non/bimg v1.1.6-0.20210807160419-b29a57356338 @@ -25,7 +26,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/vmihailenco/msgpack v4.0.4+incompatible go.uber.org/zap v1.16.0 - golang.org/x/net v0.0.0-20210119194325-5f4716e94777 + golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 gopkg.in/h2non/gock.v1 v1.0.16 gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 // indirect gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index a0342da..b10def5 100644 --- a/go.sum +++ b/go.sum @@ -38,11 +38,14 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bsm/redislock v0.7.2 h1:jggqOio8JyX9FJBKIfjF3fTxAu/v7zC5mAID9LveqG4= +github.com/bsm/redislock v0.7.2/go.mod h1:kS2g0Yvlymc9Dz8V3iVYAtLAaSVruYbAFdYBDrmC5WU= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -92,12 +95,14 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-redis/cache/v8 v8.3.1-0.20210129151214-1cdfea0b552b h1:rbIQkTQmFym9UIXf1xYGRFIt2mW2fgNqTi/Q+tJmF1c= github.com/go-redis/cache/v8 v8.3.1-0.20210129151214-1cdfea0b552b/go.mod h1:mmEp4GotPrc2iq1ch5hEQ3zMPFQDmFpJWEKZbI7545A= github.com/go-redis/redis/v8 v8.4.11/go.mod h1:d5yY/TlkQyYBSBHnXUmnf1OrHbyQere5JV4dLKwvXmo= -github.com/go-redis/redis/v8 v8.8.0 h1:fDZP58UN/1RD3DjtTXP/fFZ04TFohSYhjZDkcDe2dnw= github.com/go-redis/redis/v8 v8.8.0/go.mod h1:F7resOH5Kdug49Otu24RjHWwgK7u9AmtqWMnCV1iP5Y= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-redis/redismock/v8 v8.0.6 h1:rtuijPgGynsRB2Y7KDACm09WvjHWS4RaG44Nm7rcj4Y= github.com/go-redis/redismock/v8 v8.0.6/go.mod h1:sDIF73OVsmaKzYe/1FJXGiCQ4+oHYbzjpaL9Vor0sS4= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -118,8 +123,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/brotli/go/cbrotli v0.0.0-20210127140805-63be8a994019 h1:XYW4NntIMcMzsu+XjMKziKuSgthVc/nSnDrFu/iJuzA= github.com/google/brotli/go/cbrotli v0.0.0-20210127140805-63be8a994019/go.mod h1:nOPhAkwVliJdNTkj3gXpljmWhjc4wCaVqbMJcPKWP4s= @@ -131,8 +138,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/readahead v0.0.0-20161222183148-eaceba169032 h1:6Be3nkuJFyRfCgr6qTIzmRp8y9QwDIbqy/nYr9WDPos= github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= @@ -242,8 +250,9 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/ncw/swift v1.0.43/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -251,14 +260,16 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= -github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -378,13 +389,9 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/otel v0.16.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= -go.opentelemetry.io/otel v0.19.0 h1:Lenfy7QHRXPZVsw/12CWpxX6d/JkrX8wrx2vO8G80Ng= go.opentelemetry.io/otel v0.19.0/go.mod h1:j9bF567N9EfomkSidSfmMwIwIBuP37AMAIzVW85OxSg= -go.opentelemetry.io/otel/metric v0.19.0 h1:dtZ1Ju44gkJkYvo+3qGqVXmf88tc+a42edOywypengg= go.opentelemetry.io/otel/metric v0.19.0/go.mod h1:8f9fglJPRnXuskQmKpnad31lcLJ2VmNNqIsx/uIwBSc= -go.opentelemetry.io/otel/oteltest v0.19.0 h1:YVfA0ByROYqTwOxqHVZYZExzEpfZor+MU1rU+ip2v9Q= go.opentelemetry.io/otel/oteltest v0.19.0/go.mod h1:tI4yxwh8U21v7JD6R3BcA/2+RBoTKFexE/PJ/nSO7IA= -go.opentelemetry.io/otel/trace v0.19.0 h1:1ucYlenXIDA1OlHVLDZKX0ObXV5RLaq06DtUKz5e5zc= go.opentelemetry.io/otel/trace v0.19.0/go.mod h1:4IXiNextNOpPnRlI4ryK69mn5iC84bjBWZQA5DXz/qg= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -452,8 +459,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -493,13 +500,15 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -562,8 +571,10 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/cache/redis.go b/pkg/cache/redis.go index 33df372..65a5e7a 100644 --- a/pkg/cache/redis.go +++ b/pkg/cache/redis.go @@ -65,6 +65,7 @@ func NewRedisCluster(redisAddress []string, clientConfig map[string]string) *Red return &RedisCache{cache} } + func (c *RedisCache) getKey(obj *object.FileObject) string { return "mort-v1:" + obj.GetResponseCacheKey() } diff --git a/pkg/config/config.go b/pkg/config/config.go index 2d69994..e5ea4ca 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -299,10 +299,6 @@ func (c *Config) validateServer() error { c.Server.LockTimeout = 30 } - if c.Server.QueueLen == 0 { - c.Server.QueueLen = 5 - } - if c.Server.Cache.MaxCacheItemSize == 0 { c.Server.Cache.MaxCacheItemSize = 5 * 2 << 20 } else { diff --git a/pkg/config/types.go b/pkg/config/types.go index 9a4a1ba..c9e4aaa 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -155,15 +155,21 @@ type CacheCfg struct { ClientConfig map[string]string `yaml:"clientConfig"` } +// LockCfg configure redis lock +type LockCfg struct { + Type string `yaml:"type"` + Address []string `yaml:"address"` + ClientConfig map[string]string `yaml:"clientConfig"` +} + // Server configure HTTP server type Server struct { - LogLevel string `yaml:"logLevel"` - InternalListen string `yaml:"internalListen"` - SingleListen string `yaml:"listen"` - RequestTimeout int `yaml:"requestTimeout"` - LockTimeout int `yaml:"lockTimeout"` - // Unused, intention unknown - QueueLen int `yaml:"queueLen"` + LogLevel string `yaml:"logLevel"` + InternalListen string `yaml:"internalListen"` + SingleListen string `yaml:"listen"` + RequestTimeout int `yaml:"requestTimeout"` + LockTimeout int `yaml:"lockTimeout"` + Lock *LockCfg `yaml:"redisLock"` Listen []string `yaml:"listens"` Monitoring string `yaml:"monitoring"` PlaceholderStr string `yaml:"placeholder"` diff --git a/pkg/lock/lock.go b/pkg/lock/lock.go index 86f104e..3e9f389 100644 --- a/pkg/lock/lock.go +++ b/pkg/lock/lock.go @@ -1,23 +1,27 @@ package lock import ( + "context" + + "github.com/aldor007/mort/pkg/config" "github.com/aldor007/mort/pkg/response" ) // Lock is responding for collapsing request for same object type Lock interface { // Lock try get a lock for given key - Lock(key string) (observer LockResult, acquired bool) + Lock(ctx context.Context, key string) (observer LockResult, acquired bool) // Release remove lock for given key - Release(key string) + Release(ctx context.Context, key string) // NotifyAndRelease remove lock for given key and notify all clients waiting for result - NotifyAndRelease(key string, res *response.Response) + NotifyAndRelease(ctx context.Context, key string, res *response.Response) } // LockResult contain struct type LockResult struct { ResponseChan chan *response.Response // channel on which you get response Cancel chan bool // channel for notify about cancel of waiting + Error error // error when creating error } type lockData struct { @@ -44,16 +48,30 @@ type NopLock struct { } // Lock always return that lock was acquired -func (l *NopLock) Lock(_ string) (LockResult, bool) { +func (l *NopLock) Lock(_ context.Context, _ string) (LockResult, bool) { return LockResult{}, true } // Release do nothing -func (l *NopLock) Release(_ string) { +func (l *NopLock) Release(_ context.Context, _ string) { } // NotifyAndRelease do nothing -func (l *NopLock) NotifyAndRelease(_ string, _ *response.Response) { +func (l *NopLock) NotifyAndRelease(_ context.Context, _ string, _ *response.Response) { + +} +func Create(lockCfg *config.LockCfg) Lock { + if lockCfg == nil { + return NewMemoryLock() + } + switch lockCfg.Type { + case "redis": + return NewRedisLock(lockCfg.Address, lockCfg.ClientConfig) + case "redis-cluster": + return NewRedisCluster(lockCfg.Address, lockCfg.ClientConfig) + default: + return NewMemoryLock() + } } diff --git a/pkg/lock/lock_test.go b/pkg/lock/lock_test.go index 4b20a3e..b4cee16 100644 --- a/pkg/lock/lock_test.go +++ b/pkg/lock/lock_test.go @@ -1,27 +1,30 @@ package lock import ( + "context" + "testing" + "github.com/aldor007/mort/pkg/response" "github.com/stretchr/testify/assert" - "testing" ) func TestNewNopLock(t *testing.T) { l := NewNopLock() - _, locked := l.Lock("a") + ctx := context.Background() + _, locked := l.Lock(ctx, "a") assert.True(t, locked) - _, locked = l.Lock("a") + _, locked = l.Lock(ctx, "a") assert.True(t, locked) - l.NotifyAndRelease("a", response.NewNoContent(200)) + l.NotifyAndRelease(ctx, "a", response.NewNoContent(200)) - _, locked = l.Lock("a") + _, locked = l.Lock(ctx, "a") assert.True(t, locked) - l.Release("a") + l.Release(ctx, "a") - _, locked = l.Lock("a") + _, locked = l.Lock(ctx, "a") assert.True(t, locked) } diff --git a/pkg/lock/memory.go b/pkg/lock/memory.go index 4f4359d..e97c9c4 100644 --- a/pkg/lock/memory.go +++ b/pkg/lock/memory.go @@ -1,6 +1,7 @@ package lock import ( + "context" "sync" "github.com/aldor007/mort/pkg/monitoring" @@ -42,7 +43,7 @@ func notifyListeners(lock lockData, respFactory func() (*response.Response, bool } // NotifyAndRelease tries notify all waiting goroutines about response -func (m *MemoryLock) NotifyAndRelease(key string, originalResponse *response.Response) { +func (m *MemoryLock) NotifyAndRelease(_ context.Context, key string, originalResponse *response.Response) { m.lock.Lock() lock, ok := m.internal[key] if !ok { @@ -62,7 +63,7 @@ func (m *MemoryLock) NotifyAndRelease(key string, originalResponse *response.Res // Current synchronous notification is simpler compared to asynchronous implementation. // The asynchronous implementation might be tricky since the response in not buffered mode must be // protected from being read before it is copied. Otherwise CopyWithStream in a worst case will deliver partial body - // since it can read in parallel with HTTP handler. To prevent such behaviour extra temporary copy of response + // since it can read in parallel with HTTP handler. To prevent such behavior extra temporary copy of response // must be created before returning from this method. Of course such creation must // also take into account whether the originalResponse is buffered or not. // The time spend on notifying listeners is negligible compared to the total time of image processing, @@ -79,7 +80,7 @@ func (m *MemoryLock) NotifyAndRelease(key string, originalResponse *response.Res } // Lock create unique entry in memory map -func (m *MemoryLock) Lock(key string) (LockResult, bool) { +func (m *MemoryLock) Lock(_ context.Context, key string) (LockResult, bool) { m.lock.Lock() defer m.lock.Unlock() lock, ok := m.internal[key] @@ -95,7 +96,7 @@ func (m *MemoryLock) Lock(key string) (LockResult, bool) { } // Release remove entry from memory map -func (m *MemoryLock) Release(key string) { +func (m *MemoryLock) Release(_ context.Context, key string) { m.lock.RLock() _, ok := m.internal[key] m.lock.RUnlock() diff --git a/pkg/lock/memory_test.go b/pkg/lock/memory_test.go index 581634a..66dab53 100644 --- a/pkg/lock/memory_test.go +++ b/pkg/lock/memory_test.go @@ -1,11 +1,13 @@ package lock import ( + "context" "errors" - "github.com/aldor007/mort/pkg/response" - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/aldor007/mort/pkg/response" + "github.com/stretchr/testify/assert" ) func TestNewMemoryLock(t *testing.T) { @@ -17,19 +19,20 @@ func TestNewMemoryLock(t *testing.T) { func TestMemoryLock_Lock(t *testing.T) { l := NewMemoryLock() key := "klucz" - c, acquired := l.Lock(key) + ctx := context.Background() + c, acquired := l.Lock(ctx, key) assert.True(t, acquired, "Should acquire lock") assert.Nil(t, c.ResponseChan, "shouldn't return channel") - resChan, lock := l.Lock(key) + resChan, lock := l.Lock(ctx, key) assert.False(t, lock, "Shouldn't acquire lock") assert.NotNil(t, resChan, "should return channel") - l.Release(key) + l.Release(ctx, key) - c, acquired = l.Lock(key) + c, acquired = l.Lock(ctx, key) assert.True(t, acquired, "Should acquire lock after release") assert.Nil(t, c.ResponseChan, "shouldn't return channel after release") @@ -38,17 +41,18 @@ func TestMemoryLock_Lock(t *testing.T) { func TestMemoryLock_NotifyAndReleaseWhenError(t *testing.T) { l := NewMemoryLock() key := "kluczi2" - c, acquired := l.Lock(key) + ctx := context.Background() + c, acquired := l.Lock(ctx, key) assert.True(t, acquired, "Should acquire lock") assert.Nil(t, c.ResponseChan, "shouldn't return channel") - result, lock := l.Lock(key) + result, lock := l.Lock(ctx, key) assert.False(t, lock, "Shouldn't acquire lock") assert.NotNil(t, result, "should return channel") - go l.NotifyAndRelease(key, response.NewError(400, errors.New("invalid transform"))) + go l.NotifyAndRelease(ctx, key, response.NewError(400, errors.New("invalid transform"))) timer := time.NewTimer(time.Second * 2) select { @@ -66,18 +70,19 @@ func TestMemoryLock_NotifyAndReleaseWhenError(t *testing.T) { func TestMemoryLock_NotifyAndRelease(t *testing.T) { l := NewMemoryLock() key := "kluczi22" - c, acquired := l.Lock(key) + ctx := context.Background() + c, acquired := l.Lock(ctx, key) assert.True(t, acquired, "Should acquire lock") assert.Nil(t, c.ResponseChan, "shouldn't return channel") - result, lock := l.Lock(key) + result, lock := l.Lock(ctx, key) assert.False(t, lock, "Shouldn't acquire lock") assert.NotNil(t, result.ResponseChan, "should return channel") buf := make([]byte, 1000) - go l.NotifyAndRelease(key, response.NewBuf(200, buf)) + go l.NotifyAndRelease(ctx, key, response.NewBuf(200, buf)) timer := time.NewTimer(time.Second * 2) select { @@ -99,30 +104,32 @@ func TestMemoryLock_NotifyAndRelease(t *testing.T) { func TestMemoryLock_NotifyAndRelease2(t *testing.T) { l := NewMemoryLock() key := "kluczi222" - c, acquired := l.Lock(key) + ctx := context.Background() + c, acquired := l.Lock(ctx, key) assert.True(t, acquired, "Should acquire lock") assert.Nil(t, c.ResponseChan, "shouldn't return channel") - l.NotifyAndRelease("no-key", response.NewError(400, errors.New("invalid transform"))) - l.NotifyAndRelease(key, response.NewNoContent(200)) + l.NotifyAndRelease(ctx, "no-key", response.NewError(400, errors.New("invalid transform"))) + l.NotifyAndRelease(ctx, key, response.NewNoContent(200)) } func BenchmarkMemoryLock_NotifyAndRelease(b *testing.B) { l := NewMemoryLock() key := "aaa" + ctx := context.Background() buf := make([]byte, 10) - l.Lock(key) + l.Lock(ctx, key) go time.AfterFunc(time.Millisecond*time.Duration(500), func() { - l.NotifyAndRelease(key, response.NewBuf(200, buf)) + l.NotifyAndRelease(ctx, key, response.NewBuf(200, buf)) }) for i := 0; i < b.N; i++ { - result, acquired := l.Lock(key) + result, acquired := l.Lock(ctx, key) multi := 500 % (i + 1) if acquired { go time.AfterFunc(time.Millisecond*time.Duration(multi), func() { - l.NotifyAndRelease(key, response.NewBuf(200, buf)) + l.NotifyAndRelease(ctx, key, response.NewBuf(200, buf)) }) } else { go func(r LockResult) { diff --git a/pkg/lock/redis.go b/pkg/lock/redis.go new file mode 100644 index 0000000..a84e353 --- /dev/null +++ b/pkg/lock/redis.go @@ -0,0 +1,108 @@ +package lock + +import ( + "sync" + "time" + + "github.com/aldor007/mort/pkg/response" + "github.com/bsm/redislock" + goRedis "github.com/go-redis/redis/v8" + + "context" + "strings" +) + +func parseAddress(addrs []string) map[string]string { + mp := make(map[string]string, len(addrs)) + + for _, addr := range addrs { + parts := strings.Split(addr, ":") + mp[parts[0]] = parts[0] + ":" + parts[1] + } + + return mp +} + +// RedisLock is in Redis lock for single mort instance +type RedisLock struct { + client *redislock.Client + memoryLock *MemoryLock + locks map[string]*redislock.Lock + lock sync.RWMutex +} + +// NewRedis create connection to redis and update it config from clientConfig map +func NewRedisLock(redisAddress []string, clientConfig map[string]string) *RedisLock { + ring := goRedis.NewRing(&goRedis.RingOptions{ + Addrs: parseAddress(redisAddress), + }) + + if clientConfig != nil { + for key, value := range clientConfig { + ring.ConfigSet(context.Background(), key, value) + } + } + + locker := redislock.New(ring) + + return &RedisLock{client: locker, memoryLock: NewMemoryLock(), locks: make(map[string]*redislock.Lock)} +} + +func NewRedisCluster(redisAddress []string, clientConfig map[string]string) *RedisLock { + ring := goRedis.NewClusterClient(&goRedis.ClusterOptions{ + Addrs: redisAddress, + }) + if clientConfig != nil { + for key, value := range clientConfig { + ring.ConfigSet(context.Background(), key, value) + } + } + + locker := redislock.New(ring) + + return &RedisLock{client: locker, memoryLock: NewMemoryLock(), locks: make(map[string]*redislock.Lock)} +} + +// NotifyAndRelease tries notify all waiting goroutines about response +func (m *RedisLock) NotifyAndRelease(ctx context.Context, key string, originalResponse *response.Response) { + m.lock.Lock() + defer m.lock.Unlock() + lock, ok := m.locks[key] + if ok { + lock.Release(ctx) + delete(m.locks, key) + } + + m.memoryLock.NotifyAndRelease(ctx, key, originalResponse) +} + +// Lock create unique entry in Redis map +func (m *RedisLock) Lock(ctx context.Context, key string) (result LockResult, ok bool) { + m.lock.Lock() + defer m.lock.Unlock() + lock, ok := m.locks[key] + if ok { + lock.Refresh(ctx, time.Millisecond*500, nil) + } else { + lock, err := m.client.Obtain(ctx, key, 60*time.Second, nil) + if err != nil { + result.Error = err + return + } + m.locks[key] = lock + + } + return m.memoryLock.Lock(ctx, key) +} + +// Release remove entry from Redis map +func (m *RedisLock) Release(ctx context.Context, key string) { + m.lock.Lock() + defer m.lock.Unlock() + lock, ok := m.locks[key] + if ok { + lock.Release(ctx) + delete(m.locks, key) + m.memoryLock.Release(ctx, key) + } +} diff --git a/pkg/lock/redis_test.go b/pkg/lock/redis_test.go new file mode 100644 index 0000000..6328a88 --- /dev/null +++ b/pkg/lock/redis_test.go @@ -0,0 +1,140 @@ +package lock + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/aldor007/mort/pkg/response" + "github.com/bsm/redislock" + "github.com/go-redis/redismock/v8" + "github.com/stretchr/testify/assert" +) + +func TestNewRedisLock(t *testing.T) { + l := NewRedisLock([]string{"1.1.1.1:1234"}, nil) + + assert.NotNil(t, l, "New Redis should return not nil") +} + +func TestRedisLock_Lock(t *testing.T) { + l := NewRedisLock([]string{"1.1.1.1:1234"}, nil) + client, mock := redismock.NewClientMock() + l.client = redislock.New(client) + key := "klucz" + mock.Regexp().ExpectSetNX(key, `[a-zA-Z0-9]+`, 60*time.Second).SetVal(true) + ctx := context.Background() + c, acquired := l.Lock(ctx, key) + + assert.True(t, acquired, "Should acquire lock") + assert.Nil(t, c.ResponseChan, "shouldn't return channel") + assert.Nil(t, c.Error, "error should be nil") + + resChan, lock := l.Lock(ctx, key) + + assert.False(t, lock, "Shouldn't acquire lock") + assert.NotNil(t, resChan, "should return channel") + + mock.ClearExpect() + mock.Regexp().ExpectSetNX(key, `[a-zA-Z0-9]+`, 60*time.Second).SetVal(true) + l.Release(ctx, key) + + c, acquired = l.Lock(ctx, key) + + assert.True(t, acquired, "Should acquire lock after release") + assert.Nil(t, c.ResponseChan, "shouldn't return channel after release") + assert.Nil(t, c.Error, "error should be nil") +} + +func TestRedisLock_NotifyAndReleaseWhenError(t *testing.T) { + l := NewRedisLock([]string{"1.1.1.1:1234"}, nil) + client, mock := redismock.NewClientMock() + l.client = redislock.New(client) + key := "kluczi2" + ctx := context.Background() + mock.Regexp().ExpectSetNX(key, `[a-zA-Z0-9]+`, 60*time.Second).SetVal(true) + c, acquired := l.Lock(ctx, key) + + assert.True(t, acquired, "Should acquire lock") + assert.Nil(t, c.ResponseChan, "shouldn't return channel") + assert.Nil(t, c.Error, "error should be nil") + + mock.ClearExpect() + mock.Regexp().ExpectSetNX(key, `[a-zA-Z0-9]+`, 60*time.Second).SetVal(true) + result, lock := l.Lock(ctx, key) + + assert.False(t, lock, "Shouldn't acquire lock") + assert.NotNil(t, result, "should return channel") + + go l.NotifyAndRelease(ctx, key, response.NewError(400, errors.New("invalid transform"))) + + timer := time.NewTimer(time.Second * 2) + select { + case <-timer.C: + t.Fatalf("Timeout while waiting for response propagation") + return + case res := <-result.ResponseChan: + assert.NotNil(t, res, "Response shouldn't be nil") + if res != nil { + assert.Equal(t, res.StatusCode, 400) + } + } +} + +func TestRedisLock_NotifyAndRelease(t *testing.T) { + l := NewRedisLock([]string{"1.1.1.1:1234"}, nil) + client, mock := redismock.NewClientMock() + l.client = redislock.New(client) + key := "kluczi22" + ctx := context.Background() + mock.ClearExpect() + mock.Regexp().ExpectSetNX(key, `[a-zA-Z0-9]+`, 60*time.Second).SetVal(true) + c, acquired := l.Lock(ctx, key) + + assert.True(t, acquired, "Should acquire lock") + assert.Nil(t, c.ResponseChan, "shouldn't return channel") + + mock.ClearExpect() + mock.Regexp().ExpectSetNX(key, `[a-zA-Z0-9]+`, 60*time.Second).SetVal(true) + result, lock := l.Lock(ctx, key) + + assert.False(t, lock, "Shouldn't acquire lock") + assert.NotNil(t, result.ResponseChan, "should return channel") + + buf := make([]byte, 1000) + go l.NotifyAndRelease(ctx, key, response.NewBuf(200, buf)) + + timer := time.NewTimer(time.Second * 2) + select { + case <-timer.C: + t.Fatalf("timeout waiting for lock") + return + case res := <-result.ResponseChan: + assert.NotNil(t, res, "Response should't be nil") + if res != nil { + assert.Equal(t, res.StatusCode, 200, "Response should have sc = 200") + } + buf2, err := res.Body() + assert.Nil(t, err) + assert.Equal(t, len(buf), len(buf2)) + + } +} + +func TestRedisLock_NotifyAndRelease2(t *testing.T) { + l := NewRedisCluster([]string{"1.1.1.1:1234"}, nil) + client, mock := redismock.NewClientMock() + l.client = redislock.New(client) + key := "kluczi222" + ctx := context.Background() + mock.ClearExpect() + mock.Regexp().ExpectSetNX(key, `[a-zA-Z0-9]+`, 60*time.Second).SetVal(true) + c, acquired := l.Lock(ctx, key) + + assert.True(t, acquired, "Should acquire lock") + assert.Nil(t, c.ResponseChan, "shouldn't return channel") + + l.NotifyAndRelease(ctx, "no-key", response.NewError(400, errors.New("invalid transform"))) + l.NotifyAndRelease(ctx, key, response.NewNoContent(200)) +} diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 83d2d77..7239a0c 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -122,9 +122,9 @@ func (r *RequestProcessor) replyWithError(obj *object.FileObject, sc int, err er } go func() { - lockData, locked := r.collapse.Lock(errorObject.Key) + lockData, locked := r.collapse.Lock(obj.Ctx, errorObject.Key) if locked { - defer r.collapse.Release(errorObject.Key) + defer r.collapse.Release(obj.Ctx, errorObject.Key) monitoring.Log().Info("Lock acquired for error response", obj.LogData()...) parent := response.NewBuf(200, r.serverConfig.Placeholder.Buf) transformsTab := []transforms.Transforms{obj.Transforms} @@ -201,14 +201,19 @@ func handlePUT(req *http.Request, obj *object.FileObject) *response.Response { func (r *RequestProcessor) collapseGET(req *http.Request, obj *object.FileObject) *response.Response { ctx := obj.Ctx - lockResult, locked := r.collapse.Lock(obj.Key) + lockResult, locked := r.collapse.Lock(ctx, obj.Key) if locked { monitoring.Log().Info("Lock acquired", obj.LogData()...) res := r.handleGET(req, obj) - r.collapse.NotifyAndRelease(obj.Key, res) + r.collapse.NotifyAndRelease(ctx, obj.Key, res) return res } + if lockResult.Error != nil { + monitoring.Log().Warn("Error acquiring lock", obj.LogData()...) + return r.replyWithError(obj, 500, lockResult.Error) + } + monitoring.Report().Inc("collapsed_count") monitoring.Log().Info("Lock not acquired", obj.LogData()...) timer := time.NewTimer(r.lockTimeout)