From d3ce8a54834001d6e3bf0dfa624fdba66fc9f78f Mon Sep 17 00:00:00 2001 From: Marc Ole Bulling Date: Thu, 11 Aug 2022 15:34:27 +0200 Subject: [PATCH] Migrate to AWS SDK v2 (not functional yet) --- go.mod | 19 ++- go.sum | 52 ++++++--- internal/storage/cloudstorage/aws/AwsS3.go | 130 ++++++++++++--------- 3 files changed, 135 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index 35b377b4..74b31f37 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,11 @@ go 1.19 require ( git.mills.io/prologic/bitcask v1.0.2 github.com/NYTimes/gziphandler v1.1.1 - github.com/aws/aws-sdk-go v1.42.22 + github.com/aws/aws-sdk-go-v2 v1.16.10 + github.com/aws/aws-sdk-go-v2/config v1.16.0 + github.com/aws/aws-sdk-go-v2/credentials v1.12.12 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.24 + github.com/aws/aws-sdk-go-v2/service/s3 v1.27.4 github.com/caarlos0/env/v6 v6.9.1 github.com/jinzhu/copier v0.3.5 github.com/johannesboyne/gofakes3 v0.0.0-20210415062230-4b6b67a85d38 @@ -20,6 +24,19 @@ require ( require ( github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.16.12 // indirect + github.com/aws/smithy-go v1.12.1 // indirect github.com/gofrs/flock v0.8.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum index 002078ff..0cbd54e2 100644 --- a/go.sum +++ b/go.sum @@ -53,9 +53,44 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.17.4 h1:L2KFocQhg48kIzEAV98SnSz3nmIZ3UDFP+vU647KO3c= github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.42.22 h1:EwcM7/+Ytg6xK+jbeM2+f9OELHqPiEiEKetT/GgAr7I= -github.com/aws/aws-sdk-go v1.42.22/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go-v2 v1.16.10 h1:+yDD0tcuHRQZgqONkpDwzepqmElQaSlFPymHRHR9mrc= +github.com/aws/aws-sdk-go-v2 v1.16.10/go.mod h1:WTACcleLz6VZTp7fak4EO5b9Q4foxbn+8PIz3PmyKlo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4 h1:zfT11pa7ifu/VlLDpmc5OY2W4nYmnKkFDGeMVnmqAI0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4/go.mod h1:ES0I1GBs+YYgcDS1ek47Erbn4TOL811JKqBXtgzqyZ8= +github.com/aws/aws-sdk-go-v2/config v1.16.0 h1:LxHC50cwOLxYo67NEpwpNUiOi6ngXfDpEETphSZ6bAw= +github.com/aws/aws-sdk-go-v2/config v1.16.0/go.mod h1:eatrtwIm5WdvASoYCy5oPkinfiwiYFg2jLG9tJoKzkE= +github.com/aws/aws-sdk-go-v2/credentials v1.12.12 h1:iShu6VaWZZZfUZvlGtRjl+g1lWk44g1QmiCTD4KS0jI= +github.com/aws/aws-sdk-go-v2/credentials v1.12.12/go.mod h1:vFHC2HifIWHebmoVsfpqliKuqbAY2LaVlvy03JzF4c4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.11 h1:zZHPdM2x09/0F8D7XyVvQnP2/jaW7bEMmtcSCPYq/iI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.11/go.mod h1:38Asv/UyQbDNpSXCurZRlDMjzIl6J+wUe8vY3TtUuzA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.24 h1:9MflwbI3Ua4PFyCNo39nnJ2ZYaQ/GabPUPdutegSJUs= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.24/go.mod h1:W970x9QKWWb0Y30Num5dFFji/qRQSt0UP4UzbM3sYCo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17 h1:U8DZvyFFesBmK62dYC6BRXm4Cd/wPP3aPcecu3xv/F4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.17/go.mod h1:6qtGip7sJEyvgsLjphRZWF9qPe3xJf1mL/MM01E35Wc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11 h1:GMp98usVW5tzQhxd26KWhoNQPlR2noIlfbzqjVGBhLU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.11/go.mod h1:cYAfnB+9ZkmZWpQWmPDsuIGm4EA+6k2ZVtxKjw/XJBY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.18 h1:/spg6h3tG4pefphbvhpgdMtFMegSajPPSEJd1t8lnpc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.18/go.mod h1:hTHq8hL4bAxJyng364s9d4IUGXZOs7Y5LSqAhIiIQ2A= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.8 h1:9PY5a+kHQzC6d9eR+KLNSJP3DHDLYmPFA5/+eSDBo9o= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.8/go.mod h1:pcQfUOFVK4lMnSzgX3dCA81UsA9YCilRUSYgkjSU2i8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.4 h1:akfcyqM9SvrBKWZOkBcXAGDrHfKaEP4Aca8H/bCiLW8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.4/go.mod h1:oehQLbMQkppKLXvpx/1Eo0X47Fe+0971DXC9UjGnKcI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.12 h1:eNQYkKjDSLDjIbBQ85rIkjpBGgnavrl/U3YKDdxAz14= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.12/go.mod h1:k2HaF2yfT082M+kKo3Xdf4rd5HGKvDmrPC5Kwzc2KUw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.11 h1:GkYtp4gi4wdWUV+pPetjk5y2aDxbr0t8n5OjVBwZdII= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.11/go.mod h1:OEofCUKF7Hri4ShOCokF6k6hGq9PCB2sywt/9rLSXjY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.11 h1:ZBLEKweAzBBtJa8H+MTFfVyvo+eHdM8xec5oTm9IlqI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.11/go.mod h1:mNS1VHxYXPNqxIdCTxf87j9ROfTMa4fNpIkA+iAfz0g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.27.4 h1:0RPAahwT63znFepvhfS+/WYtT+gEuAwaeNcCrzTQMH0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.27.4/go.mod h1:wcpDmROpK5W7oWI6JcJIYGrVpHbF/Pu+FHxyBXyoa1E= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.15 h1:HaIE5/TtKr66qZTJpvMifDxH4lRt2JZawbkLYOo1F+Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.15/go.mod h1:dDVD4ElJRTQXx7dOQ59EkqGyNU9tnwy1RKln+oLIOTU= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.12 h1:YU9UHPukkCCnETHEExOptF/BxPvGJKXO/NBx+RMQ/2A= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.12/go.mod h1:b53qpmhHk7mTL2J/tfG6f38neZiyBQSiNXGCuNKq4+4= +github.com/aws/smithy-go v1.12.1 h1:yQRC55aXN/y1W10HgwHle01DRuV9Dpf31iGkotjt3Ag= +github.com/aws/smithy-go v1.12.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -153,8 +188,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -320,8 +356,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -487,16 +521,13 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -519,7 +550,6 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -534,8 +564,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -596,7 +624,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -722,7 +749,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/storage/cloudstorage/aws/AwsS3.go b/internal/storage/cloudstorage/aws/AwsS3.go index 03cb7bac..8eac789e 100644 --- a/internal/storage/cloudstorage/aws/AwsS3.go +++ b/internal/storage/cloudstorage/aws/AwsS3.go @@ -6,13 +6,12 @@ import ( "context" "errors" "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/forceu/gokapi/internal/models" "io" "net/http" @@ -32,6 +31,9 @@ const IsMockApi = false // Init reads the credentials for AWS. Returns true if valid func Init(config models.AwsConfig) bool { + if config.Endpoint != "" && !strings.HasPrefix(config.Endpoint, "http") { + config.Endpoint = "https://" + config.Endpoint + } awsConfig = config ok, err := IsValidLogin(config) if err != nil { @@ -78,22 +80,42 @@ func IsValidLogin(config models.AwsConfig) (bool, error) { return true, nil } -func createSession() *session.Session { - s3Config := &aws.Config{ - Credentials: credentials.NewStaticCredentials(awsConfig.KeyId, awsConfig.KeySecret, ""), - Endpoint: aws.String(awsConfig.Endpoint), - Region: aws.String(awsConfig.Region), - S3ForcePathStyle: aws.Bool(true), +func createClient() (*s3.Client, error) { + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + if awsConfig.Endpoint != "" && service == s3.ServiceID && region == awsConfig.Region { + return aws.Endpoint{ + PartitionID: "aws", + URL: awsConfig.Endpoint, + SigningRegion: awsConfig.Region, + }, nil + } + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + }) + + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(awsConfig.Region), + config.WithEndpointResolverWithOptions(customResolver), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(awsConfig.KeyId, awsConfig.KeySecret, "")), + ) + if err != nil { + return nil, err } - return session.Must(session.NewSession(s3Config)) + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + o.Region = awsConfig.Region + }) + return client, nil } // Upload uploads a file to AWS func Upload(input io.Reader, file models.File) (string, error) { - sess := createSession() - uploader := s3manager.NewUploader(sess) + client, err := createClient() + if err != nil { + return "", err + } - result, err := uploader.Upload(&s3manager.UploadInput{ + uploader := manager.NewUploader(client) + result, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{ Bucket: aws.String(file.AwsBucket), Key: aws.String(file.SHA1), Body: input, @@ -106,10 +128,13 @@ func Upload(input io.Reader, file models.File) (string, error) { // Download downloads a file from AWS, used for encrypted files and testing func Download(writer io.WriterAt, file models.File) (int64, error) { - sess := createSession() - downloader := s3manager.NewDownloader(sess) + client, err := createClient() + if err != nil { + return 0, err + } + downloader := manager.NewDownloader(client) - size, err := downloader.Download(writer, &s3.GetObjectInput{ + size, err := downloader.Download(context.TODO(), writer, &s3.GetObjectInput{ Bucket: aws.String(file.AwsBucket), Key: aws.String(file.SHA1), }) @@ -122,28 +147,28 @@ func Download(writer io.WriterAt, file models.File) (int64, error) { // RedirectToDownload creates a presigned link that is valid for 15 seconds and redirects the // client to this url func RedirectToDownload(w http.ResponseWriter, r *http.Request, file models.File, forceDownload bool) error { - sess := createSession() - s3svc := s3.New(sess) + client, err := createClient() + if err != nil { + return err + } contentDisposition := "inline; filename=\"" + file.Name + "\"" if forceDownload { contentDisposition = "Attachment; filename=\"" + file.Name + "\"" } - - req, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{ + presignClient := s3.NewPresignClient(client) + presignResult, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(file.AwsBucket), Key: aws.String(file.SHA1), ResponseContentDisposition: aws.String(contentDisposition), ResponseCacheControl: aws.String("no-store"), ResponseContentType: aws.String(file.ContentType), - }) - - url, err := req.Presign(15 * time.Second) + }, s3.WithPresignExpires(15*time.Second)) if err != nil { return err } - http.Redirect(w, r, url, http.StatusTemporaryRedirect) + http.Redirect(w, r, presignResult.URL, http.StatusTemporaryRedirect) return nil } @@ -159,41 +184,42 @@ func getTimeoutContext() (context.Context, context.CancelFunc) { // FileExists returns true if the object is stored in S3 func FileExists(file models.File) (bool, int64, error) { - sess := createSession() - svc := s3.New(sess) + client, err := createClient() + if err != nil { + return false, 0, err + } ctx, cancelCtx := getTimeoutContext() defer cancelCtx() - info, err := svc.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ + info, err := client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(file.AwsBucket), Key: aws.String(file.SHA1), }) if err != nil { - aerr, ok := err.(awserr.Error) - if ok { - if aerr.Code() == "NotFound" { - return false, 0, nil - } + var errorNotFound *types.NotFound + if errors.As(err, &errorNotFound) { + return false, 0, nil } - if aerr.Code() == request.CanceledErrorCode { - return false, 0, errors.New("Timeout - could not connect to " + *svc.Config.Endpoint) + fmt.Println(err.Error()) // TODO + if true { // TODO + return false, 0, errors.New("Timeout - could not connect to " + awsConfig.Endpoint) } return false, 0, err } - return true, *info.ContentLength, nil + return true, info.ContentLength, nil } // DeleteObject deletes a file from S3 func DeleteObject(file models.File) (bool, error) { - sess := createSession() - svc := s3.New(sess) - + client, err := createClient() + if err != nil { + return false, err + } ctx, cancelCtx := getTimeoutContext() defer cancelCtx() - - _, err := svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ + _, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(file.AwsBucket), Key: aws.String(file.SHA1), }) @@ -206,19 +232,19 @@ func DeleteObject(file models.File) (bool, error) { // IsCorsCorrectlySet returns true if CORS rules allow download from Gokapi func IsCorsCorrectlySet(bucket, gokapiUrl string) (bool, error) { - sess := createSession() - svc := s3.New(sess) + client, err := createClient() + if err != nil { + return false, err + } input := &s3.GetBucketCorsInput{ Bucket: aws.String(bucket), } - ctx, cancelCtx := getTimeoutContext() defer cancelCtx() - result, err := svc.GetBucketCorsWithContext(ctx, input) + result, err := client.GetBucketCors(ctx, input) if err != nil { - aerr, ok := err.(awserr.Error) - if ok && aerr.Code() == "NoSuchCorsConfiguration" { + if err.Error() == "NoSuchCorsConfiguration" { // TODO return false, nil } return false, err @@ -226,10 +252,10 @@ func IsCorsCorrectlySet(bucket, gokapiUrl string) (bool, error) { for _, rule := range result.CORSRules { for _, origin := range rule.AllowedOrigins { - if *origin == "*" { + if origin == "*" { return true, nil } - if strings.HasPrefix(gokapiUrl, *origin) { + if strings.HasPrefix(gokapiUrl, origin) { return true, nil } }