diff --git a/Makefile b/Makefile
index ef8d9991..92a5d070 100644
--- a/Makefile
+++ b/Makefile
@@ -103,6 +103,10 @@ down/services:
docker rm -f opa || true
docker rm -f keycloak || true
+.PHONY: down/tracing-services
+down/tracing-services:
+ docker rm -f jaeger || true
+
.PHONY: down/metrics-services
down/metrics-services:
docker rm -f prometheus || true
@@ -113,6 +117,11 @@ setup/metrics-services:
docker run --rm -d --name prometheus -v $(CURRENT_DIR)/local-resources/prometheus/prometheus.yml:/prometheus/prometheus.yml --network=host prom/prometheus:v2.18.0 --web.listen-address=:9191
docker run --rm -d --name grafana --network=host grafana/grafana:7.0.3
+.PHONY: setup/tracing-services
+setup/tracing-services: down/tracing-services
+ @echo "Setup tracing services"
+ docker run --name jaeger -d -p 6831:6831/udp -p 16686:16686 jaegertracing/all-in-one:latest
+
.PHONY: setup/services
setup/services: down/services
tar czvf local-resources/opa/bundle.tar.gz --directory=local-resources/opa/bundle example/
diff --git a/conf/config-example.yaml b/conf/config-example.yaml
index ebdd47aa..18e1d0c7 100644
--- a/conf/config-example.yaml
+++ b/conf/config-example.yaml
@@ -219,6 +219,7 @@ targets:
region: eu-west-1
s3Endpoint:
disableSSL: false
+ # s3ListMaxKeys: 1000
# credentials:
# accessKey:
# env: AWS_ACCESS_KEY_ID
diff --git a/docs/configuration.md b/docs/configuration.md
index 6957b0f5..d72b782a 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -132,14 +132,15 @@ This feature is powered by [go-chi/cors](https://github.com/go-chi/cors). You ca
## BucketConfiguration
-| Key | Type | Required | Default | Description |
-| ----------- | --------------------------------------------------------------- | -------- | ----------- | ---------------------------------------- |
-| name | String | Yes | None | Bucket name in S3 provider |
-| prefix | String | No | None | Bucket prefix |
-| region | String | No | `us-east-1` | Bucket region |
-| s3Endpoint | String | No | None | Custom S3 Endpoint for non AWS S3 bucket |
-| credentials | [BucketCredentialConfiguration](#bucketcredentialconfiguration) | No | None | Credentials to access S3 bucket |
-| disableSSL | Boolean | No | `false` | Disable SSL connection |
+| Key | Type | Required | Default | Description |
+| ------------- | --------------------------------------------------------------- | -------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| name | String | Yes | None | Bucket name in S3 provider |
+| prefix | String | No | None | Bucket prefix |
+| region | String | No | `us-east-1` | Bucket region |
+| s3Endpoint | String | No | None | Custom S3 Endpoint for non AWS S3 bucket |
+| credentials | [BucketCredentialConfiguration](#bucketcredentialconfiguration) | No | None | Credentials to access S3 bucket |
+| disableSSL | Boolean | No | `false` | Disable SSL connection |
+| s3ListMaxKeys | Integer | No | `1000` | This flag will be used for the max pagination list management of files and "folders" in S3. In S3 list requests, the limit is fixed to 1000 items maximum. S3-Proxy will allow to increase this by making multiple requests to S3. Warning: This will increase the memory and CPU usage. |
## BucketCredentialConfiguration
@@ -470,6 +471,7 @@ targets:
region: eu-west-1
s3Endpoint:
disableSSL: false
+ # s3ListMaxKeys: 1000
# credentials:
# accessKey:
# env: AWS_ACCESS_KEY_ID
diff --git a/go.mod b/go.mod
index bcc84283..11fe602f 100644
--- a/go.mod
+++ b/go.mod
@@ -3,11 +3,11 @@ module github.com/oxyno-zeta/s3-proxy
go 1.13
require (
- github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect
+ github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
github.com/Masterminds/goutils v1.1.0 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible
- github.com/aws/aws-sdk-go v1.32.6
+ github.com/aws/aws-sdk-go v1.36.26
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/dimiro1/health v0.0.0-20191019130555-c5cbb4d46ffc
github.com/dustin/go-humanize v1.0.0
@@ -18,29 +18,29 @@ require (
github.com/go-playground/validator/v10 v10.4.1
github.com/gobwas/glob v0.2.3
github.com/golang/mock v1.4.4
- github.com/google/uuid v1.1.1 // indirect
- github.com/huandu/xstrings v1.3.0 // indirect
- github.com/imdario/mergo v0.3.8 // indirect
+ github.com/google/uuid v1.1.4 // indirect
+ github.com/huandu/xstrings v1.3.2 // indirect
+ github.com/imdario/mergo v0.3.11 // indirect
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.1 // indirect
github.com/opentracing/opentracing-go v1.2.0
- github.com/pelletier/go-toml v1.6.0 // indirect
- github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
+ github.com/pelletier/go-toml v1.8.1 // indirect
+ github.com/pquerna/cachecontrol v0.0.0-20201205024021-ac21108117ac // indirect
github.com/prometheus/client_golang v1.9.0
github.com/sirupsen/logrus v1.7.0
- github.com/spf13/afero v1.2.2 // indirect
+ github.com/spf13/afero v1.5.1 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.7.1
- github.com/stretchr/testify v1.6.1
+ github.com/stretchr/testify v1.7.0
github.com/thoas/go-funk v0.7.0
github.com/uber/jaeger-client-go v2.25.0+incompatible
github.com/uber/jaeger-lib v2.4.0+incompatible
golang.org/x/net v0.0.0-20201224014010-6772e930b67b
- golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5
+ golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
- gopkg.in/ini.v1 v1.52.0 // indirect
- gopkg.in/square/go-jose.v2 v2.4.1 // indirect
+ gopkg.in/ini.v1 v1.62.0 // indirect
+ gopkg.in/square/go-jose.v2 v2.5.1 // indirect
)
diff --git a/go.sum b/go.sum
index 5f1467b4..c5f7aac4 100644
--- a/go.sum
+++ b/go.sum
@@ -36,8 +36,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
-github.com/HdrHistogram/hdrhistogram-go v0.9.0 h1:dpujRju0R4M/QZzcnR1LH1qm+TVG3UzkWdp5tH1WMcg=
-github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw=
+github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg=
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
@@ -64,8 +64,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.32.6 h1:HoswAabUWgnrUF7X/9dr4WRgrr8DyscxXvTDm7Qw/5c=
-github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.36.26 h1:710u7Q1xpbzpigh/RPPUXwr/5X/VwCxTXSeam8icgos=
+github.com/aws/aws-sdk-go v1.36.26/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -102,6 +102,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -158,7 +159,6 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
@@ -212,6 +212,8 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -224,8 +226,8 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0=
+github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@@ -265,18 +267,20 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/huandu/xstrings v1.3.0 h1:gvV6jG9dTgFEncxo+AF7PH6MZXi/vZl25owA/8Dg8Wo=
-github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
+github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc h1:JJPhSHowepOF2+ElJVyb9jgt5ZyBkPMkPuhS0uODSFs=
github.com/johannesboyne/gofakes3 v0.0.0-20200716060623-6b2b4cb092cc/go.mod h1:fNiSoOiEI5KlkWXn26OwKnNe58ilTIkpBlgOrt7Olu8=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -296,12 +300,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
@@ -343,6 +350,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -367,8 +376,8 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
-github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
+github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
+github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -377,11 +386,12 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
-github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/pquerna/cachecontrol v0.0.0-20201205024021-ac21108117ac h1:jWKYCNlX4J5s8M0nHYkh7Y7c9gRVDEb3mq51j5J0F5M=
+github.com/pquerna/cachecontrol v0.0.0-20201205024021-ac21108117ac/go.mod h1:hoLfEwdY11HjRfKFH6KqnPsfxlo3BP6bJehpDv8t6sQ=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
@@ -447,8 +457,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.5.1 h1:VHu76Lk0LSP1x254maIu2bplkWpfBWI+B+6fdoZprcg=
+github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
@@ -476,10 +486,10 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/thoas/go-funk v0.7.0 h1:GmirKrs6j6zJbhJIficOsz2aAI7700KsU/5YrdHRM1Y=
@@ -522,6 +532,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -595,6 +606,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -603,8 +615,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 h1:Lm4OryKCca1vehdsWogr9N4t7NfZxLbJoc/H0w4K4S4=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3 h1:BaN3BAqnopnKjvl+15DYP6LLrbBHfbfmlFYzmFj/Q9Q=
+golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -662,6 +674,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -829,18 +842,20 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.52.0 h1:j+Lt/M1oPPejkniCg1TkWE2J3Eh1oZTsHSXzMTzUXn4=
-gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y=
-gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w=
+gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@@ -850,6 +865,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
diff --git a/pkg/s3-proxy/config/config.go b/pkg/s3-proxy/config/config.go
index be24a7a4..5d2850b0 100644
--- a/pkg/s3-proxy/config/config.go
+++ b/pkg/s3-proxy/config/config.go
@@ -21,6 +21,9 @@ const DefaultLogFormat = "json"
// DefaultBucketRegion Default bucket region
const DefaultBucketRegion = "us-east-1"
+// DefaultBucketS3ListMaxKeys Default bucket S3 list max keys
+const DefaultBucketS3ListMaxKeys int64 = 1000
+
// DefaultTemplateFolderListPath Default template folder list path
const DefaultTemplateFolderListPath = "templates/folder-list.tpl"
@@ -255,12 +258,13 @@ type OPAServerAuthorization struct {
// BucketConfig Bucket configuration
type BucketConfig struct {
- Name string `mapstructure:"name" validate:"required"`
- Prefix string `mapstructure:"prefix"`
- Region string `mapstructure:"region"`
- S3Endpoint string `mapstructure:"s3Endpoint"`
- Credentials *BucketCredentialConfig `mapstructure:"credentials" validate:"omitempty,dive"`
- DisableSSL bool `mapstructure:"disableSSL"`
+ Name string `mapstructure:"name" validate:"required"`
+ Prefix string `mapstructure:"prefix"`
+ Region string `mapstructure:"region"`
+ S3Endpoint string `mapstructure:"s3Endpoint"`
+ Credentials *BucketCredentialConfig `mapstructure:"credentials" validate:"omitempty,dive"`
+ DisableSSL bool `mapstructure:"disableSSL"`
+ S3ListMaxKeys int64 `mapstructure:"s3ListMaxKeys" validate:"gt=0"`
}
// BucketCredentialConfig Bucket Credentials configurations
diff --git a/pkg/s3-proxy/config/managercontext.go b/pkg/s3-proxy/config/managercontext.go
index aa489769..621f192c 100644
--- a/pkg/s3-proxy/config/managercontext.go
+++ b/pkg/s3-proxy/config/managercontext.go
@@ -416,6 +416,10 @@ func loadBusinessDefaultValues(out *Config) error {
if item.Bucket != nil && item.Bucket.Region == "" {
item.Bucket.Region = DefaultBucketRegion
}
+ // Manage default configuration for bucket S3 List Max Keys
+ if item.Bucket != nil && item.Bucket.S3ListMaxKeys == 0 {
+ item.Bucket.S3ListMaxKeys = DefaultBucketS3ListMaxKeys
+ }
// Manage default configuration for target actions
if item.Actions == nil {
item.Actions = &ActionsConfig{GET: &GetActionConfig{Enabled: true}}
diff --git a/pkg/s3-proxy/config/managercontext_integration_test.go b/pkg/s3-proxy/config/managercontext_integration_test.go
index 042cb355..09e23826 100644
--- a/pkg/s3-proxy/config/managercontext_integration_test.go
+++ b/pkg/s3-proxy/config/managercontext_integration_test.go
@@ -90,8 +90,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
},
Actions: &ActionsConfig{
GET: &GetActionConfig{Enabled: true},
@@ -154,8 +155,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Env: "ENV1",
@@ -268,8 +270,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Path: os.TempDir() + "/secret1",
@@ -366,8 +369,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -447,8 +451,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "value1",
@@ -627,8 +632,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -698,8 +704,9 @@ log:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -819,8 +826,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -888,8 +896,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "SECRET1",
@@ -1009,8 +1018,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -1081,8 +1091,9 @@ configuration with error
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -1212,8 +1223,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -1289,8 +1301,9 @@ authProviders:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
@@ -1433,8 +1446,9 @@ targets:
Path: []string{"/test/"},
},
Bucket: &BucketConfig{
- Name: "bucket1",
- Region: "us-east-1",
+ Name: "bucket1",
+ Region: "us-east-1",
+ S3ListMaxKeys: 1000,
Credentials: &BucketCredentialConfig{
AccessKey: &CredentialConfig{
Value: "VALUE1",
diff --git a/pkg/s3-proxy/config/managercontext_test.go b/pkg/s3-proxy/config/managercontext_test.go
index a7f58e63..dfc91ce0 100644
--- a/pkg/s3-proxy/config/managercontext_test.go
+++ b/pkg/s3-proxy/config/managercontext_test.go
@@ -290,7 +290,7 @@ func Test_loadBusinessDefaultValues(t *testing.T) {
Targets: []*TargetConfig{
{
Actions: &ActionsConfig{GET: &GetActionConfig{Enabled: false}},
- Bucket: &BucketConfig{Region: DefaultBucketRegion},
+ Bucket: &BucketConfig{Region: DefaultBucketRegion, S3ListMaxKeys: DefaultBucketS3ListMaxKeys},
Templates: &TargetTemplateConfig{},
},
},
@@ -305,7 +305,7 @@ func Test_loadBusinessDefaultValues(t *testing.T) {
Targets: []*TargetConfig{
{
Actions: &ActionsConfig{GET: &GetActionConfig{Enabled: false}},
- Bucket: &BucketConfig{Region: "test"},
+ Bucket: &BucketConfig{Region: "test", S3ListMaxKeys: 100},
Resources: []*Resource{
{
OIDC: &ResourceOIDC{
@@ -328,7 +328,7 @@ func Test_loadBusinessDefaultValues(t *testing.T) {
Targets: []*TargetConfig{
{
Actions: &ActionsConfig{GET: &GetActionConfig{Enabled: false}},
- Bucket: &BucketConfig{Region: "test"},
+ Bucket: &BucketConfig{Region: "test", S3ListMaxKeys: 100},
Resources: []*Resource{
{
Methods: []string{"GET"},
diff --git a/pkg/s3-proxy/s3client/s3Context.go b/pkg/s3-proxy/s3client/s3Context.go
index 21eea762..e1d121c9 100644
--- a/pkg/s3-proxy/s3client/s3Context.go
+++ b/pkg/s3-proxy/s3client/s3Context.go
@@ -36,59 +36,103 @@ const PutObjectOperation = "put-object"
// DeleteObjectOperation Delete object operation
const DeleteObjectOperation = "delete-object"
+const s3MaxKeys int64 = 1000
+
// ListFilesAndDirectories List files and directories
func (s3ctx *s3Context) ListFilesAndDirectories(key string) ([]*ListElementOutput, error) {
- // Create child trace
- childTrace := s3ctx.parentTrace.GetChildTrace("s3-bucket.list-objects-request")
- childTrace.SetTag("s3-bucket.bucket-name", s3ctx.target.Bucket.Name)
- childTrace.SetTag("s3-bucket.bucket-region", s3ctx.target.Bucket.Region)
- childTrace.SetTag("s3-bucket.bucket-prefix", s3ctx.target.Bucket.Prefix)
- childTrace.SetTag("s3-bucket.bucket-s3-endpoint", s3ctx.target.Bucket.S3Endpoint)
- childTrace.SetTag("s3-proxy.target-name", s3ctx.target.Name)
-
- defer childTrace.Finish()
-
// List files on path
folders := make([]*ListElementOutput, 0)
files := make([]*ListElementOutput, 0)
- err := s3ctx.svcClient.ListObjectsV2Pages(
- &s3.ListObjectsV2Input{
- Bucket: aws.String(s3ctx.target.Bucket.Name),
- Prefix: aws.String(key),
- Delimiter: aws.String("/"),
- },
- func(page *s3.ListObjectsV2Output, lastPage bool) bool {
- // Manage folders
- for _, item := range page.CommonPrefixes {
- name := strings.TrimPrefix(*item.Prefix, key)
- folders = append(folders, &ListElementOutput{
- Type: FolderType,
- Key: *item.Prefix,
- Name: name,
- })
- }
- // Manage files
- for _, item := range page.Contents {
- name := strings.TrimPrefix(*item.Key, key)
- if name != "" {
- files = append(files, &ListElementOutput{
- Type: FileType,
- ETag: *item.ETag,
- Name: name,
- LastModified: *item.LastModified,
- Size: *item.Size,
- Key: *item.Key,
+ // Prepare next token structure
+ var nextToken *string
+ // Temporary max elements for limits
+ tmpMaxElements := s3ctx.target.Bucket.S3ListMaxKeys
+ // Loop control
+ loopControl := true
+ // Initialize max keys
+ maxKeys := s3MaxKeys
+ // Check size of max keys
+ if s3ctx.target.Bucket.S3ListMaxKeys < maxKeys {
+ maxKeys = s3ctx.target.Bucket.S3ListMaxKeys
+ }
+
+ // Loop
+ for loopControl {
+ // Create child trace
+ childTrace := s3ctx.parentTrace.GetChildTrace("s3-bucket.list-objects-request")
+ childTrace.SetTag("s3-bucket.bucket-name", s3ctx.target.Bucket.Name)
+ childTrace.SetTag("s3-bucket.bucket-region", s3ctx.target.Bucket.Region)
+ childTrace.SetTag("s3-bucket.bucket-prefix", s3ctx.target.Bucket.Prefix)
+ childTrace.SetTag("s3-bucket.bucket-s3-endpoint", s3ctx.target.Bucket.S3Endpoint)
+ childTrace.SetTag("s3-proxy.target-name", s3ctx.target.Name)
+
+ // Request S3
+ err := s3ctx.svcClient.ListObjectsV2Pages(
+ &s3.ListObjectsV2Input{
+ Bucket: aws.String(s3ctx.target.Bucket.Name),
+ Prefix: aws.String(key),
+ Delimiter: aws.String("/"),
+ MaxKeys: aws.Int64(maxKeys),
+ ContinuationToken: nextToken,
+ },
+ func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+ // Store next token
+ nextToken = page.NextContinuationToken
+
+ // Check if keycount exists
+ if page.KeyCount != nil {
+ // Remove current keys to tmp max elements
+ tmpMaxElements -= *page.KeyCount
+ // Update max keys if needed
+ if tmpMaxElements < maxKeys {
+ maxKeys = tmpMaxElements
+ }
+ }
+
+ // Manage loop control
+ loopControl = nextToken != nil && tmpMaxElements > 0
+
+ // Manage folders
+ for _, item := range page.CommonPrefixes {
+ name := strings.TrimPrefix(*item.Prefix, key)
+ folders = append(folders, &ListElementOutput{
+ Type: FolderType,
+ Key: *item.Prefix,
+ Name: name,
})
}
- }
- return lastPage
- })
- // Metrics
- s3ctx.metricsCtx.IncS3Operations(s3ctx.target.Name, s3ctx.target.Bucket.Name, ListObjectsOperation)
- // Check if errors exists
- if err != nil {
- return nil, err
+
+ // Manage files
+ for _, item := range page.Contents {
+ name := strings.TrimPrefix(*item.Key, key)
+ if name != "" {
+ files = append(files, &ListElementOutput{
+ Type: FileType,
+ ETag: *item.ETag,
+ Name: name,
+ LastModified: *item.LastModified,
+ Size: *item.Size,
+ Key: *item.Key,
+ })
+ }
+ }
+
+ return lastPage
+ },
+ )
+
+ // Metrics
+ s3ctx.metricsCtx.IncS3Operations(s3ctx.target.Name, s3ctx.target.Bucket.Name, ListObjectsOperation)
+
+ // End trace
+ childTrace.Finish()
+
+ // Check if errors exists
+ if err != nil {
+ return nil, err
+ }
}
+
// Concat folders and files
all := append(folders, files...)
diff --git a/pkg/s3-proxy/server/server_integration_test.go b/pkg/s3-proxy/server/server_integration_test.go
index 03ed28bc..936046f1 100644
--- a/pkg/s3-proxy/server/server_integration_test.go
+++ b/pkg/s3-proxy/server/server_integration_test.go
@@ -1947,7 +1947,6 @@ func TestPublicRouter(t *testing.T) {
}
func TestTracing(t *testing.T) {
- // trueValue := true
accessKey := "YOUR-ACCESSKEYID"
secretAccessKey := "YOUR-SECRETACCESSKEY"
region := "eu-central-1"
@@ -2163,7 +2162,6 @@ func TestTracing(t *testing.T) {
// This is in a separate test because this one will need a real server to discuss with OIDC server
func TestOIDCAuthentication(t *testing.T) {
- // trueValue := true
accessKey := "YOUR-ACCESSKEYID"
secretAccessKey := "YOUR-SECRETACCESSKEY"
region := "eu-central-1"
@@ -2860,7 +2858,6 @@ func TestOIDCAuthentication(t *testing.T) {
}
func TestCORS(t *testing.T) {
- // trueValue := true
accessKey := "YOUR-ACCESSKEYID"
secretAccessKey := "YOUR-SECRETACCESSKEY"
region := "eu-central-1"
@@ -3226,3 +3223,431 @@ func TestCORS(t *testing.T) {
})
}
}
+
+func TestIndexLargeBucket(t *testing.T) {
+ accessKey := "YOUR-ACCESSKEYID"
+ secretAccessKey := "YOUR-SECRETACCESSKEY"
+ region := "eu-central-1"
+ bucket := "test-bucket"
+ s3server, err := setupFakeS3(
+ accessKey,
+ secretAccessKey,
+ region,
+ bucket,
+ )
+ defer s3server.Close()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ tplConfig := &config.TemplateConfig{
+ FolderList: "../../../templates/folder-list.tpl",
+ TargetList: "../../../templates/target-list.tpl",
+ NotFound: "../../../templates/not-found.tpl",
+ Forbidden: "../../../templates/forbidden.tpl",
+ BadRequest: "../../../templates/bad-request.tpl",
+ InternalServerError: "../../../templates/internal-server-error.tpl",
+ Unauthorized: "../../../templates/unauthorized.tpl",
+ }
+ targetsCfg := []*config.TargetConfig{
+ {
+ Name: "target1",
+ Bucket: &config.BucketConfig{
+ Name: bucket,
+ Region: region,
+ S3Endpoint: s3server.URL,
+ Credentials: &config.BucketCredentialConfig{
+ AccessKey: &config.CredentialConfig{Value: accessKey},
+ SecretKey: &config.CredentialConfig{Value: secretAccessKey},
+ },
+ DisableSSL: true,
+ S3ListMaxKeys: 1000,
+ },
+ Mount: &config.MountConfig{
+ Path: []string{"/"},
+ },
+ Actions: &config.ActionsConfig{
+ GET: &config.GetActionConfig{Enabled: true},
+ },
+ IndexDocument: "index.html",
+ },
+ }
+
+ // Create go mock controller
+ ctrl := gomock.NewController(t)
+ cfgManagerMock := cmocks.NewMockManager(ctrl)
+
+ // Load configuration in manager
+ cfgManagerMock.EXPECT().GetConfig().AnyTimes().Return(&config.Config{
+ Server: &config.ServerConfig{
+ Port: 8080,
+ },
+ ListTargets: &config.ListTargetsConfig{},
+ Tracing: &config.TracingConfig{},
+ Templates: tplConfig,
+ Targets: targetsCfg,
+ })
+
+ logger := log.NewLogger()
+ // Create tracing service
+ tsvc, err := tracing.New(cfgManagerMock, logger)
+ assert.NoError(t, err)
+
+ svr := &Server{
+ logger: logger,
+ cfgManager: cfgManagerMock,
+ metricsCl: metricsCtx,
+ tracingSvc: tsvc,
+ }
+ got, err := svr.generateRouter()
+ if err != nil {
+ t.Errorf("TestIndexLargeBucket.GenerateRouter() error = %v", err)
+ return
+ }
+ w := httptest.NewRecorder()
+ req, err := http.NewRequest(
+ "GET",
+ "http://localhost/folder3/",
+ nil,
+ )
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ got.ServeHTTP(w, req)
+
+ // Test status code
+ if w.Code != 200 {
+ t.Errorf("TestIndexLargeBucket.GenerateRouter() status code = %v, expected status code %v", w.Code, 200)
+ }
+ // Test body
+ body := w.Body.String()
+ expectedBody := "Hello folder3!
"
+ if body != expectedBody {
+ t.Errorf("TestIndexLargeBucket.GenerateRouter() body = \"%v\", expected body \"%v\"", body, expectedBody)
+ }
+}
+
+func TestListLargeBucketAndSmallMaxKeys(t *testing.T) {
+ accessKey := "YOUR-ACCESSKEYID"
+ secretAccessKey := "YOUR-SECRETACCESSKEY"
+ region := "eu-central-1"
+ bucket := "test-bucket"
+ maxKeys := 500
+ s3server, err := setupFakeS3(
+ accessKey,
+ secretAccessKey,
+ region,
+ bucket,
+ )
+ defer s3server.Close()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ tplConfig := &config.TemplateConfig{
+ FolderList: "../../../templates/folder-list.tpl",
+ TargetList: "../../../templates/target-list.tpl",
+ NotFound: "../../../templates/not-found.tpl",
+ Forbidden: "../../../templates/forbidden.tpl",
+ BadRequest: "../../../templates/bad-request.tpl",
+ InternalServerError: "../../../templates/internal-server-error.tpl",
+ Unauthorized: "../../../templates/unauthorized.tpl",
+ }
+ targetsCfg := []*config.TargetConfig{
+ {
+ Name: "target1",
+ Bucket: &config.BucketConfig{
+ Name: bucket,
+ Region: region,
+ S3Endpoint: s3server.URL,
+ Credentials: &config.BucketCredentialConfig{
+ AccessKey: &config.CredentialConfig{Value: accessKey},
+ SecretKey: &config.CredentialConfig{Value: secretAccessKey},
+ },
+ DisableSSL: true,
+ S3ListMaxKeys: int64(maxKeys),
+ },
+ Mount: &config.MountConfig{
+ Path: []string{"/"},
+ },
+ Actions: &config.ActionsConfig{
+ GET: &config.GetActionConfig{Enabled: true},
+ },
+ },
+ }
+
+ // Create go mock controller
+ ctrl := gomock.NewController(t)
+ cfgManagerMock := cmocks.NewMockManager(ctrl)
+
+ // Load configuration in manager
+ cfgManagerMock.EXPECT().GetConfig().AnyTimes().Return(&config.Config{
+ Server: &config.ServerConfig{
+ Port: 8080,
+ },
+ ListTargets: &config.ListTargetsConfig{},
+ Tracing: &config.TracingConfig{},
+ Templates: tplConfig,
+ Targets: targetsCfg,
+ })
+
+ logger := log.NewLogger()
+ // Create tracing service
+ tsvc, err := tracing.New(cfgManagerMock, logger)
+ assert.NoError(t, err)
+
+ svr := &Server{
+ logger: logger,
+ cfgManager: cfgManagerMock,
+ metricsCl: metricsCtx,
+ tracingSvc: tsvc,
+ }
+ got, err := svr.generateRouter()
+ if err != nil {
+ t.Errorf("TestListLargeBucketAndSmallMaxKeys.GenerateRouter() error = %v", err)
+ return
+ }
+ w := httptest.NewRecorder()
+ req, err := http.NewRequest(
+ "GET",
+ "http://localhost/folder3/",
+ nil,
+ )
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ got.ServeHTTP(w, req)
+
+ // Test status code
+ if w.Code != 200 {
+ t.Errorf("TestListLargeBucketAndSmallMaxKeys.GenerateRouter() status code = %v, expected status code %v", w.Code, 200)
+ }
+ // Test body
+ body := w.Body.String()
+ if strings.Count(body, "\"/folder3/") != maxKeys {
+ t.Errorf("TestListLargeBucketAndSmallMaxKeys.GenerateRouter() body = \"%v\", must contains %d elements in the list", body, maxKeys)
+ }
+}
+
+func TestListLargeBucketAndMaxKeysGreaterThanS3MaxKeys(t *testing.T) {
+ accessKey := "YOUR-ACCESSKEYID"
+ secretAccessKey := "YOUR-SECRETACCESSKEY"
+ region := "eu-central-1"
+ bucket := "test-bucket"
+ maxKeys := 1500
+ s3server, err := setupFakeS3(
+ accessKey,
+ secretAccessKey,
+ region,
+ bucket,
+ )
+ defer s3server.Close()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ tplConfig := &config.TemplateConfig{
+ FolderList: "../../../templates/folder-list.tpl",
+ TargetList: "../../../templates/target-list.tpl",
+ NotFound: "../../../templates/not-found.tpl",
+ Forbidden: "../../../templates/forbidden.tpl",
+ BadRequest: "../../../templates/bad-request.tpl",
+ InternalServerError: "../../../templates/internal-server-error.tpl",
+ Unauthorized: "../../../templates/unauthorized.tpl",
+ }
+ targetsCfg := []*config.TargetConfig{
+ {
+ Name: "target1",
+ Bucket: &config.BucketConfig{
+ Name: bucket,
+ Region: region,
+ S3Endpoint: s3server.URL,
+ Credentials: &config.BucketCredentialConfig{
+ AccessKey: &config.CredentialConfig{Value: accessKey},
+ SecretKey: &config.CredentialConfig{Value: secretAccessKey},
+ },
+ DisableSSL: true,
+ S3ListMaxKeys: int64(maxKeys),
+ },
+ Mount: &config.MountConfig{
+ Path: []string{"/"},
+ },
+ Actions: &config.ActionsConfig{
+ GET: &config.GetActionConfig{Enabled: true},
+ },
+ },
+ }
+
+ // Create go mock controller
+ ctrl := gomock.NewController(t)
+ cfgManagerMock := cmocks.NewMockManager(ctrl)
+
+ // Load configuration in manager
+ cfgManagerMock.EXPECT().GetConfig().AnyTimes().Return(&config.Config{
+ Server: &config.ServerConfig{
+ Port: 8080,
+ },
+ ListTargets: &config.ListTargetsConfig{},
+ Tracing: &config.TracingConfig{},
+ Templates: tplConfig,
+ Targets: targetsCfg,
+ })
+
+ logger := log.NewLogger()
+ // Create tracing service
+ tsvc, err := tracing.New(cfgManagerMock, logger)
+ assert.NoError(t, err)
+
+ svr := &Server{
+ logger: logger,
+ cfgManager: cfgManagerMock,
+ metricsCl: metricsCtx,
+ tracingSvc: tsvc,
+ }
+ got, err := svr.generateRouter()
+ if err != nil {
+ t.Errorf("TestListLargeBucketAndMaxKeysGreaterThanS3MaxKeys.GenerateRouter() error = %v", err)
+ return
+ }
+ w := httptest.NewRecorder()
+ req, err := http.NewRequest(
+ "GET",
+ "http://localhost/folder3/",
+ nil,
+ )
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ got.ServeHTTP(w, req)
+
+ // Test status code
+ if w.Code != 200 {
+ t.Errorf("TestListLargeBucketAndMaxKeysGreaterThanS3MaxKeys.GenerateRouter() status code = %v, expected status code %v", w.Code, 200)
+ }
+ // Test body
+ body := w.Body.String()
+ if strings.Count(body, "\"/folder3/") != maxKeys {
+ t.Errorf("TestListLargeBucketAndMaxKeysGreaterThanS3MaxKeys.GenerateRouter() body = \"%v\", must contains %d elements in the list", body, maxKeys)
+ }
+}
+
+func TestFolderWithSubFolders(t *testing.T) {
+ accessKey := "YOUR-ACCESSKEYID"
+ secretAccessKey := "YOUR-SECRETACCESSKEY"
+ region := "eu-central-1"
+ bucket := "test-bucket"
+ s3server, err := setupFakeS3(
+ accessKey,
+ secretAccessKey,
+ region,
+ bucket,
+ )
+ defer s3server.Close()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ tplConfig := &config.TemplateConfig{
+ FolderList: "../../../templates/folder-list.tpl",
+ TargetList: "../../../templates/target-list.tpl",
+ NotFound: "../../../templates/not-found.tpl",
+ Forbidden: "../../../templates/forbidden.tpl",
+ BadRequest: "../../../templates/bad-request.tpl",
+ InternalServerError: "../../../templates/internal-server-error.tpl",
+ Unauthorized: "../../../templates/unauthorized.tpl",
+ }
+ targetsCfg := []*config.TargetConfig{
+ {
+ Name: "target1",
+ Bucket: &config.BucketConfig{
+ Name: bucket,
+ Region: region,
+ S3Endpoint: s3server.URL,
+ Credentials: &config.BucketCredentialConfig{
+ AccessKey: &config.CredentialConfig{Value: accessKey},
+ SecretKey: &config.CredentialConfig{Value: secretAccessKey},
+ },
+ DisableSSL: true,
+ },
+ Mount: &config.MountConfig{
+ Path: []string{"/"},
+ },
+ Actions: &config.ActionsConfig{
+ GET: &config.GetActionConfig{Enabled: true},
+ },
+ },
+ }
+
+ // Create go mock controller
+ ctrl := gomock.NewController(t)
+ cfgManagerMock := cmocks.NewMockManager(ctrl)
+
+ // Load configuration in manager
+ cfgManagerMock.EXPECT().GetConfig().AnyTimes().Return(&config.Config{
+ Server: &config.ServerConfig{
+ Port: 8080,
+ },
+ ListTargets: &config.ListTargetsConfig{},
+ Tracing: &config.TracingConfig{},
+ Templates: tplConfig,
+ Targets: targetsCfg,
+ })
+
+ logger := log.NewLogger()
+ // Create tracing service
+ tsvc, err := tracing.New(cfgManagerMock, logger)
+ assert.NoError(t, err)
+
+ svr := &Server{
+ logger: logger,
+ cfgManager: cfgManagerMock,
+ metricsCl: metricsCtx,
+ tracingSvc: tsvc,
+ }
+ got, err := svr.generateRouter()
+ if err != nil {
+ t.Errorf("TestFolderWithSubFolders.GenerateRouter() error = %v", err)
+ return
+ }
+ w := httptest.NewRecorder()
+ req, err := http.NewRequest(
+ "GET",
+ "http://localhost/folder4/",
+ nil,
+ )
+ if err != nil {
+ t.Error(err)
+ return
+ }
+
+ got.ServeHTTP(w, req)
+
+ // Test status code
+ if w.Code != 200 {
+ t.Errorf("TestFolderWithSubFolders.GenerateRouter() status code = %v, expected status code %v", w.Code, 200)
+ }
+ // Test body
+ body := w.Body.String()
+ if !strings.Contains(body, "\"/folder4/test.txt") {
+ t.Errorf("TestFolderWithSubFolders.GenerateRouter() body = \"%v\", must contains text.txt file", body)
+ }
+ if !strings.Contains(body, "\"/folder4/index.html") {
+ t.Errorf("TestFolderWithSubFolders.GenerateRouter() body = \"%v\", must contains index.html file", body)
+ }
+ if !strings.Contains(body, "\"/folder4/sub1/") {
+ t.Errorf("TestFolderWithSubFolders.GenerateRouter() body = \"%v\", must contains sub1 folder", body)
+ }
+ if !strings.Contains(body, "\"/folder4/sub2/") {
+ t.Errorf("TestFolderWithSubFolders.GenerateRouter() body = \"%v\", must contains sub2 folder", body)
+ }
+}
diff --git a/pkg/s3-proxy/server/utils_integration_test.go b/pkg/s3-proxy/server/utils_integration_test.go
index 98108437..84f3586b 100644
--- a/pkg/s3-proxy/server/utils_integration_test.go
+++ b/pkg/s3-proxy/server/utils_integration_test.go
@@ -3,6 +3,7 @@
package server
import (
+ "fmt"
"net/http/httptest"
"strings"
@@ -48,9 +49,21 @@ func setupFakeS3(accessKey, secretAccessKey, region, bucket string) (*httptest.S
"folder1/test.txt": "Hello folder1!",
"folder1/index.html": "Hello folder1!
",
"folder2/index.html": "Hello folder2!
",
+ "folder3/index.html": "Hello folder3!
",
+ "folder3/test.txt": "Hello folder3!",
+ "folder4/test.txt": "Hello folder4!",
+ "folder4/index.html": "Hello folder4!
",
+ "folder4/sub1/test.txt": "Hello folder4!",
+ "folder4/sub2/test.txt": "Hello folder4!",
"templates/folder-list.tpl": "fake template !",
}
+ // Inject large number of elements
+ for i := 0; i < 2000; i++ {
+ // Update map of files
+ files[fmt.Sprintf("folder3/%d", i)] = fmt.Sprintf("content %d", i)
+ }
+
// Upload files
for k, v := range files {
_, err = s3Client.PutObject(&s3.PutObjectInput{