++ mktemp + cfg=/tmp/tmp.aj7fTmDbhJ + envsubst '$CRYOSTAT_ACCESS_KEY $CRYOSTAT_SECRET_KEY' + names=() ++ echo archivedrecordings,archivedreports,eventtemplates,probes ++ tr , '\n' + for i in $(echo "$CRYOSTAT_BUCKETS" | tr ',' '\n') + len=18 + '[' 18 -lt 3 ']' + '[' 18 -gt 63 ']' + names+=("${i}") + for i in $(echo "$CRYOSTAT_BUCKETS" | tr ',' '\n') + len=15 + '[' 15 -lt 3 ']' + '[' 15 -gt 63 ']' + names+=("${i}") + for i in $(echo "$CRYOSTAT_BUCKETS" | tr ',' '\n') + len=14 + '[' 14 -lt 3 ']' + '[' 14 -gt 63 ']' + names+=("${i}") + for i in $(echo "$CRYOSTAT_BUCKETS" | tr ',' '\n') + len=6 + '[' 6 -lt 3 ']' + '[' 6 -gt 63 ']' + names+=("${i}") + set -e + VOLUME_MIN=40 + createBuckets archivedrecordings archivedreports eventtemplates probes + NUM_VOLUMES=40 + DATA_DIR=/data + waitForStartup + timeout 2s weed shell + echo cluster.check ++ df -P -B1 /data ++ tail -1 ++ tr -s ' ' ++ cut '-d ' -f 4 + AVAILABLE_DISK_BYTES=302950043648 + STORAGE_CAPACITY=302950043648 ++ echo 302950043648 ++ numfmt --from=iec --suffix=B ++ tr -d B + STORAGE_CAPACITY_BYTES=302950043648 + VOLUME_SIZE_BYTES=7573751091 + FLAGS=("-filer.allowedOrigins=${FILER_ORIGINS:-0.0.0.0}") + '[' 0 '!=' 1 ']' + FLAGS+=("-filer.exposeDirectoryData=false" "-filer.disableDirListing" "-webdav=false") + '[' 1 = 1 ']' + FLAGS+=("-filer.encryptVolumeData") + exec weed -logtostderr=true server -dir=/data -volume.max=40 -volume.fileSizeLimitMB=4096 -master.volumeSizeLimitMB=7222 -master.volumePreallocate=false -filer.allowedOrigins=0.0.0.0 -filer.exposeDirectoryData=false -filer.disableDirListing -webdav=false -filer.encryptVolumeData -s3 -s3.config=/tmp/tmp.aj7fTmDbhJ I1121 21:13:43.550279 master.go:282 current: 10.128.0.56:9333 peers: I1121 21:13:43.550350 file_util.go:27 Folder /data Permission: -rwxrwxrwx I1121 21:13:43.550436 master.go:282 current: 10.128.0.56:9333 peers:10.128.0.56:9333 I1121 21:13:43.550502 file_util.go:27 Folder /data Permission: -rwxrwxrwx I1121 21:13:43.550673 master_server.go:134 Volume Size Limit is 7222 MB I1121 21:13:43.551958 volume_grpc_client_to_master.go:43 checkWithMaster 10.128.0.56:9333: get master 10.128.0.56:9333 configuration: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 10.128.0.56:19333: connect: connection refused" I1121 21:13:43.552331 master.go:163 Start Seaweed Master 30GB 3.79 at 0.0.0.0:9333 I1121 21:13:43.552652 raft_server.go:119 Starting RaftServer with 10.128.0.56:9333 I1121 21:13:43.555317 raft_server.go:168 current cluster leader: I1121 21:13:45.342937 volume_grpc_client_to_master.go:43 checkWithMaster 10.128.0.56:9333: get master 10.128.0.56:9333 configuration: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 10.128.0.56:19333: connect: connection refused" + echo 'Waiting for cluster to be ready for bucket creation...' Waiting for cluster to be ready for bucket creation... + sleep 10 I1121 21:13:45.551633 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:46.552381 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:47.133675 volume_grpc_client_to_master.go:43 checkWithMaster 10.128.0.56:9333: get master 10.128.0.56:9333 configuration: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 10.128.0.56:19333: connect: connection refused" I1121 21:13:47.553494 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:48.554346 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:48.925426 volume_grpc_client_to_master.go:43 checkWithMaster 10.128.0.56:9333: get master 10.128.0.56:9333 configuration: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 10.128.0.56:19333: connect: connection refused" I1121 21:13:49.555851 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:50.556683 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:50.716698 volume_grpc_client_to_master.go:43 checkWithMaster 10.128.0.56:9333: get master 10.128.0.56:9333 configuration: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 10.128.0.56:19333: connect: connection refused" I1121 21:13:51.557332 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:52.507983 volume_grpc_client_to_master.go:43 checkWithMaster 10.128.0.56:9333: get master 10.128.0.56:9333 configuration: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 10.128.0.56:19333: connect: connection refused" I1121 21:13:52.558472 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:53.559490 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:54.299262 volume_grpc_client_to_master.go:43 checkWithMaster 10.128.0.56:9333: get master 10.128.0.56:9333 configuration: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 10.128.0.56:19333: connect: connection refused" I1121 21:13:54.561069 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:55.076747 master_server.go:209 [10.128.0.56:9333] - is the leader. I1121 21:13:55.077028 master.go:214 Start Seaweed Master 30GB 3.79 grpc server at 0.0.0.0:19333 + echo cluster.check + timeout 2s weed shell I1121 21:13:55.562598 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:56.564316 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:56.577515 masterclient.go:168 No existing leader found! I1121 21:13:56.577596 raft_server.go:190 Initializing new cluster I1121 21:13:56.577651 master_server.go:183 leader change event: => 10.128.0.56:9333 I1121 21:13:56.577691 master_server.go:186 [10.128.0.56:9333] 10.128.0.56:9333 becomes leader. I1121 21:13:56.635953 disk_location.go:239 Store started on dir: /data with 0 volumes max 40 I1121 21:13:56.636079 disk_location.go:242 Store started on dir: /data with 0 ec shards I1121 21:13:56.636398 volume.go:380 Start Seaweed volume server 30GB 3.79 at 0.0.0.0:8080 I1121 21:13:56.636475 volume_grpc_client_to_master.go:52 Volume server start with seed master nodes: [10.128.0.56:9333] I1121 21:13:56.637585 volume_grpc_client_to_master.go:109 Heartbeat to: 10.128.0.56:9333 I1121 21:13:56.638013 node.go:250 topo adds child DefaultDataCenter I1121 21:13:56.638050 node.go:250 topo:DefaultDataCenter adds child DefaultRack I1121 21:13:56.638062 node.go:250 topo:DefaultDataCenter:DefaultRack adds child 10.128.0.56:8080 I1121 21:13:56.638073 node.go:250 topo:DefaultDataCenter:DefaultRack:10.128.0.56:8080 adds child I1121 21:13:56.638082 master_grpc_server.go:140 added volume server 0: 10.128.0.56:8080 [ada5e99f-ff06-4d50-9420-c5e87a50fad8] I1121 21:13:56.638093 master_grpc_server.go:50 found new uuid:10.128.0.56:8080 [ada5e99f-ff06-4d50-9420-c5e87a50fad8] , map[10.128.0.56:8080:[ada5e99f-ff06-4d50-9420-c5e87a50fad8]] I1121 21:13:56.788612 master_grpc_server.go:365 + client .master@10.128.0.56:9333 I1121 21:13:56.789118 masterclient.go:265 + master@10.128.0.56:9333 noticed .master 10.128.0.56:9333 I1121 21:13:56.862978 master_grpc_server.go:365 + client .adminShell@3d0e214d-5b74-47b7-b917-1c90f8f1cf84 I1121 21:13:56.867162 master_grpc_server.go:380 - client .adminShell@3d0e214d-5b74-47b7-b917-1c90f8f1cf84 I1121 21:13:56.867888 master_grpc_server.go:365 + client .adminShell@8b12f621-38ba-4f7a-a72c-38f94c89b9d7 I1121 21:13:56.991340 master_grpc_server.go:380 - client .adminShell@8b12f621-38ba-4f7a-a72c-38f94c89b9d7 + echo 'Cluster ready for bucket creation.' Cluster ready for bucket creation. + sleep '' sleep: invalid time interval '' Try 'sleep --help' for more information. + for name in "$@" + createBucket archivedrecordings + echo 'Creating S3 bucket archivedrecordings' Creating S3 bucket archivedrecordings + echo 's3.bucket.create -name archivedrecordings' + timeout 5s weed shell master: localhost:9333 filer: I1121 21:13:57.024127 master_grpc_server.go:365 + client .adminShell@1a897e9e-07fa-4beb-ac94-27de2ea75163 I1121 21:13:57.026461 masterclient.go:228 master localhost:9333 redirected to leader 10.128.0.56:9333 I1121 21:13:57.026952 master_grpc_server.go:380 - client .adminShell@1a897e9e-07fa-4beb-ac94-27de2ea75163 I1121 21:13:57.027512 master_grpc_server.go:365 + client .adminShell@c479d2ba-8114-4aee-bb16-e854c03d9457 .master: localhost:9333 > > error: getOrCreateConnection : fail to dial : failed to exit idle mode: passthrough: received empty target in Build() I1121 21:13:57.060598 master_grpc_server.go:380 - client .adminShell@c479d2ba-8114-4aee-bb16-e854c03d9457 + for name in "$@" + createBucket archivedreports + echo 'Creating S3 bucket archivedreports' Creating S3 bucket archivedreports + echo 's3.bucket.create -name archivedreports' + timeout 5s weed shell master: localhost:9333 filer: I1121 21:13:57.092023 master_grpc_server.go:365 + client .adminShell@f0fd6743-e03d-4bdd-becc-5d4bc4a60262 I1121 21:13:57.092340 masterclient.go:228 master localhost:9333 redirected to leader 10.128.0.56:9333 I1121 21:13:57.092755 master_grpc_server.go:380 - client .adminShell@f0fd6743-e03d-4bdd-becc-5d4bc4a60262 I1121 21:13:57.093545 master_grpc_server.go:365 + client .adminShell@321a5263-9304-4dcd-9e56-a3bceccbb212 .master: localhost:9333 > error: getOrCreateConnection : fail to dial : failed to exit idle mode: passthrough: received empty target in Build() > I1121 21:13:57.255637 master_grpc_server.go:380 - client .adminShell@321a5263-9304-4dcd-9e56-a3bceccbb212 + for name in "$@" + createBucket eventtemplates Creating S3 bucket eventtemplates + echo 'Creating S3 bucket eventtemplates' + echo 's3.bucket.create -name eventtemplates' + timeout 5s weed shell master: localhost:9333 filer: I1121 21:13:57.284328 master_grpc_server.go:365 + client .adminShell@8f97b483-61ee-4eca-b68d-0201bd4d4267 I1121 21:13:57.284589 masterclient.go:228 master localhost:9333 redirected to leader 10.128.0.56:9333 I1121 21:13:57.284989 master_grpc_server.go:380 - client .adminShell@8f97b483-61ee-4eca-b68d-0201bd4d4267 I1121 21:13:57.285823 master_grpc_server.go:365 + client .adminShell@012bf8a3-cb1b-4573-b61f-e7d41b6fa114 .master: localhost:9333 > > error: getOrCreateConnection : fail to dial : failed to exit idle mode: passthrough: received empty target in Build() I1121 21:13:57.328256 master_grpc_server.go:380 - client .adminShell@012bf8a3-cb1b-4573-b61f-e7d41b6fa114 + for name in "$@" + createBucket probes + echo 'Creating S3 bucket probes' Creating S3 bucket probes + echo 's3.bucket.create -name probes' + timeout 5s weed shell master: localhost:9333 filer: I1121 21:13:57.361431 master_grpc_server.go:365 + client .adminShell@6b11d0b0-d0fe-4b36-830f-199a19e070ff I1121 21:13:57.361694 masterclient.go:228 master localhost:9333 redirected to leader 10.128.0.56:9333 I1121 21:13:57.362269 master_grpc_server.go:380 - client .adminShell@6b11d0b0-d0fe-4b36-830f-199a19e070ff I1121 21:13:57.363060 master_grpc_server.go:365 + client .adminShell@42686c4f-6452-4281-84ce-b8e0678ce31b .master: localhost:9333 > error: getOrCreateConnection : fail to dial : failed to exit idle mode: passthrough: received empty target in Build() > I1121 21:13:57.478948 master_grpc_server.go:380 - client .adminShell@42686c4f-6452-4281-84ce-b8e0678ce31b I1121 21:13:57.565530 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:58.557583 filer_server.go:171 default to create filer store dir in /data/filerldb2 I1121 21:13:58.558033 leveldb2_store.go:43 filer store leveldb2 dir: /data/filerldb2 I1121 21:13:58.558061 file_util.go:27 Folder /data/filerldb2 Permission: -rwxr-xr-x I1121 21:13:58.558104 master_grpc_server.go:365 + client .filer@10.128.0.56:8888 I1121 21:13:58.558281 masterclient.go:265 + master@10.128.0.56:9333 noticed .filer 10.128.0.56:8888 I1121 21:13:58.566059 s3.go:217 wait to connect to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:58.607416 filer.go:153 create filer.store.id to -1586782763 I1121 21:13:58.607435 configuration.go:28 configured filer store to leveldb2 I1121 21:13:58.607994 master_client.go:20 the cluster has 1 filer I1121 21:13:58.608016 filer_server.go:201 10.128.0.56:8888 bootstrap from peers [node_type:"filer" address:"10.128.0.56:8888" is_add:true created_at_ns:1732223638558122245] I1121 21:13:58.608063 filer.go:111 10.128.0.56:8888 aggregate from peers [10.128.0.56:8888] I1121 21:13:58.608231 filer.go:317 Start Seaweed Filer 30GB 3.79 at 10.128.0.56:8888 I1121 21:13:58.608276 meta_aggregator.go:76 loopSubscribeToOneFiler read 10.128.0.56:8888 start from 2024-11-21 21:12:58.608007828 +0000 UTC 1732223578608007828 I1121 21:13:58.609224 meta_aggregator.go:178 subscribing remote 10.128.0.56:8888 meta change: 2024-11-21 21:12:58.608007828 +0000 UTC, clientId:2023734233 I1121 21:13:58.609768 filer_grpc_server_sub_meta.go:305 + local listener filer:10.128.0.56:8888@10.128.0.56:38968 clientId -2023734233 clientEpoch 1 I1121 21:13:58.609782 filer_grpc_server_sub_meta.go:124 + filer:10.128.0.56:8888@10.128.0.56:38968 local subscribe / from 2024-11-21 21:12:58.608007828 +0000 UTC clientId:-2023734233 I1121 21:13:58.609793 filer_grpc_server_sub_meta.go:137 read on disk filer:10.128.0.56:8888@10.128.0.56:38968 local subscribe / from 2024-11-21 21:12:58.608007828 +0000 UTC I1121 21:13:58.609818 filer_grpc_server_sub_meta.go:156 read in memory filer:10.128.0.56:8888@10.128.0.56:38968 local subscribe / from 2024-11-21 21:12:58.608007828 +0000 UTC I1121 21:13:59.566642 s3.go:213 S3 read filer buckets dir: /buckets I1121 21:13:59.566661 s3.go:220 connected to filer 10.128.0.56:8888 grpc address 10.128.0.56:18888 I1121 21:13:59.567258 s3api_circuit_breaker.go:35 s3 circuit breaker not configured I1121 21:13:59.568649 s3.go:369 Start Seaweed S3 API Server 30GB 3.79 at http port 8333 I1121 21:13:59.569458 filer_grpc_server_sub_meta.go:305 + listener s3@10.128.0.56:38974 clientId -1388071164 clientEpoch 2 I1121 21:13:59.569474 filer_grpc_server_sub_meta.go:42 s3@10.128.0.56:38974 starts to subscribe /etc/ from 2024-11-21 21:13:59.56668218 +0000 UTC I1121 21:13:59.720326 master_grpc_server.go:365 + client .adminShell@67931fb7-3ff6-4af1-89b3-6dd8de48ff32 I1121 21:13:59.721117 master_grpc_server.go:380 - client .adminShell@67931fb7-3ff6-4af1-89b3-6dd8de48ff32 I1121 21:13:59.721706 master_grpc_server.go:365 + client .adminShell@7c848e7c-9a65-4dae-b7bc-ce3f7b7fa999