From f2b702a22f7a31fa532fb94907cc61bbdf50da2f Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Wed, 20 Dec 2023 19:34:22 +0800 Subject: [PATCH 1/3] fix: trim minio basepath when merging chunks --- routers/api/actions/artifacts_chunks.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/routers/api/actions/artifacts_chunks.go b/routers/api/actions/artifacts_chunks.go index c7ab70afa950e..5d87ef9ad7295 100644 --- a/routers/api/actions/artifacts_chunks.go +++ b/routers/api/actions/artifacts_chunks.go @@ -10,11 +10,13 @@ import ( "io" "path/filepath" "sort" + "strings" "time" "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) @@ -26,6 +28,7 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext, contentRange := ctx.Req.Header.Get("Content-Range") start, end, length := int64(0), int64(0), int64(0) if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil { + log.Warn("parse content range error: %v, content-range: %s", err, contentRange) return -1, fmt.Errorf("parse content range error: %v", err) } // build chunk store path @@ -64,10 +67,16 @@ type chunkFileItem struct { Path string } -func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chunkFileItem, error) { +func listChunksByRunID(st storage.ObjectStorage, runID int64, basepath string) (map[int64][]*chunkFileItem, error) { storageDir := fmt.Sprintf("tmp%d", runID) var chunks []*chunkFileItem if err := st.IterateObjects(storageDir, func(path string, obj storage.Object) error { + // if basepath is settings.Actions.ArtifactStorage.MinioConfig.BasePath, it should trim it + if basepath != "" { + log.Debug("listChunksByRunID, trim basepath: %s, path: %s", basepath, path) + basepath = strings.TrimPrefix(basepath, "/") + path = strings.TrimPrefix(path, basepath+"/") + } item := chunkFileItem{Path: path} if _, err := fmt.Sscanf(path, filepath.Join(storageDir, "%d-%d-%d.chunk"), &item.ArtifactID, &item.Start, &item.End); err != nil { return fmt.Errorf("parse content range error: %v", err) @@ -94,8 +103,11 @@ func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID int if err != nil { return err } + // if use minio, it should trim basepath when iterate objects for chunks + // if use local, basepath is empty + basepath := setting.Actions.ArtifactStorage.MinioConfig.BasePath // read all uploading chunks from storage - chunksMap, err := listChunksByRunID(st, runID) + chunksMap, err := listChunksByRunID(st, runID, basepath) if err != nil { return err } From b70a3cc82b6651eeb40124492f74c951374403a6 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Wed, 20 Dec 2023 20:04:08 +0800 Subject: [PATCH 2/3] fix: update artifact chunk basename to ignore directory value in storage settings --- routers/api/actions/artifacts_chunks.go | 28 ++++++++++--------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/routers/api/actions/artifacts_chunks.go b/routers/api/actions/artifacts_chunks.go index 5d87ef9ad7295..60e08a304b626 100644 --- a/routers/api/actions/artifacts_chunks.go +++ b/routers/api/actions/artifacts_chunks.go @@ -8,15 +8,14 @@ import ( "encoding/base64" "fmt" "io" + "path" "path/filepath" "sort" - "strings" "time" "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) @@ -32,7 +31,7 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext, return -1, fmt.Errorf("parse content range error: %v", err) } // build chunk store path - storagePath := fmt.Sprintf("tmp%d/%d-%d-%d.chunk", runID, artifact.ID, start, end) + storagePath := fmt.Sprintf("tmp%d/%d-%d-%d-%d.chunk", runID, runID, artifact.ID, start, end) // use io.TeeReader to avoid reading all body to md5 sum. // it writes data to hasher after reading end // if hash is not matched, delete the read-end result @@ -61,24 +60,22 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext, } type chunkFileItem struct { + RunID int64 ArtifactID int64 Start int64 End int64 Path string } -func listChunksByRunID(st storage.ObjectStorage, runID int64, basepath string) (map[int64][]*chunkFileItem, error) { +func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chunkFileItem, error) { storageDir := fmt.Sprintf("tmp%d", runID) var chunks []*chunkFileItem - if err := st.IterateObjects(storageDir, func(path string, obj storage.Object) error { - // if basepath is settings.Actions.ArtifactStorage.MinioConfig.BasePath, it should trim it - if basepath != "" { - log.Debug("listChunksByRunID, trim basepath: %s, path: %s", basepath, path) - basepath = strings.TrimPrefix(basepath, "/") - path = strings.TrimPrefix(path, basepath+"/") - } - item := chunkFileItem{Path: path} - if _, err := fmt.Sscanf(path, filepath.Join(storageDir, "%d-%d-%d.chunk"), &item.ArtifactID, &item.Start, &item.End); err != nil { + if err := st.IterateObjects(storageDir, func(fpath string, obj storage.Object) error { + baseName := filepath.Base(fpath) + // when read chunks from storage, it only contains storage dir and basename, + // no matter the subdirectory setting in storage config + item := chunkFileItem{Path: path.Join(storageDir, baseName)} + if _, err := fmt.Sscanf(baseName, "%d-%d-%d-%d.chunk", &item.RunID, &item.ArtifactID, &item.Start, &item.End); err != nil { return fmt.Errorf("parse content range error: %v", err) } chunks = append(chunks, &item) @@ -103,11 +100,8 @@ func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID int if err != nil { return err } - // if use minio, it should trim basepath when iterate objects for chunks - // if use local, basepath is empty - basepath := setting.Actions.ArtifactStorage.MinioConfig.BasePath // read all uploading chunks from storage - chunksMap, err := listChunksByRunID(st, runID, basepath) + chunksMap, err := listChunksByRunID(st, runID) if err != nil { return err } From 364497582edacd32038f246dd0bf75cd6b15716c Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Wed, 20 Dec 2023 23:16:51 +0800 Subject: [PATCH 3/3] update filepath to handle file's path from storage --- routers/api/actions/artifacts_chunks.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routers/api/actions/artifacts_chunks.go b/routers/api/actions/artifacts_chunks.go index 60e08a304b626..36432a0ca084e 100644 --- a/routers/api/actions/artifacts_chunks.go +++ b/routers/api/actions/artifacts_chunks.go @@ -8,7 +8,6 @@ import ( "encoding/base64" "fmt" "io" - "path" "path/filepath" "sort" "time" @@ -74,7 +73,7 @@ func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chun baseName := filepath.Base(fpath) // when read chunks from storage, it only contains storage dir and basename, // no matter the subdirectory setting in storage config - item := chunkFileItem{Path: path.Join(storageDir, baseName)} + item := chunkFileItem{Path: storageDir + "/" + baseName} if _, err := fmt.Sscanf(baseName, "%d-%d-%d-%d.chunk", &item.RunID, &item.ArtifactID, &item.Start, &item.End); err != nil { return fmt.Errorf("parse content range error: %v", err) }