Small PR fixes, and optimisation of GetFilesInDir

wtsi-hgi · Feb 18, 2025 · 0befb81 · 0befb81
1 parent 3579f8b
commit 0befb81
Show file tree

Hide file tree

Showing 8 changed files with 58 additions and 27 deletions.
diff --git a/put/baton.go b/put/baton.go
@@ -502,7 +502,7 @@ func metaToAVUs(meta map[string]string) []ex.AVU {
 	return avus
 }
 
-func (b *Baton) removeFile(path string) error {
+func (b *Baton) RemoveFile(path string) error {
 	it := remotePathToRodsItem(path)
 
 	err := timeoutOp(func() error {
@@ -514,7 +514,7 @@ func (b *Baton) removeFile(path string) error {
 	return err
 }
 
-func (b *Baton) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
+func (b *Baton) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
 	it := &ex.RodsItem{
 		IPath: dirToSearch,
 		IAVUs: metaToAVUs(meta),
@@ -539,7 +539,7 @@ func (b *Baton) queryMeta(dirToSearch string, meta map[string]string) ([]string,
 	return paths, err
 }
 
-// RemoveDirFromIRODS removes the given directory from iRODS given it is empty.
+// RemoveDir removes the given directory from iRODS given it is empty.
 func (b *Baton) RemoveDir(path string) error {
 	it := &ex.RodsItem{
 		IPath: path,

diff --git a/put/mock.go b/put/mock.go
@@ -258,13 +258,13 @@ func (l *LocalHandler) RemoveDir(path string) error {
 	return os.Remove(path)
 }
 
-func (l *LocalHandler) removeFile(path string) error {
+func (l *LocalHandler) RemoveFile(path string) error {
 	delete(l.meta, path)
 
 	return os.Remove(path)
 }
 
-func (l *LocalHandler) queryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
+func (l *LocalHandler) QueryMeta(dirToSearch string, meta map[string]string) ([]string, error) {
 	var objects []string
 
 	for path, pathMeta := range l.meta {

diff --git a/put/put.go b/put/put.go
@@ -124,11 +124,11 @@ type Handler interface {
 	// RemoveDir deletes a given empty folder
 	RemoveDir(path string) error
 
-	// removeFile deletes a given file
-	removeFile(path string) error
+	// RemoveFile deletes a given file
+	RemoveFile(path string) error
 
-	// queryMeta return paths to all objects with given metadata
-	queryMeta(dirToSearch string, meta map[string]string) ([]string, error)
+	// QueryMeta return paths to all objects with given metadata
+	QueryMeta(dirToSearch string, meta map[string]string) ([]string, error)
 }
 
 // FileReadTester is a function that attempts to open and read the given path,

diff --git a/put/put_test.go b/put/put_test.go
@@ -173,7 +173,7 @@ func TestPutMock(t *testing.T) {
 				Convey("RemoveFile removes a file", func() {
 					filePath := requests[0].Remote
 
-					err = lh.removeFile(filePath)
+					err = lh.RemoveFile(filePath)
 					So(err, ShouldBeNil)
 
 					_, err = os.Stat(filePath)
@@ -193,18 +193,18 @@ func TestPutMock(t *testing.T) {
 				})
 
 				Convey("queryMeta returns all paths with matching metadata", func() {
-					paths, errq := lh.queryMeta("", map[string]string{"a": "1"})
+					paths, errq := lh.QueryMeta("", map[string]string{"a": "1"})
 					So(errq, ShouldBeNil)
 
 					So(len(paths), ShouldEqual, 5)
 
-					paths, err = lh.queryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester})
+					paths, err = lh.QueryMeta("", map[string]string{MetaKeyRequester: requests[0].Requester})
 					So(err, ShouldBeNil)
 
 					So(len(paths), ShouldEqual, 1)
 
 					Convey("queryMeta only returns paths in the provided scope", func() {
-						paths, errq := lh.queryMeta(expectedCollections[1], map[string]string{"a": "1"})
+						paths, errq := lh.QueryMeta(expectedCollections[1], map[string]string{"a": "1"})
 						So(errq, ShouldBeNil)
 
 						So(len(paths), ShouldEqual, 2)

diff --git a/put/remove.go b/put/remove.go
@@ -67,7 +67,7 @@ func RemovePathFromSetInIRODS(handler Handler, transformer PathTransformer, path
 // same file, if not, it removes the file.
 func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer PathTransformer,
 	meta map[string]string) error {
-	err := handler.removeFile(path)
+	err := handler.RemoveFile(path)
 	if err != nil {
 		return err
 	}
@@ -81,7 +81,7 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer
 		return err
 	}
 
-	items, err := handler.queryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]})
+	items, err := handler.QueryMeta(dirToSearch, map[string]string{MetaKeyRemoteHardlink: meta[MetaKeyRemoteHardlink]})
 	if err != nil {
 		return err
 	}
@@ -90,7 +90,7 @@ func handleHardlinkAndRemoveFromIRODS(handler Handler, path string, transformer
 		return nil
 	}
 
-	return handler.removeFile(meta[MetaKeyRemoteHardlink])
+	return handler.RemoveFile(meta[MetaKeyRemoteHardlink])
 }
 
 // RemoveDirFromIRODS removes the remote path of a given directory from iRODS.

diff --git a/server/server.go b/server/server.go
@@ -29,6 +29,7 @@ package server
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -40,7 +41,6 @@ import (
 	jqs "github.com/VertebrateResequencing/wr/jobqueue/scheduler"
 	"github.com/VertebrateResequencing/wr/queue"
 	"github.com/gammazero/workerpool"
-	"github.com/goccy/go-json"
 	"github.com/inconshreveable/log15"
 	gas "github.com/wtsi-hgi/go-authserver"
 	"github.com/wtsi-hgi/ibackup/put"

diff --git a/set/db.go b/set/db.go
@@ -369,23 +369,41 @@ func (d *DB) RemoveDirEntry(setID string, path string) error {
 
 // GetFilesInDir returns all file paths from inside the given directory (and all
 // nested inside) for the given set using the db.
-func (d *DB) GetFilesInDir(setID string, dirpath string) ([]string, error) {
-	var filepaths []string
-
-	entries, err := d.getEntries(setID, discoveredBucket)
+func (d *DBRO) GetFilesInDir(setID string, dirpath string) ([]string, error) {
+	filepaths, err := d.getPathsWithPrefix(setID, discoveredBucket, dirpath)
 	if err != nil {
 		return nil, err
 	}
 
-	for _, entry := range entries {
-		path := entry.Path
+	return filepaths, nil
+}
+
+// getPathsWithPrefix returns all the filepaths for the given set from the given sub
+// bucket prefix, that have the given prefix.
+func (d *DBRO) getPathsWithPrefix(setID, bucketName, prefix string) ([]string, error) {
+	var entries []string
 
-		if strings.HasPrefix(path, dirpath) {
-			filepaths = append(filepaths, path)
+	err := d.db.View(func(tx *bolt.Tx) error {
+		subBucketName := []byte(bucketName + separator + setID)
+		setsBucket := tx.Bucket([]byte(setsBucket))
+
+		entriesBucket := setsBucket.Bucket(subBucketName)
+		if entriesBucket == nil {
+			return nil
 		}
-	}
 
-	return filepaths, nil
+		c := entriesBucket.Cursor()
+
+		prefix := []byte(prefix)
+
+		for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() {
+			entries = append(entries, string(k))
+		}
+
+		return nil
+	})
+
+	return entries, err
 }
 
 // SetFileEntries sets the file paths for the given backup set. Only supply

diff --git a/set/set_test.go b/set/set_test.go
@@ -358,6 +358,19 @@ func TestSetDB(t *testing.T) {
 				err = db.SetFileEntries(set2.ID(), []string{"/a/b.txt", "/c/k.txt"})
 				So(err, ShouldBeNil)
 
+				Convey("You can get all paths containing a prefix", func() {
+					err = db.SetFileEntries(set2.ID(), []string{"/a/a/j.txt", "/a/b/c/k.txt",
+						"/a/b/c/l.txt", "/a/b/d/m.txt", "/c/n.txt"})
+					So(err, ShouldBeNil)
+
+					files, errp := db.getPathsWithPrefix(set2.ID(), fileBucket, "/a/b/c/")
+					So(errp, ShouldBeNil)
+
+					So(len(files), ShouldEqual, 2)
+					So(files, ShouldContain, "/a/b/c/k.txt")
+					So(files, ShouldContain, "/a/b/c/l.txt")
+				})
+
 				Convey("Then remove files and dirs from the sets", func() {
 					err = db.removeEntry(set.ID(), "/a/b.txt", fileBucket)
 					So(err, ShouldBeNil)