feat: customize ignore paths of indexes

Author: xhofe

internal/bootstrap/data/setting.go

@@ -131,7 +131,8 @@ func InitialSettings() []model.SettingItem {
 
 		// single settings
 		{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
-		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.SINGLE},
+		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX},
+		{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
 	}
 	if flags.Dev {

internal/conf/const.go

@@ -41,7 +41,10 @@ const (
 	PrivacyRegs         = "privacy_regs"
 	OcrApi              = "ocr_api"
 	FilenameCharMapping = "filename_char_mapping"
-	SearchIndex         = "search_index"
+
+	// index
+	SearchIndex = "search_index"
+	IgnorePaths = "ignore_paths"
 
 	// aria2
 	Aria2Uri    = "aria2_uri"

internal/model/setting.go

@@ -1,12 +1,13 @@
 package model
 
 const (
-	SITE = iota
+	SINGLE = iota
+	SITE
 	STYLE
 	PREVIEW
 	GLOBAL
-	SINGLE
 	ARIA2
+	INDEX
 )
 
 const (

internal/search/build.go

@@ -11,8 +11,10 @@ import (
 	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/pkg/mq"
 	"github.com/alist-org/alist/v3/pkg/utils"
+	mapset "github.com/deckarep/golang-set/v2"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -22,18 +24,8 @@ var (
 )
 
 func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error {
-	storages, err := db.GetEnabledStorages()
-	if err != nil {
-		return err
-	}
-	var skipDrivers = []string{"AList V2", "AList V3"}
-	for _, storage := range storages {
-		if utils.SliceContains(skipDrivers, storage.Driver) {
-			// TODO: request for indexing permission
-			ignorePaths = append(ignorePaths, storage.MountPath)
-		}
-	}
 	var (
+		err      error
 		objCount uint64 = 0
 		fi       model.Obj
 	)
@@ -154,3 +146,73 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 func Clear(ctx context.Context) error {
 	return instance.Clear(ctx)
 }
+
+func Update(parent string, objs []model.Obj) {
+	if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
+		return
+	}
+	ignorePaths, err := GetIgnorePaths()
+	if err != nil {
+		log.Errorf("update search index error while get ignore paths: %+v", err)
+		return
+	}
+	if isIgnorePath(parent, ignorePaths) {
+		return
+	}
+	ctx := context.Background()
+	// only update when index have built
+	progress, err := Progress()
+	if err != nil {
+		log.Errorf("update search index error while get progress: %+v", err)
+		return
+	}
+	if !progress.IsDone {
+		return
+	}
+	nodes, err := instance.Get(ctx, parent)
+	if err != nil {
+		log.Errorf("update search index error while get nodes: %+v", err)
+		return
+	}
+	now := mapset.NewSet[string]()
+	for i := range objs {
+		now.Add(objs[i].GetName())
+	}
+	old := mapset.NewSet[string]()
+	for i := range nodes {
+		old.Add(nodes[i].Name)
+	}
+	// delete data that no longer exists
+	toDelete := old.Difference(now)
+	toAdd := now.Difference(old)
+	for i := range nodes {
+		if toDelete.Contains(nodes[i].Name) {
+			err = instance.Del(ctx, path.Join(parent, nodes[i].Name))
+			if err != nil {
+				log.Errorf("update search index error while del old node: %+v", err)
+				return
+			}
+		}
+	}
+	for i := range objs {
+		if toAdd.Contains(objs[i].GetName()) {
+			err = Index(ctx, parent, objs[i])
+			if err != nil {
+				log.Errorf("update search index error while index new node: %+v", err)
+				return
+			}
+			// build index if it's a folder
+			if objs[i].IsDir() {
+				err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, ignorePaths, -1, false)
+				if err != nil {
+					log.Errorf("update search index error while build index: %+v", err)
+					return
+				}
+			}
+		}
+	}
+}
+
+func init() {
+	op.RegisterObjsUpdateHook(Update)
+}

internal/search/update.go

@@ -1,6 +1,8 @@
 package search
 
 import (
+	"strings"
+
 	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
@@ -32,3 +34,32 @@ func WriteProgress(progress *model.IndexProgress) {
 		log.Errorf("save progress error: %+v", err)
 	}
 }
+
+func GetIgnorePaths() ([]string, error) {
+	storages, err := db.GetEnabledStorages()
+	if err != nil {
+		return nil, err
+	}
+	ignorePaths := make([]string, 0)
+	var skipDrivers = []string{"AList V2", "AList V3"}
+	for _, storage := range storages {
+		if utils.SliceContains(skipDrivers, storage.Driver) {
+			// TODO: request for indexing permission
+			ignorePaths = append(ignorePaths, storage.MountPath)
+		}
+	}
+	customIgnorePaths := setting.GetStr(conf.IgnorePaths)
+	if customIgnorePaths != "" {
+		ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...)
+	}
+	return ignorePaths, nil
+}
+
+func isIgnorePath(path string, ignorePaths []string) bool {
+	for _, ignorePath := range ignorePaths {
+		if strings.HasPrefix(path, ignorePath) {
+			return true
+		}
+	}
+	return false
+}

internal/search/progress.go internal/search/util.go

@@ -25,14 +25,20 @@ func BuildIndex(c *gin.Context) {
 		common.ErrorStrResp(c, "index is running", 400)
 		return
 	}
+	ignorePaths, err := search.GetIgnorePaths()
+	if err != nil {
+		common.ErrorResp(c, err, 500)
+		return
+	}
+	ignorePaths = append(ignorePaths, req.IgnorePaths...)
 	go func() {
 		ctx := context.Background()
 		err := search.Clear(ctx)
 		if err != nil {
 			log.Errorf("clear index error: %+v", err)
 			return
 		}
-		err = search.BuildIndex(context.Background(), req.Paths, req.IgnorePaths, req.MaxDepth, true)
+		err = search.BuildIndex(context.Background(), req.Paths, ignorePaths, req.MaxDepth, true)
 		if err != nil {
 			log.Errorf("build index error: %+v", err)
 		}