fix(scheduler): sync instance info memory leak (#1485) (#1493)

Co-authored-by: Muzry <muzrry@gmail.com>
erda-project · Aug 18, 2021 · a38520d · a38520d
1 parent 2c863de
commit a38520d
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 9 deletions.
diff --git a/modules/scheduler/executor/plugins/k8s/instanceinfosync/gc.go b/modules/scheduler/executor/plugins/k8s/instanceinfosync/gc.go
@@ -22,12 +22,15 @@ import (
 	"github.com/erda-project/erda/modules/scheduler/instanceinfo"
 )
 
-// gcDeadInstancesInDB Recover the instance of phase=Dead 15 days ago
-func gcDeadInstancesInDB(dbclient *instanceinfo.Client) error {
+// gcDeadInstancesInDB Recover the instance of phase=Dead 7 days ago
+func gcDeadInstancesInDB(dbclient *instanceinfo.Client, clusterName string) error {
 	r := dbclient.InstanceReader()
 	w := dbclient.InstanceWriter()
 
-	instances, err := r.ByPhase(instanceinfo.InstancePhaseDead).ByFinishedTime(7).Do()
+	// list instance info in database with limit 3000
+	r.Limit(3000)
+
+	instances, err := r.ByCluster(clusterName).ByPhase(instanceinfo.InstancePhaseDead).ByFinishedTime(7).Do()
 	if err != nil {
 		return err
 	}

diff --git a/modules/scheduler/executor/plugins/k8s/instanceinfosync/sync.go b/modules/scheduler/executor/plugins/k8s/instanceinfosync/sync.go
@@ -93,7 +93,7 @@ func (s *Syncer) watchSync(ctx context.Context) {
 }
 
 func (s *Syncer) gc(ctx context.Context) {
-	go s.gcDeadInstances(ctx)
+	go s.gcDeadInstances(ctx, s.clustername)
 	go s.gcServices(ctx)
 }
 
@@ -268,17 +268,17 @@ func (s *Syncer) watchSyncEvent(ctx context.Context) {
 	}
 }
 
-func (s *Syncer) gcDeadInstances(ctx context.Context) {
-	if err := gcDeadInstancesInDB(s.dbupdater); err != nil {
+func (s *Syncer) gcDeadInstances(ctx context.Context, clusterName string) {
+	if err := gcDeadInstancesInDB(s.dbupdater, clusterName); err != nil {
 		logrus.Errorf("failed to gcInstancesInDB: %v", err)
 	}
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.After(time.Duration(24) * time.Hour):
+		case <-time.After(time.Duration(1) * time.Hour):
 		}
-		if err := gcDeadInstancesInDB(s.dbupdater); err != nil {
+		if err := gcDeadInstancesInDB(s.dbupdater, clusterName); err != nil {
 			logrus.Errorf("failed to gcInstancesInDB: %v", err)
 		}
 	}

diff --git a/modules/scheduler/executor/plugins/k8s/k8s.go b/modules/scheduler/executor/plugins/k8s/k8s.go
@@ -127,7 +127,6 @@ func init() {
 
 		parentctx, cancelSyncInstanceinfo := context.WithCancel(context.Background())
 		k.instanceinfoSyncCancelFunc = cancelSyncInstanceinfo
-
 		go func() {
 			for {
 				select {