fix: clean up website dataset when updating chunk settings (#4420)

Author: I-Info

Diff for: packages/service/core/dataset/controller.ts

@@ -90,15 +90,6 @@ export async function delDatasetRelevantData({
     '_id teamId datasetId fileId metadata'
   ).lean();
 
-  const removeJobScheduler = async () => {
-    await Promise.all(
-      datasets.map((dataset) => {
-        if (dataset.type === DatasetTypeEnum.websiteDataset)
-          return removeWebsiteSyncJobScheduler(String(dataset._id));
-      })
-    );
-  };
-
   await retryFn(async () => {
     await Promise.all([
       // delete training data
@@ -116,9 +107,7 @@ export async function delDatasetRelevantData({
       // Delete Image and file
       delCollectionRelatedSource({ collections }),
       // Delete vector data
-      deleteDatasetDataVector({ teamId, datasetIds }),
-      // Remove job scheduler
-      removeJobScheduler()
+      deleteDatasetDataVector({ teamId, datasetIds })
     ]);
   });
 

Diff for: projects/app/src/pages/api/core/dataset/delete.ts

@@ -9,6 +9,8 @@ import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant'
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema';
 import { removeImageByPath } from '@fastgpt/service/common/file/image/controller';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { removeWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
 
 async function handler(req: NextApiRequest) {
   const { id: datasetId } = req.query as {
@@ -40,6 +42,13 @@ async function handler(req: NextApiRequest) {
     datasetId: { $in: datasetIds }
   });
 
+  await Promise.all(
+    datasets.map((dataset) => {
+      if (dataset.type === DatasetTypeEnum.websiteDataset)
+        return removeWebsiteSyncJobScheduler(String(dataset._id));
+    })
+  );
+
   // delete all dataset.data and pg data
   await mongoSessionRun(async (session) => {
     // delete dataset data

Diff for: projects/app/src/pages/api/core/dataset/update.ts

@@ -35,6 +35,7 @@ import {
   removeWebsiteSyncJobScheduler,
   upsertWebsiteSyncJobScheduler
 } from '@fastgpt/service/core/dataset/websiteSync';
+import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
 
 export type DatasetUpdateQuery = {};
 export type DatasetUpdateResponse = any;
@@ -119,6 +120,11 @@ async function handler(
   });
 
   const onUpdate = async (session: ClientSession) => {
+    if (dataset.type === DatasetTypeEnum.websiteDataset && chunkSettings) {
+      // clean up dataset
+      await delDatasetRelevantData({ datasets: [dataset], session });
+    }
+
     await MongoDataset.findByIdAndUpdate(
       id,
       {