fix: invalid dataset data clear (#3927)

* fix: collection list count

* fix: collection list count

* fix: invalid dataset data clear

* update ts

* perf: cron clear invalid data

* perf: init

* perf: clear invalid code

* update init

* perf: clear invalid code

* perf: clear invalid code

* perf: init count

* batch init

* batch init

* batch init

* batch init

* add comment

* perf: init

* fix: api proxy type

packages/service/core/dataset/collection/controller.ts

@@ -227,7 +227,13 @@ export const delCollectionRelatedSource = async ({
   collections,
   session
 }: {
-  collections: DatasetCollectionSchemaType[];
+  collections: {
+    teamId: string;
+    fileId?: string;
+    metadata?: {
+      relatedImgId?: string;
+    };
+  }[];
   session: ClientSession;
 }) => {
   if (collections.length === 0) return;
@@ -259,11 +265,13 @@ export const delCollectionRelatedSource = async ({
 export async function delCollection({
   collections,
   session,
-  delRelatedSource
+  delImg = true,
+  delFile = true
 }: {
   collections: DatasetCollectionSchemaType[];
   session: ClientSession;
-  delRelatedSource: boolean;
+  delImg: boolean;
+  delFile: boolean;
 }) {
   if (collections.length === 0) return;
 
@@ -281,9 +289,18 @@ export async function delCollection({
     collectionId: { $in: collectionIds }
   });
 
-  /* file and imgs */
-  if (delRelatedSource) {
-    await delCollectionRelatedSource({ collections, session });
+  if (delImg) {
+    await delImgByRelatedId({
+      teamId,
+      relateIds: collections.map((item) => item?.metadata?.relatedImgId || '').filter(Boolean),
+      session
+    });
+  }
+  if (delFile) {
+    await delFileByFileIdList({
+      bucketName: BucketNameEnum.dataset,
+      fileIdList: collections.map((item) => item?.fileId || '').filter(Boolean)
+    });
   }
 
   // Delete dataset_datas
@@ -309,48 +326,3 @@ export async function delCollection({
   // no session delete: delete files, vector data
   await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
 }
-
-/**
- * delete delOnlyCollection
- */
-export async function delOnlyCollection({
-  collections,
-  session
-}: {
-  collections: DatasetCollectionSchemaType[];
-  session: ClientSession;
-}) {
-  if (collections.length === 0) return;
-
-  const teamId = collections[0].teamId;
-
-  if (!teamId) return Promise.reject('teamId is not exist');
-
-  const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
-  const collectionIds = collections.map((item) => String(item._id));
-
-  // delete training data
-  await MongoDatasetTraining.deleteMany({
-    teamId,
-    datasetId: { $in: datasetIds },
-    collectionId: { $in: collectionIds }
-  });
-
-  // delete dataset.datas
-  await MongoDatasetData.deleteMany(
-    { teamId, datasetId: { $in: datasetIds }, collectionId: { $in: collectionIds } },
-    { session }
-  );
-
-  // delete collections
-  await MongoDatasetCollection.deleteMany(
-    {
-      teamId,
-      _id: { $in: collectionIds }
-    },
-    { session }
-  );
-
-  // no session delete: delete files, vector data
-  await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
-}

packages/service/core/dataset/collection/utils.ts

@@ -174,6 +174,14 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
   }
 
   await mongoSessionRun(async (session) => {
+    // Delete old collection
+    await delCollection({
+      collections: [collection],
+      delImg: false,
+      delFile: false,
+      session
+    });
+
     // Create new collection
     await createCollectionAndInsertData({
       session,
@@ -208,13 +216,6 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
         updateTime: new Date()
       }
     });
-
-    // Delete old collection
-    await delCollection({
-      collections: [collection],
-      delRelatedSource: false,
-      session
-    });
   });
 
   return DatasetCollectionSyncResultEnum.success;

packages/service/core/dataset/data/dataTextSchema.ts

@@ -1,6 +1,6 @@
 import { connectionMongo, getMongoModel } from '../../../common/mongo';
 const { Schema } = connectionMongo;
-import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
+import { DatasetDataTextSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
 import { DatasetCollectionName } from '../schema';
 import { DatasetColCollectionName } from '../collection/schema';
@@ -45,7 +45,7 @@ try {
   console.log(error);
 }
 
-export const MongoDatasetDataText = getMongoModel<DatasetDataSchemaType>(
+export const MongoDatasetDataText = getMongoModel<DatasetDataTextSchemaType>(
   DatasetDataTextCollectionName,
   DatasetDataTextSchema
 );