ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-10-18 17:51:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.6.7-alpha commit (#743)

Browse Source 
Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-01-19 11:17:28 +08:00
committed by GitHub
parent 8ee7407c4c
commit c031e6dcc9
324 changed files with 8509 additions and 4757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
packages/service/core/dataset/data/controller.ts
View File

@@ -1,87 +1,2 @@
import { MongoDatasetData } from './schema';
import { MongoDatasetTraining } from '../training/schema';
import { delFileByFileIdList, delFileByMetadata } from '../../../common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { MongoDatasetCollection } from '../collection/schema';
import { delay } from '@fastgpt/global/common/system/utils';
import { delImgByFileIdList } from '../../../common/file/image/controller';
import { deleteDatasetDataVector } from '../../../common/vectorStore/controller';
/* delete all data by datasetIds */
export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) {
datasetIds = datasetIds.map((item) => String(item));
// delete training data(There could be a training mission)
await MongoDatasetTraining.deleteMany({
datasetId: { $in: datasetIds }
});
await delay(2000);
// delete dataset.datas
await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
// delete pg data
await deleteDatasetDataVector({ datasetIds });
// delete collections
await MongoDatasetCollection.deleteMany({
datasetId: { $in: datasetIds }
});
// delete related files
await Promise.all(
datasetIds.map((id) => delFileByMetadata({ bucketName: BucketNameEnum.dataset, datasetId: id }))
);
}
/**
* delete all data by collectionIds
*/
export async function delCollectionRelevantData({
collectionIds,
fileIds
}: {
collectionIds: string[];
fileIds: string[];
}) {
collectionIds = collectionIds.filter(Boolean).map((item) => String(item));
const filterFileIds = fileIds.filter(Boolean).map((item) => String(item));
// delete training data
await MongoDatasetTraining.deleteMany({
collectionId: { $in: collectionIds }
});
await delay(2000);
// delete dataset.datas
await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } });
// delete pg data
await deleteDatasetDataVector({ collectionIds });
// delete collections
await MongoDatasetCollection.deleteMany({
_id: { $in: collectionIds }
});
// delete file and imgs
await Promise.all([
delImgByFileIdList(filterFileIds),
delFileByFileIdList({
bucketName: BucketNameEnum.dataset,
fileIdList: filterFileIds
})
]);
}
/**
* delete one data by mongoDataId
*/
export async function delDatasetDataByDataId({
collectionId,
mongoDataId
}: {
collectionId: string;
mongoDataId: string;
}) {
await deleteDatasetDataVector({ collectionId, dataIds: [mongoDataId] });
await MongoDatasetData.findByIdAndDelete(mongoDataId);
}