mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 05:12:39 +00:00

* feat: add third party account config (#3443) * temp * editor workflow variable style * add team to dispatch * i18n * delete console * change openai account position * fix * fix * fix * fix * fix * 4.8.17 test (#3461) * perf: external provider config * perf: ui * feat: add template config (#3434) * change template position * template config * delete console * delete * fix * fix * perf: Mongo visutal field (#3464) * remve invalid code * perf: team member visutal code * perf: virtual search; perf: search test data * fix: ts * fix: image response headers * perf: template code * perf: auth layout;perf: auto save (#3472) * perf: auth layout * perf: auto save * perf: auto save * fix: template guide display & http input support external variables (#3475) * fix: template guide display * http editor support external workflow variables * perf: auto save;fix: ifelse checker line break; (#3478) * perf: auto save * perf: auto save * fix: ifelse checker line break * perf: doc * perf: doc * fix: update var type error * 4.8.17 test (#3479) * perf: auto save * perf: auto save * perf: template code * 4.8.17 test (#3480) * perf: auto save * perf: auto save * perf: model price model * feat: add react memo * perf: model provider filter * fix: ts (#3481) * perf: auto save * perf: auto save * fix: ts * simple app tool select (#3473) * workflow plugin userguide & simple tool ui * simple tool filter * reuse component * change component to hook * fix * perf: too selector modal (#3484) * perf: auto save * perf: auto save * perf: markdown render * perf: too selector * fix: app version require tmbId * perf: templates refresh * perf: templates refresh * hide auto save error tip * perf: toolkit guide --------- Co-authored-by: heheer <heheer@sealos.io>
110 lines
2.8 KiB
TypeScript
110 lines
2.8 KiB
TypeScript
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
|
|
import { MongoDatasetCollection } from './collection/schema';
|
|
import { MongoDataset } from './schema';
|
|
import { delCollectionRelatedSource } from './collection/controller';
|
|
import { ClientSession } from '../../common/mongo';
|
|
import { MongoDatasetTraining } from './training/schema';
|
|
import { MongoDatasetData } from './data/schema';
|
|
import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
|
|
|
|
/* ============= dataset ========== */
|
|
/* find all datasetId by top datasetId */
|
|
export async function findDatasetAndAllChildren({
|
|
teamId,
|
|
datasetId,
|
|
fields
|
|
}: {
|
|
teamId: string;
|
|
datasetId: string;
|
|
fields?: string;
|
|
}): Promise<DatasetSchemaType[]> {
|
|
const find = async (id: string) => {
|
|
const children = await MongoDataset.find(
|
|
{
|
|
teamId,
|
|
parentId: id
|
|
},
|
|
fields
|
|
).lean();
|
|
|
|
let datasets = children;
|
|
|
|
for (const child of children) {
|
|
const grandChildrenIds = await find(child._id);
|
|
datasets = datasets.concat(grandChildrenIds);
|
|
}
|
|
|
|
return datasets;
|
|
};
|
|
const [dataset, childDatasets] = await Promise.all([
|
|
MongoDataset.findById(datasetId).lean(),
|
|
find(datasetId)
|
|
]);
|
|
|
|
if (!dataset) {
|
|
return Promise.reject('Dataset not found');
|
|
}
|
|
|
|
return [dataset, ...childDatasets];
|
|
}
|
|
|
|
export async function getCollectionWithDataset(collectionId: string) {
|
|
const data = await MongoDatasetCollection.findById(collectionId)
|
|
.populate<{ dataset: DatasetSchemaType }>('dataset')
|
|
.lean();
|
|
if (!data) {
|
|
return Promise.reject('Collection is not exist');
|
|
}
|
|
return data;
|
|
}
|
|
|
|
/* delete all data by datasetIds */
|
|
export async function delDatasetRelevantData({
|
|
datasets,
|
|
session
|
|
}: {
|
|
datasets: DatasetSchemaType[];
|
|
session: ClientSession;
|
|
}) {
|
|
if (!datasets.length) return;
|
|
|
|
const teamId = datasets[0].teamId;
|
|
|
|
if (!teamId) {
|
|
return Promise.reject('TeamId is required');
|
|
}
|
|
|
|
const datasetIds = datasets.map((item) => item._id);
|
|
|
|
// delete training data
|
|
await MongoDatasetTraining.deleteMany({
|
|
teamId,
|
|
datasetId: { $in: datasetIds }
|
|
});
|
|
|
|
// Get _id, teamId, fileId, metadata.relatedImgId for all collections
|
|
const collections = await MongoDatasetCollection.find(
|
|
{
|
|
teamId,
|
|
datasetId: { $in: datasetIds }
|
|
},
|
|
'_id teamId datasetId fileId metadata',
|
|
{ session }
|
|
).lean();
|
|
|
|
// image and file
|
|
await delCollectionRelatedSource({ collections, session });
|
|
|
|
// delete collections
|
|
await MongoDatasetCollection.deleteMany({
|
|
teamId,
|
|
datasetId: { $in: datasetIds }
|
|
}).session(session);
|
|
|
|
// delete dataset.datas(Not need session)
|
|
await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } });
|
|
|
|
// no session delete: delete files, vector data
|
|
await deleteDatasetDataVector({ teamId, datasetIds });
|
|
}
|