Files
FastGPT/packages/service/core/dataset/controller.ts
Archer b520988c64 V4.8.17 feature (#3485)
* feat: add third party account config (#3443)

* temp

* editor workflow variable style

* add team to dispatch

* i18n

* delete console

* change openai account position

* fix

* fix

* fix

* fix

* fix

* 4.8.17 test (#3461)

* perf: external provider config

* perf: ui

* feat: add template config (#3434)

* change template position

* template config

* delete console

* delete

* fix

* fix

* perf: Mongo visutal field (#3464)

* remve invalid code

* perf: team member visutal code

* perf: virtual search; perf: search test data

* fix: ts

* fix: image response headers

* perf: template code

* perf: auth layout;perf: auto save (#3472)

* perf: auth layout

* perf: auto save

* perf: auto save

* fix: template guide display & http input support external variables (#3475)

* fix: template guide display

* http editor support external workflow variables

* perf: auto save;fix: ifelse checker line break; (#3478)

* perf: auto save

* perf: auto save

* fix: ifelse checker line break

* perf: doc

* perf: doc

* fix: update var type error

* 4.8.17 test (#3479)

* perf: auto save

* perf: auto save

* perf: template code

* 4.8.17 test (#3480)

* perf: auto save

* perf: auto save

* perf: model price model

* feat: add react memo

* perf: model provider filter

* fix: ts (#3481)

* perf: auto save

* perf: auto save

* fix: ts

* simple app tool select (#3473)

* workflow plugin userguide & simple tool ui

* simple tool filter

* reuse component

* change component to hook

* fix

* perf: too selector modal (#3484)

* perf: auto save

* perf: auto save

* perf: markdown render

* perf: too selector

* fix: app version require tmbId

* perf: templates refresh

* perf: templates refresh

* hide auto save error tip

* perf: toolkit guide

---------

Co-authored-by: heheer <heheer@sealos.io>
2024-12-27 20:05:12 +08:00

110 lines
2.8 KiB
TypeScript

import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { MongoDatasetCollection } from './collection/schema';
import { MongoDataset } from './schema';
import { delCollectionRelatedSource } from './collection/controller';
import { ClientSession } from '../../common/mongo';
import { MongoDatasetTraining } from './training/schema';
import { MongoDatasetData } from './data/schema';
import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
/* ============= dataset ========== */
/* find all datasetId by top datasetId */
export async function findDatasetAndAllChildren({
teamId,
datasetId,
fields
}: {
teamId: string;
datasetId: string;
fields?: string;
}): Promise<DatasetSchemaType[]> {
const find = async (id: string) => {
const children = await MongoDataset.find(
{
teamId,
parentId: id
},
fields
).lean();
let datasets = children;
for (const child of children) {
const grandChildrenIds = await find(child._id);
datasets = datasets.concat(grandChildrenIds);
}
return datasets;
};
const [dataset, childDatasets] = await Promise.all([
MongoDataset.findById(datasetId).lean(),
find(datasetId)
]);
if (!dataset) {
return Promise.reject('Dataset not found');
}
return [dataset, ...childDatasets];
}
export async function getCollectionWithDataset(collectionId: string) {
const data = await MongoDatasetCollection.findById(collectionId)
.populate<{ dataset: DatasetSchemaType }>('dataset')
.lean();
if (!data) {
return Promise.reject('Collection is not exist');
}
return data;
}
/* delete all data by datasetIds */
export async function delDatasetRelevantData({
datasets,
session
}: {
datasets: DatasetSchemaType[];
session: ClientSession;
}) {
if (!datasets.length) return;
const teamId = datasets[0].teamId;
if (!teamId) {
return Promise.reject('TeamId is required');
}
const datasetIds = datasets.map((item) => item._id);
// delete training data
await MongoDatasetTraining.deleteMany({
teamId,
datasetId: { $in: datasetIds }
});
// Get _id, teamId, fileId, metadata.relatedImgId for all collections
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds }
},
'_id teamId datasetId fileId metadata',
{ session }
).lean();
// image and file
await delCollectionRelatedSource({ collections, session });
// delete collections
await MongoDatasetCollection.deleteMany({
teamId,
datasetId: { $in: datasetIds }
}).session(session);
// delete dataset.datas(Not need session)
await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } });
// no session delete: delete files, vector data
await deleteDatasetDataVector({ teamId, datasetIds });
}