4.6.8-alpha (#804)

* perf: redirect request and err log replace

perf: dataset openapi

feat: session

fix: retry input error

feat: 468 doc

sub page

feat: standard sub

perf: rerank tip

perf: rerank tip

perf: api sdk

perf: openapi

sub plan

perf: sub ui

fix: ts

* perf: init log

* fix: variable select

* sub page

* icon

* perf: llm model config

* perf: menu ux

* perf: system store

* perf: publish app name

* fix: init data

* perf: flow edit ux

* fix: value type format and ux

* fix prompt editor default value (#13)

* fix prompt editor default value

* fix prompt editor update when not focus

* add key with variable

---------

Co-authored-by: Archer <545436317@qq.com>

* fix: value type

* doc

* i18n

* import path

* home page

* perf: mongo session running

* fix: ts

* perf: use toast

* perf: flow edit

* perf: sse response

* slider ui

* fetch error

* fix prompt editor rerender when not focus by key defaultvalue (#14)

* perf: prompt editor

* feat: dataset search concat

* perf: doc

* fix:ts

* perf: doc

* fix json editor onblur value (#15)

* faq

* vector model default config

* ipv6

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/service/core/dataset/collection/controller.ts

@@ -15,6 +15,7 @@ import { delImgByRelatedId } from '../../../common/file/image/controller';
 import { deleteDatasetDataVector } from '../../../common/vectorStore/controller';
 import { delFileByFileIdList } from '../../../common/file/gridfs/controller';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { ClientSession } from '../../../common/mongo';
 
 export async function createOneCollection({
   teamId,
@@ -35,41 +36,53 @@ export async function createOneCollection({
   hashRawText,
   rawTextLength,
   metadata = {},
+  session,
   ...props
-}: CreateDatasetCollectionParams & { teamId: string; tmbId: string; [key: string]: any }) {
-  const { _id } = await MongoDatasetCollection.create({
-    ...props,
-    teamId,
-    tmbId,
-    parentId: parentId || null,
-    datasetId,
-    name,
-    type,
+}: CreateDatasetCollectionParams & {
+  teamId: string;
+  tmbId: string;
+  [key: string]: any;
+  session?: ClientSession;
+}) {
+  const [collection] = await MongoDatasetCollection.create(
+    [
+      {
+        ...props,
+        teamId,
+        tmbId,
+        parentId: parentId || null,
+        datasetId,
+        name,
+        type,
 
-    trainingType,
-    chunkSize,
-    chunkSplitter,
-    qaPrompt,
+        trainingType,
+        chunkSize,
+        chunkSplitter,
+        qaPrompt,
 
-    fileId,
-    rawLink,
+        fileId,
+        rawLink,
 
-    rawTextLength,
-    hashRawText,
-    metadata
-  });
+        rawTextLength,
+        hashRawText,
+        metadata
+      }
+    ],
+    { session }
+  );
 
   // create default collection
   if (type === DatasetCollectionTypeEnum.folder) {
     await createDefaultCollection({
       datasetId,
-      parentId: _id,
+      parentId: collection._id,
       teamId,
-      tmbId
+      tmbId,
+      session
     });
   }
 
-  return _id;
+  return collection;
 }
 
 // create default collection
@@ -78,34 +91,43 @@ export function createDefaultCollection({
   datasetId,
   parentId,
   teamId,
-  tmbId
+  tmbId,
+  session
 }: {
   name?: '手动录入' | '手动标注';
   datasetId: string;
   parentId?: string;
   teamId: string;
   tmbId: string;
+  session?: ClientSession;
 }) {
-  return MongoDatasetCollection.create({
-    name,
-    teamId,
-    tmbId,
-    datasetId,
-    parentId,
-    type: DatasetCollectionTypeEnum.virtual,
-    trainingType: TrainingModeEnum.chunk,
-    chunkSize: 0,
-    updateTime: new Date('2099')
-  });
+  return MongoDatasetCollection.create(
+    [
+      {
+        name,
+        teamId,
+        tmbId,
+        datasetId,
+        parentId,
+        type: DatasetCollectionTypeEnum.virtual,
+        trainingType: TrainingModeEnum.chunk,
+        chunkSize: 0,
+        updateTime: new Date('2099')
+      }
+    ],
+    { session }
+  );
 }
 
 /**
  * delete collection and it related data
  */
 export async function delCollectionAndRelatedSources({
-  collections
+  collections,
+  session
 }: {
   collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  session: ClientSession;
 }) {
   if (collections.length === 0) return;
 
@@ -128,24 +150,25 @@ export async function delCollectionAndRelatedSources({
   await delay(2000);
 
   // delete dataset.datas
-  await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } });
-  // delete pg data
-  await deleteDatasetDataVector({ teamId, collectionIds });
-
-  // delete file and imgs
-  await Promise.all([
-    delImgByRelatedId({
-      teamId,
-      relateIds: relatedImageIds
-    }),
-    delFileByFileIdList({
-      bucketName: BucketNameEnum.dataset,
-      fileIdList
-    })
-  ]);
-
+  await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }, { session });
+  // delete imgs
+  await delImgByRelatedId({
+    teamId,
+    relateIds: relatedImageIds,
+    session
+  });
   // delete collections
-  await MongoDatasetCollection.deleteMany({
-    _id: { $in: collectionIds }
+  await MongoDatasetCollection.deleteMany(
+    {
+      _id: { $in: collectionIds }
+    },
+    { session }
+  );
+
+  // no session delete: delete files, vector data
+  await deleteDatasetDataVector({ teamId, collectionIds });
+  await delFileByFileIdList({
+    bucketName: BucketNameEnum.dataset,
+    fileIdList
   });
 }

packages/service/core/dataset/collection/utils.ts

@@ -9,6 +9,7 @@ import {
   TrainingModeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import { hashStr } from '@fastgpt/global/common/string/tools';
+import { ClientSession } from '../../../common/mongo';
 
 /**
  * get all collection by top collectionId
@@ -149,17 +150,17 @@ export const getCollectionAndRawText = async ({
 
 /* link collection start load data */
 export const reloadCollectionChunks = async ({
-  collectionId,
   collection,
   tmbId,
   billId,
-  rawText
+  rawText,
+  session
 }: {
-  collectionId?: string;
-  collection?: CollectionWithDatasetType;
+  collection: CollectionWithDatasetType;
   tmbId: string;
   billId?: string;
   rawText?: string;
+  session: ClientSession;
 }) => {
   const {
     title,
@@ -168,7 +169,6 @@ export const reloadCollectionChunks = async ({
     isSameRawText
   } = await getCollectionAndRawText({
     collection,
-    collectionId,
     newRawText: rawText
   });
 
@@ -186,6 +186,7 @@ export const reloadCollectionChunks = async ({
     if (col.trainingType === TrainingModeEnum.qa) return col.datasetId.agentModel;
     return Promise.reject('Training model error');
   })();
+
   await MongoDatasetTraining.insertMany(
     chunks.map((item, i) => ({
       teamId: col.teamId,
@@ -199,13 +200,18 @@ export const reloadCollectionChunks = async ({
       q: item,
       a: '',
       chunkIndex: i
-    }))
+    })),
+    { session }
   );
 
   // update raw text
-  await MongoDatasetCollection.findByIdAndUpdate(col._id, {
-    ...(title && { name: title }),
-    rawTextLength: newRawText.length,
-    hashRawText: hashStr(newRawText)
-  });
+  await MongoDatasetCollection.findByIdAndUpdate(
+    col._id,
+    {
+      ...(title && { name: title }),
+      rawTextLength: newRawText.length,
+      hashRawText: hashStr(newRawText)
+    },
+    { session }
+  );
 };