V4.8.17 feature (#3485)

* feat: add third party account config (#3443) * temp * editor workflow variable style * add team to dispatch * i18n * delete console * change openai account position * fix * fix * fix * fix * fix * 4.8.17 test (#3461) * perf: external provider config * perf: ui * feat: add template config (#3434) * change template position * template config * delete console * delete * fix * fix * perf: Mongo visutal field (#3464) * remve invalid code * perf: team member visutal code * perf: virtual search; perf: search test data * fix: ts * fix: image response headers * perf: template code * perf: auth layout；perf: auto save (#3472) * perf: auth layout * perf: auto save * perf: auto save * fix: template guide display & http input support external variables (#3475) * fix: template guide display * http editor support external workflow variables * perf: auto save;fix: ifelse checker line break; (#3478) * perf: auto save * perf: auto save * fix: ifelse checker line break * perf: doc * perf: doc * fix: update var type error * 4.8.17 test (#3479) * perf: auto save * perf: auto save * perf: template code * 4.8.17 test (#3480) * perf: auto save * perf: auto save * perf: model price model * feat: add react memo * perf: model provider filter * fix: ts (#3481) * perf: auto save * perf: auto save * fix: ts * simple app tool select (#3473) * workflow plugin userguide & simple tool ui * simple tool filter * reuse component * change component to hook * fix * perf: too selector modal (#3484) * perf: auto save * perf: auto save * perf: markdown render * perf: too selector * fix: app version require tmbId * perf: templates refresh * perf: templates refresh * hide auto save error tip * perf: toolkit guide --------- Co-authored-by: heheer <heheer@sealos.io>
2025-10-14 15:11:13 +00:00 · 2024-12-27 20:05:12 +08:00
parent a209856d48
commit b520988c64
207 changed files with 2943 additions and 1378 deletions
--- a/packages/service/core/dataset/collection/controller.ts
+++ b/packages/service/core/dataset/collection/controller.ts
@@ -4,11 +4,7 @@ import {
 } from '@fastgpt/global/core/dataset/constants';
 import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
 import { MongoDatasetCollection } from './schema';
-import {
-  CollectionWithDatasetType,
-  DatasetCollectionSchemaType,
-  DatasetSchemaType
-} from '@fastgpt/global/core/dataset/type';
+import { DatasetCollectionSchemaType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetTraining } from '../training/schema';
 import { MongoDatasetData } from '../data/schema';
 import { delImgByRelatedId } from '../../../common/file/image/controller';
@@ -230,7 +226,7 @@ export const delCollectionRelatedSource = async ({
  collections,
  session
 }: {
-  collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  collections: DatasetCollectionSchemaType[];
  session: ClientSession;
 }) => {
  if (collections.length === 0) return;
@@ -264,7 +260,7 @@ export async function delCollection({
  session,
  delRelatedSource
 }: {
-  collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  collections: DatasetCollectionSchemaType[];
  session: ClientSession;
  delRelatedSource: boolean;
 }) {
@@ -274,16 +270,7 @@ export async function delCollection({

  if (!teamId) return Promise.reject('teamId is not exist');

-  const datasetIds = Array.from(
-    new Set(
-      collections.map((item) => {
-        if (typeof item.datasetId === 'string') {
-          return String(item.datasetId);
-        }
-        return String(item.datasetId._id);
-      })
-    )
-  );
+  const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
  const collectionIds = collections.map((item) => String(item._id));

  // delete training data
@@ -324,7 +311,7 @@ export async function delOnlyCollection({
  collections,
  session
 }: {
-  collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  collections: DatasetCollectionSchemaType[];
  session: ClientSession;
 }) {
  if (collections.length === 0) return;
@@ -333,16 +320,7 @@ export async function delOnlyCollection({

  if (!teamId) return Promise.reject('teamId is not exist');

-  const datasetIds = Array.from(
-    new Set(
-      collections.map((item) => {
-        if (typeof item.datasetId === 'string') {
-          return String(item.datasetId);
-        }
-        return String(item.datasetId._id);
-      })
-    )
-  );
+  const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
  const collectionIds = collections.map((item) => String(item._id));

  // delete training data
--- a/packages/service/core/dataset/collection/schema.ts
+++ b/packages/service/core/dataset/collection/schema.ts
@@ -100,6 +100,13 @@ const DatasetCollectionSchema = new Schema({
  }
 });

+DatasetCollectionSchema.virtual('dataset', {
+  ref: DatasetCollectionName,
+  localField: 'datasetId',
+  foreignField: '_id',
+  justOne: true
+});
+
 try {
  // auth file
  DatasetCollectionSchema.index({ teamId: 1, fileId: 1 });
--- a/packages/service/core/dataset/collection/utils.ts
+++ b/packages/service/core/dataset/collection/utils.ts
@@ -130,7 +130,7 @@ export const collectionTagsToTagLabel = async ({
 };

 export const syncCollection = async (collection: CollectionWithDatasetType) => {
-  const dataset = collection.datasetId;
+  const dataset = collection.dataset;

  if (
    collection.type !== DatasetCollectionTypeEnum.link &&
@@ -183,7 +183,7 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
        teamId: collection.teamId,
        tmbId: collection.tmbId,
        name: collection.name,
-        datasetId: collection.datasetId._id,
+        datasetId: collection.datasetId,
        parentId: collection.parentId,
        type: collection.type,

--- a/packages/service/core/dataset/controller.ts
+++ b/packages/service/core/dataset/controller.ts
@@ -1,4 +1,4 @@
-import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetCollection } from './collection/schema';
 import { MongoDataset } from './schema';
 import { delCollectionRelatedSource } from './collection/controller';
@@ -49,9 +49,9 @@ export async function findDatasetAndAllChildren({
 }

 export async function getCollectionWithDataset(collectionId: string) {
-  const data = (await MongoDatasetCollection.findById(collectionId)
-    .populate('datasetId')
-    .lean()) as CollectionWithDatasetType;
+  const data = await MongoDatasetCollection.findById(collectionId)
+    .populate<{ dataset: DatasetSchemaType }>('dataset')
+    .lean();
  if (!data) {
    return Promise.reject('Collection is not exist');
  }
--- a/packages/service/core/dataset/data/schema.ts
+++ b/packages/service/core/dataset/data/schema.ts
@@ -77,21 +77,32 @@ const DatasetDataSchema = new Schema({
  rebuilding: Boolean
 });

-// list collection and count data; list data; delete collection(relate data)
-DatasetDataSchema.index({
-  teamId: 1,
-  datasetId: 1,
-  collectionId: 1,
-  chunkIndex: 1,
-  updateTime: -1
+DatasetDataSchema.virtual('collection', {
+  ref: DatasetColCollectionName,
+  localField: 'collectionId',
+  foreignField: '_id',
+  justOne: true
 });
-// full text index
-DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
-// Recall vectors after data matching
-DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
-DatasetDataSchema.index({ updateTime: 1 });
-// rebuild data
-DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
+
+try {
+  // list collection and count data; list data; delete collection(relate data)
+  DatasetDataSchema.index({
+    teamId: 1,
+    datasetId: 1,
+    collectionId: 1,
+    chunkIndex: 1,
+    updateTime: -1
+  });
+  // full text index
+  DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
+  // Recall vectors after data matching
+  DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
+  DatasetDataSchema.index({ updateTime: 1 });
+  // rebuild data
+  DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
+} catch (error) {
+  console.log(error);
+}

 export const MongoDatasetData = getMongoModel<DatasetDataSchemaType>(
  DatasetDataCollectionName,
--- a/packages/service/core/dataset/search/controller.ts
+++ b/packages/service/core/dataset/search/controller.ts
@@ -8,8 +8,8 @@ import { getVectorsByText } from '../../ai/embedding';
 import { getVectorModel } from '../../ai/model';
 import { MongoDatasetData } from '../data/schema';
 import {
+  DatasetCollectionSchemaType,
  DatasetDataSchemaType,
-  DatasetDataWithCollectionType,
  SearchDataResponseItemType
 } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetCollection } from '../collection/schema';
@@ -267,7 +267,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
    });

    // get q and a
-    const dataList = (await MongoDatasetData.find(
+    const dataList = await MongoDatasetData.find(
      {
        teamId,
        datasetId: { $in: datasetIds },
@@ -276,8 +276,11 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
      },
      'datasetId collectionId updateTime q a chunkIndex indexes'
    )
-      .populate('collectionId', 'name fileId rawLink externalFileId externalFileUrl')
-      .lean()) as DatasetDataWithCollectionType[];
+      .populate<{ collection: DatasetCollectionSchemaType }>(
+        'collection',
+        'name fileId rawLink externalFileId externalFileUrl'
+      )
+      .lean();

    // add score to data(It's already sorted. The first one is the one with the most points)
    const concatResults = dataList.map((data) => {
@@ -307,8 +310,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
        a: data.a,
        chunkIndex: data.chunkIndex,
        datasetId: String(data.datasetId),
-        collectionId: String(data.collectionId?._id),
-        ...getCollectionSourceData(data.collectionId),
+        collectionId: String(data.collectionId),
+        ...getCollectionSourceData(data.collection),
        score: [{ type: SearchScoreTypeEnum.embedding, value: data.score, index }]
      };

--- a/packages/service/core/dataset/training/controller.ts
+++ b/packages/service/core/dataset/training/controller.ts
@@ -34,7 +34,7 @@ export const pushDataListToTrainingQueueByCollectionId = async ({
  session?: ClientSession;
 } & PushDatasetDataProps) => {
  const {
-    datasetId: { _id: datasetId, agentModel, vectorModel }
+    dataset: { _id: datasetId, agentModel, vectorModel }
  } = await getCollectionWithDataset(collectionId);
  return pushDataListToTrainingQueue({
    ...props,