4.8.6 fix (#1963)

* feat: log store

* fix: full text search match query

* perf: mongo schema import, Avoid duplicate import

packages/service/core/app/schema.ts

@@ -1,6 +1,5 @@
 import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
-import { connectionMongo, type Model } from '../../common/mongo';
-const { Schema, model, models } = connectionMongo;
+import { Schema, getMongoModel } from '../../common/mongo';
 import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import {
   TeamCollectionName,
@@ -21,6 +20,7 @@ export const chatConfigType = {
   chatInputGuide: Object
 };
 
+// schema
 const AppSchema = new Schema({
   parentId: {
     type: Schema.Types.ObjectId,
@@ -112,15 +112,8 @@ const AppSchema = new Schema({
   ...getPermissionSchema(AppDefaultPermissionVal)
 });
 
-try {
-  AppSchema.index({ updateTime: -1 });
-  AppSchema.index({ teamId: 1, type: 1 });
-  AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
-} catch (error) {
-  console.log(error);
-}
+AppSchema.index({ updateTime: -1 });
+AppSchema.index({ teamId: 1, type: 1 });
+AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
 
-export const MongoApp: Model<AppType> =
-  models[AppCollectionName] || model(AppCollectionName, AppSchema);
-
-MongoApp.syncIndexes();
+export const MongoApp = getMongoModel<AppType>(AppCollectionName, AppSchema);

packages/service/core/app/version/schema.ts

@@ -1,4 +1,4 @@
-import { connectionMongo, type Model } from '../../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
 import { chatConfigType } from '../schema';
@@ -34,7 +34,7 @@ try {
   console.log(error);
 }
 
-export const MongoAppVersion: Model<AppVersionSchemaType> =
-  models[AppVersionCollectionName] || model(AppVersionCollectionName, AppVersionSchema);
-
-MongoAppVersion.syncIndexes();
+export const MongoAppVersion = getMongoModel<AppVersionSchemaType>(
+  AppVersionCollectionName,
+  AppVersionSchema
+);

packages/service/core/chat/chatItemSchema.ts

@@ -1,4 +1,4 @@
-import { connectionMongo, type Model } from '../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type';
 import { ChatRoleMap } from '@fastgpt/global/core/chat/constants';
@@ -9,7 +9,6 @@ import {
 } from '@fastgpt/global/support/user/team/constant';
 import { AppCollectionName } from '../app/schema';
 import { userCollectionName } from '../../support/user/schema';
-import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 
 export const ChatItemCollectionName = 'chatitems';
@@ -99,7 +98,4 @@ try {
   console.log(error);
 }
 
-export const MongoChatItem: Model<ChatItemType> =
-  models[ChatItemCollectionName] || model(ChatItemCollectionName, ChatItemSchema);
-
-MongoChatItem.syncIndexes();
+export const MongoChatItem = getMongoModel<ChatItemType>(ChatItemCollectionName, ChatItemSchema);

packages/service/core/chat/chatSchema.ts

@@ -1,4 +1,4 @@
-import { connectionMongo, type Model } from '../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
 import { ChatSourceMap } from '@fastgpt/global/core/chat/constants';
@@ -98,6 +98,4 @@ try {
   console.log(error);
 }
 
-export const MongoChat: Model<ChatType> =
-  models[chatCollectionName] || model(chatCollectionName, ChatSchema);
-MongoChat.syncIndexes();
+export const MongoChat = getMongoModel<ChatType>(chatCollectionName, ChatSchema);

packages/service/core/chat/inputGuide/schema.ts

@@ -1,5 +1,5 @@
 import { AppCollectionName } from '../../app/schema';
-import { connectionMongo, type Model } from '../../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d';
 
@@ -23,7 +23,7 @@ try {
   console.log(error);
 }
 
-export const MongoChatInputGuide: Model<ChatInputGuideSchemaType> =
-  models[ChatInputGuideCollectionName] || model(ChatInputGuideCollectionName, ChatInputGuideSchema);
-
-MongoChatInputGuide.syncIndexes();
+export const MongoChatInputGuide = getMongoModel<ChatInputGuideSchemaType>(
+  ChatInputGuideCollectionName,
+  ChatInputGuideSchema
+);

packages/service/core/dataset/collection/schema.ts

@@ -1,4 +1,4 @@
-import { connectionMongo, type Model } from '../../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants';
@@ -94,9 +94,6 @@ const DatasetCollectionSchema = new Schema({
   }
 });
 
-export const MongoDatasetCollection: Model<DatasetCollectionSchemaType> =
-  models[DatasetColCollectionName] || model(DatasetColCollectionName, DatasetCollectionSchema);
-
 try {
   // auth file
   DatasetCollectionSchema.index({ teamId: 1, fileId: 1 });
@@ -111,8 +108,11 @@ try {
 
   // get forbid
   // DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 });
-
-  MongoDatasetCollection.syncIndexes({ background: true });
 } catch (error) {
   console.log(error);
 }
+
+export const MongoDatasetCollection = getMongoModel<DatasetCollectionSchemaType>(
+  DatasetColCollectionName,
+  DatasetCollectionSchema
+);

packages/service/core/dataset/data/schema.ts

@@ -1,4 +1,4 @@
-import { connectionMongo, type Model } from '../../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import {
@@ -77,27 +77,23 @@ const DatasetDataSchema = new Schema({
   rebuilding: Boolean
 });
 
-export const MongoDatasetData: Model<DatasetDataSchemaType> =
-  models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
+// list collection and count data; list data; delete collection(relate data)
+DatasetDataSchema.index({
+  teamId: 1,
+  datasetId: 1,
+  collectionId: 1,
+  chunkIndex: 1,
+  updateTime: -1
+});
+// full text index
+DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
+// Recall vectors after data matching
+DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
+DatasetDataSchema.index({ updateTime: 1 });
+// rebuild data
+DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
 
-try {
-  // list collection and count data; list data; delete collection(relate data)
-  DatasetDataSchema.index({
-    teamId: 1,
-    datasetId: 1,
-    collectionId: 1,
-    chunkIndex: 1,
-    updateTime: -1
-  });
-  // full text index
-  DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
-  // Recall vectors after data matching
-  DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
-  DatasetDataSchema.index({ updateTime: 1 });
-  // rebuild data
-  DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
-
-  MongoDatasetData.syncIndexes({ background: true });
-} catch (error) {
-  console.log(error);
-}
+export const MongoDatasetData = getMongoModel<DatasetDataSchemaType>(
+  DatasetDataCollectionName,
+  DatasetDataSchema
+);

packages/service/core/dataset/schema.ts

@@ -1,4 +1,4 @@
-import { connectionMongo, type Model } from '../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import {
@@ -11,7 +11,6 @@ import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
-import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
 import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
 
 export const DatasetCollectionName = 'datasets';
@@ -99,6 +98,4 @@ try {
   console.log(error);
 }
 
-export const MongoDataset: Model<DatasetSchemaType> =
-  models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);
-MongoDataset.syncIndexes();
+export const MongoDataset = getMongoModel<DatasetSchemaType>(DatasetCollectionName, DatasetSchema);

packages/service/core/dataset/search/controller.ts

@@ -212,7 +212,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
                   {
                     $match: {
                       $expr: { $eq: ['$_id', '$$collectionId'] },
-                      forbid: { $eq: false } // 直接在lookup阶段过滤
+                      forbid: { $eq: true } // 匹配被禁用的数据
                     }
                   },
                   {
@@ -226,7 +226,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
             },
             {
               $match: {
-                collection: { $ne: [] }
+                collection: { $eq: [] } // 没有 forbid=true 的数据
               }
             },
             {

packages/service/core/dataset/training/schema.ts

@@ -1,5 +1,5 @@
 /* 模型的知识库 */
-import { connectionMongo, type Model } from '../../../common/mongo';
+import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
 import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
@@ -103,7 +103,7 @@ try {
   console.log(error);
 }
 
-export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> =
-  models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema);
-
-MongoDatasetTraining.syncIndexes();
+export const MongoDatasetTraining = getMongoModel<DatasetTrainingSchemaType>(
+  DatasetTrainingCollectionName,
+  TrainingDataSchema
+);

packages/service/core/workflow/dispatch/agent/extract.ts

@@ -198,7 +198,7 @@ ${description ? `- ${description}` : ''}
       required: []
     }
   };
-  console.log(properties);
+
   return {
     filterMessages,
     agentFunction