v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/service/common/mongo/init.ts

@@ -34,6 +34,12 @@ export async function connectMongo({
       retryReads: true
     });
 
+    mongoose.connection.on('error', (error) => {
+      console.log('mongo error', error);
+      global.mongodb?.disconnect();
+      global.mongodb = undefined;
+    });
+
     console.log('mongo connected');
 
     afterHook && (await afterHook());

packages/service/common/system/log.ts

@@ -3,9 +3,12 @@ import dayjs from 'dayjs';
 /* add logger */
 export const addLog = {
   log(level: 'info' | 'warn' | 'error', msg: string, obj: Record<string, any> = {}) {
+    const stringifyObj = JSON.stringify(obj);
+    const isEmpty = Object.keys(obj).length === 0;
+
     console.log(
       `[${level.toLocaleUpperCase()}] ${dayjs().format('YYYY-MM-DD HH:mm:ss')} ${msg} ${
-        level !== 'error' ? JSON.stringify(obj) : ''
+        level !== 'error' && !isEmpty ? stringifyObj : ''
       }`
     );
 

packages/service/common/system/tools.ts

@@ -11,6 +11,7 @@ export const initFastGPTConfig = (config?: FastGPTConfigFileType) => {
   if (!config) return;
 
   global.feConfigs = config.feConfigs;
+  global.systemEnv = config.systemEnv;
   global.subPlans = config.subPlans;
 
   global.llmModels = config.llmModels;

packages/service/common/vectorStore/controller.ts

@@ -22,7 +22,7 @@ export const insertDatasetDataVector = async ({
   query: string;
   model: VectorModelItemType;
 }) => {
-  const { vectors, charsLength } = await getVectorsByText({
+  const { vectors, tokens } = await getVectorsByText({
     model,
     input: query
   });
@@ -32,27 +32,7 @@ export const insertDatasetDataVector = async ({
   });
 
   return {
-    charsLength,
+    tokens,
     insertId
   };
 };
-
-// export const updateDatasetDataVector = async ({
-//   id,
-//   ...props
-// }: InsertVectorProps & {
-//   id: string;
-//   query: string;
-//   model: VectorModelItemType;
-// }) => {
-//   // insert new vector
-//   const { charsLength, insertId } = await insertDatasetDataVector(props);
-
-//   // delete old vector
-//   await deleteDatasetDataVector({
-//     teamId: props.teamId,
-//     id
-//   });
-
-//   return { charsLength, insertId };
-// };

packages/service/core/ai/embedding/index.ts

@@ -1,6 +1,6 @@
 import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../config';
-import { replaceValidChars } from '../../chat/utils';
+import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 
 type GetVectorProps = {
   model: VectorModelItemType;
@@ -37,7 +37,7 @@ export async function getVectorsByText({ model, input }: GetVectorProps) {
         }
 
         return {
-          charsLength: replaceValidChars(input).length,
+          tokens: countPromptTokens(input),
           vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
         };
       });

packages/service/core/ai/functions/createQuestionGuide.ts

@@ -1,6 +1,6 @@
 import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
 import { getAIApi } from '../config';
-import { countGptMessagesChars } from '../../chat/utils';
+import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
 
 export const Prompt_QuestionGuide = `我不太清楚问你什么问题，请帮我生成 3 个问题，引导我继续提问。问题的长度应小于20个字符，按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
 
@@ -34,12 +34,12 @@ export async function createQuestionGuide({
   const start = answer.indexOf('[');
   const end = answer.lastIndexOf(']');
 
-  const charsLength = countGptMessagesChars(concatMessages);
+  const tokens = countGptMessagesTokens(concatMessages);
 
   if (start === -1 || end === -1) {
     return {
       result: [],
-      charsLength: 0
+      tokens: 0
     };
   }
 
@@ -51,12 +51,12 @@ export async function createQuestionGuide({
   try {
     return {
       result: JSON.parse(jsonStr),
-      charsLength
+      tokens
     };
   } catch (error) {
     return {
       result: [],
-      charsLength: 0
+      tokens: 0
     };
   }
 }

packages/service/core/ai/functions/queryExtension.ts

@@ -1,7 +1,7 @@
 import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { getAIApi } from '../config';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
-import { countGptMessagesChars } from '../../chat/utils';
+import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
 
 /* 
     query extension - 问题扩展
@@ -106,7 +106,7 @@ export const queryExtension = async ({
   rawQuery: string;
   extensionQueries: string[];
   model: string;
-  charsLength: number;
+  tokens: number;
 }> => {
   const systemFewShot = chatBg
     ? `Q: 对话背景。
@@ -148,7 +148,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: [],
       model,
-      charsLength: 0
+      tokens: 0
     };
   }
 
@@ -161,7 +161,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: queries,
       model,
-      charsLength: countGptMessagesChars(messages)
+      tokens: countGptMessagesTokens(messages)
     };
   } catch (error) {
     console.log(error);
@@ -169,7 +169,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: [],
       model,
-      charsLength: 0
+      tokens: 0
     };
   }
 };

packages/service/core/chat/utils.ts

@@ -1,11 +1,7 @@
 import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
 import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
-import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
-import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
-import type {
-  ChatCompletionContentPart,
-  ChatMessageItemType
-} from '@fastgpt/global/core/ai/type.d';
+import { countMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
+import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
 import axios from 'axios';
 
 /* slice chat context by tokens */
@@ -32,26 +28,34 @@ export function ChatContextFilter({
   const chatPrompts: ChatItemType[] = messages.slice(chatStartIndex);
 
   // reduce token of systemPrompt
-  maxTokens -= countMessagesTokens({
-    messages: systemPrompts
-  });
+  maxTokens -= countMessagesTokens(systemPrompts);
 
-  // 根据 tokens 截断内容
-  const chats: ChatItemType[] = [];
+  // Save the last chat prompt(question)
+  const question = chatPrompts.pop();
+  if (!question) {
+    return systemPrompts;
+  }
+  const chats: ChatItemType[] = [question];
 
-  // 从后往前截取对话内容
-  for (let i = chatPrompts.length - 1; i >= 0; i--) {
-    const item = chatPrompts[i];
-    chats.unshift(item);
+  // 从后往前截取对话内容, 每次需要截取2个
+  while (1) {
+    const assistant = chatPrompts.pop();
+    const user = chatPrompts.pop();
+    if (!assistant || !user) {
+      break;
+    }
 
-    const tokens = countPromptTokens(item.value, adaptRole_Chat2Message(item.obj));
+    const tokens = countMessagesTokens([assistant, user]);
     maxTokens -= tokens;
+    /* 整体 tokens 超出范围，截断  */
+    if (maxTokens < 0) {
+      break;
+    }
 
-    /* 整体 tokens 超出范围, system必须保留 */
-    if (maxTokens <= 0) {
-      if (chats.length > 1) {
-        chats.shift();
-      }
+    chats.unshift(assistant);
+    chats.unshift(user);
+
+    if (chatPrompts.length === 0) {
       break;
     }
   }
@@ -59,16 +63,6 @@ export function ChatContextFilter({
   return [...systemPrompts, ...chats];
 }
 
-export const replaceValidChars = (str: string) => {
-  const reg = /[\s\r\n]+/g;
-  return str.replace(reg, '');
-};
-export const countMessagesChars = (messages: ChatItemType[]) => {
-  return messages.reduce((sum, item) => sum + replaceValidChars(item.value).length, 0);
-};
-export const countGptMessagesChars = (messages: ChatMessageItemType[]) =>
-  messages.reduce((sum, item) => sum + replaceValidChars(item.content).length, 0);
-
 /**
     string to vision model. Follow the markdown code block rule for interception:
 

packages/service/core/dataset/schema.ts

@@ -4,6 +4,7 @@ import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import {
   DatasetStatusEnum,
   DatasetStatusMap,
+  DatasetTypeEnum,
   DatasetTypeMap
 } from '@fastgpt/global/core/dataset/constants';
 import {
@@ -39,7 +40,7 @@ const DatasetSchema = new Schema({
     type: String,
     enum: Object.keys(DatasetTypeMap),
     required: true,
-    default: 'dataset'
+    default: DatasetTypeEnum.dataset
   },
   status: {
     type: String,

packages/service/core/dataset/training/controller.ts

@@ -46,12 +46,16 @@ export async function pushDataListToTrainingQueue({
   } = await getCollectionWithDataset(collectionId);
 
   const checkModelValid = async () => {
-    if (trainingMode === TrainingModeEnum.chunk) {
-      const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
-      if (!vectorModelData) {
-        return Promise.reject(`File model ${vectorModel} is inValid`);
-      }
+    const agentModelData = datasetModelList?.find((item) => item.model === agentModel);
+    if (!agentModelData) {
+      return Promise.reject(`Vector model ${agentModel} is inValid`);
+    }
+    const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
+    if (!vectorModelData) {
+      return Promise.reject(`File model ${vectorModel} is inValid`);
+    }
 
+    if (trainingMode === TrainingModeEnum.chunk) {
       return {
         maxToken: vectorModelData.maxToken * 1.3,
         model: vectorModelData.model,
@@ -59,17 +63,14 @@ export async function pushDataListToTrainingQueue({
       };
     }
 
-    if (trainingMode === TrainingModeEnum.qa) {
-      const qaModelData = datasetModelList?.find((item) => item.model === agentModel);
-      if (!qaModelData) {
-        return Promise.reject(`Vector model ${agentModel} is inValid`);
-      }
+    if (trainingMode === TrainingModeEnum.qa || trainingMode === TrainingModeEnum.auto) {
       return {
-        maxToken: qaModelData.maxContext * 0.8,
-        model: qaModelData.model,
+        maxToken: agentModelData.maxContext * 0.8,
+        model: agentModelData.model,
         weight: 0
       };
     }
+
     return Promise.reject(`Training mode "${trainingMode}" is inValid`);
   };
 

packages/service/core/dataset/training/utils.ts

@@ -0,0 +1,41 @@
+import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
+import { addLog } from '../../../common/system/log';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { MongoDatasetTraining } from './schema';
+
+export const checkInvalidChunkAndLock = async ({
+  err,
+  errText,
+  data
+}: {
+  err: any;
+  errText: string;
+  data: DatasetTrainingSchemaType;
+}) => {
+  if (err?.response) {
+    addLog.error(`openai error: ${errText}`, {
+      status: err.response?.status,
+      statusText: err.response?.statusText,
+      data: err.response?.data
+    });
+  } else {
+    addLog.error(getErrText(err, errText), err);
+  }
+
+  if (
+    err?.message === 'invalid message format' ||
+    err?.type === 'invalid_request_error' ||
+    err?.code === 500
+  ) {
+    addLog.info('Lock training data');
+    console.log(err);
+
+    try {
+      await MongoDatasetTraining.findByIdAndUpdate(data._id, {
+        lockTime: new Date('2998/5/5')
+      });
+    } catch (error) {}
+    return true;
+  }
+  return false;
+};

packages/service/support/outLink/schema.ts

@@ -6,6 +6,7 @@ import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
+import { appCollectionName } from '../../core/app/schema';
 
 const OutLinkSchema = new Schema({
   shareId: {
@@ -24,7 +25,7 @@ const OutLinkSchema = new Schema({
   },
   appId: {
     type: Schema.Types.ObjectId,
-    ref: 'model',
+    ref: appCollectionName,
     required: true
   },
   type: {

packages/service/support/permission/auth/common.ts

@@ -1,7 +1,5 @@
-import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { parseHeaderCert } from '../controller';
 import { AuthModeType } from '../type';
-import { authOutLinkValid } from './outLink';
 import { SERVICE_LOCAL_HOST } from '../../../common/system/tools';
 
 export const authCert = async (props: AuthModeType) => {
@@ -13,25 +11,6 @@ export const authCert = async (props: AuthModeType) => {
     canWrite: true
   };
 };
-export async function authCertOrShareId({
-  shareId,
-  ...props
-}: AuthModeType & { shareId?: string }) {
-  if (!shareId) {
-    return authCert(props);
-  }
-
-  const { shareChat } = await authOutLinkValid({ shareId });
-
-  return {
-    teamId: String(shareChat.teamId),
-    tmbId: String(shareChat.tmbId),
-    authType: AuthUserTypeEnum.outLink,
-    apikey: '',
-    isOwner: false,
-    canWrite: false
-  };
-}
 
 /* auth the request from local service */
 export const authRequestFromLocal = ({ req }: AuthModeType) => {

packages/service/support/user/team/controller.ts

@@ -23,11 +23,11 @@ async function getTeamMember(match: Record<string, any>): Promise<TeamItemType>
     avatar: tmb.teamId.avatar,
     balance: tmb.teamId.balance,
     tmbId: String(tmb._id),
+    teamDomain: tmb.teamId?.teamDomain,
     role: tmb.role,
     status: tmb.status,
     defaultTeam: tmb.defaultTeam,
-    canWrite: tmb.role !== TeamMemberRoleEnum.visitor,
-    maxSize: tmb.teamId.maxSize
+    canWrite: tmb.role !== TeamMemberRoleEnum.visitor
   };
 }
 
@@ -55,14 +55,12 @@ export async function createDefaultTeam({
   teamName = 'My Team',
   avatar = '/icon/logo.svg',
   balance,
-  maxSize = 5,
   session
 }: {
   userId: string;
   teamName?: string;
   avatar?: string;
   balance?: number;
-  maxSize?: number;
   session: ClientSession;
 }) {
   // auth default team
@@ -82,7 +80,6 @@ export async function createDefaultTeam({
           name: teamName,
           avatar,
           balance,
-          maxSize,
           createTime: new Date()
         }
       ],
@@ -106,8 +103,7 @@ export async function createDefaultTeam({
     console.log('default team exist', userId);
     await MongoTeam.findByIdAndUpdate(tmb.teamId, {
       $set: {
-        ...(balance !== undefined && { balance }),
-        maxSize
+        ...(balance !== undefined && { balance })
       }
     });
   }

packages/service/support/user/team/teamSchema.ts

@@ -25,11 +25,7 @@ const TeamSchema = new Schema({
     type: Number,
     default: 0
   },
-  maxSize: {
-    type: Number,
-    default: 1
-  },
-  tagsUrl: {
+  teamDomain: {
     type: String
   },
   limit: {

packages/service/support/user/team/teamTagsSchema.ts

@@ -1,23 +1,24 @@
 import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
-import { TeamTagsSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
+import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
 import {
   TeamCollectionName,
   TeamTagsCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-const TeamTagsSchema = new Schema({
-  label: {
-    type: String,
-    required: true
-  },
+const TeamTagSchema = new Schema({
   teamId: {
     type: Schema.Types.ObjectId,
     ref: TeamCollectionName,
     required: true
   },
   key: {
-    type: String
+    type: String,
+    required: true
+  },
+  label: {
+    type: String,
+    required: true
   },
   createTime: {
     type: Date,
@@ -26,10 +27,10 @@ const TeamTagsSchema = new Schema({
 });
 
 try {
-  TeamTagsSchema.index({ teamId: 1 });
+  TeamTagSchema.index({ teamId: 1 });
 } catch (error) {
   console.log(error);
 }
 
 export const MongoTeamTags: Model<TeamTagsSchemaType> =
-  models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagsSchema);
+  models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagSchema);

packages/service/support/wallet/usage/controller.ts

@@ -31,13 +31,19 @@ export const createTrainingUsage = async ({
           {
             moduleName: 'support.wallet.moduleName.index',
             model: vectorModel,
-            charsLength: 0,
+            tokens: 0,
             amount: 0
           },
           {
             moduleName: 'support.wallet.moduleName.qa',
             model: agentModel,
-            charsLength: 0,
+            tokens: 0,
+            amount: 0
+          },
+          {
+            moduleName: 'core.dataset.training.Auto mode',
+            model: agentModel,
+            tokens: 0,
             amount: 0
           }
         ]

packages/service/support/wallet/usage/utils.ts

@@ -0,0 +1,28 @@
+import { ModelTypeEnum, getModelMap } from '../../../core/ai/model';
+
+export const formatModelChars2Points = ({
+  model,
+  tokens = 0,
+  modelType,
+  multiple = 1000
+}: {
+  model: string;
+  tokens: number;
+  modelType: `${ModelTypeEnum}`;
+  multiple?: number;
+}) => {
+  const modelData = getModelMap?.[modelType]?.(model);
+  if (!modelData) {
+    return {
+      totalPoints: 0,
+      modelName: ''
+    };
+  }
+
+  const totalPoints = (modelData.charsPointsPrice || 0) * (tokens / multiple);
+
+  return {
+    modelName: modelData.name,
+    totalPoints
+  };
+};

packages/service/type.d.ts

@@ -1,4 +1,4 @@
-import { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types';
+import { FastGPTFeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types';
 import {
   AudioSpeechModelType,
   ReRankModelItemType,
@@ -10,6 +10,7 @@ import { SubPlanType } from '@fastgpt/global/support/wallet/sub/type';
 
 declare global {
   var feConfigs: FastGPTFeConfigsType;
+  var systemEnv: SystemEnvType;
   var subPlans: SubPlanType | undefined;
 
   var llmModels: LLMModelItemType[];