v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/file/api.d.ts

@@ -1,11 +1,11 @@
 import { MongoImageTypeEnum } from './image/constants';
+import { OutLinkChatAuthProps } from '../../support/permission/chat.d';
 
-export type preUploadImgProps = {
+export type preUploadImgProps = OutLinkChatAuthProps & {
   type: `${MongoImageTypeEnum}`;
 
   expiredTime?: Date;
   metadata?: Record<string, any>;
-  shareId?: string;
 };
 export type UploadImgProps = preUploadImgProps & {
   base64Img: string;

packages/global/common/string/tiktoken/index.ts

@@ -4,6 +4,7 @@ import { Tiktoken } from 'js-tiktoken/lite';
 import { adaptChat2GptMessages } from '../../../core/chat/adapt';
 import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constant';
 import encodingJson from './cl100k_base.json';
+import { ChatMessageItemType } from '../../../core/ai/type';
 
 /* init tikToken obj */
 export function getTikTokenEnc() {
@@ -29,32 +30,35 @@ export function getTikTokenEnc() {
 /* count one prompt tokens */
 export function countPromptTokens(
   prompt = '',
-  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = ''
+  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = '',
+  tools?: any
 ) {
   const enc = getTikTokenEnc();
-  const text = `${role}\n${prompt}`;
+  const toolText = tools
+    ? JSON.stringify(tools)
+        .replace('"', '')
+        .replace('\n', '')
+        .replace(/( ){2,}/g, ' ')
+    : '';
+  const text = `${role}\n${prompt}\n${toolText}`.trim();
 
   try {
     const encodeText = enc.encode(text);
-    return encodeText.length + role.length; // 补充 role 估算值
+    const supplementaryToken = role ? 4 : 0;
+    return encodeText.length + supplementaryToken;
   } catch (error) {
     return text.length;
   }
 }
 
 /* count messages tokens */
-export function countMessagesTokens({ messages }: { messages: ChatItemType[] }) {
+export const countMessagesTokens = (messages: ChatItemType[], tools?: any) => {
   const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
 
-  let totalTokens = 0;
-  for (let i = 0; i < adaptMessages.length; i++) {
-    const item = adaptMessages[i];
-    const tokens = countPromptTokens(item.content, item.role);
-    totalTokens += tokens;
-  }
-
-  return totalTokens;
-}
+  return countGptMessagesTokens(adaptMessages, tools);
+};
+export const countGptMessagesTokens = (messages: ChatMessageItemType[], tools?: any) =>
+  messages.reduce((sum, item) => sum + countPromptTokens(item.content, item.role, tools), 0);
 
 /* slice messages from top to bottom by maxTokens */
 export function sliceMessagesTB({

packages/global/common/system/types/index.d.ts

@@ -39,9 +39,12 @@ export type FastGPTFeConfigsType = {
   systemTitle?: string;
   googleClientVerKey?: string;
   isPlus?: boolean;
+  show_phoneLogin?: boolean;
+  show_emailLogin?: boolean;
   oauth?: {
     github?: string;
     google?: string;
+    wechat?: string;
   };
   limit?: {
     exportDatasetLimitMinutes?: number;

packages/global/core/app/type.d.ts

@@ -5,7 +5,7 @@ import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.
 import { VariableInputEnum } from '../module/constants';
 import { SelectedDatasetType } from '../module/api';
 import { DatasetSearchModeEnum } from '../dataset/constants';
-import { TeamTagsSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
+import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
 export interface AppSchema {
   _id: string;
   userId: string;
@@ -20,7 +20,7 @@ export interface AppSchema {
   modules: ModuleItemType[];
   permission: `${PermissionTypeEnum}`;
   inited?: boolean;
-  teamTags: [string];
+  teamTags: string[];
 }
 
 export type AppListItemType = {

packages/global/core/chat/type.d.ts

@@ -26,23 +26,6 @@ export type ChatSchema = {
   metadata?: Record<string, any>;
 };
 
-export type teamInfoType = {
-  avatar: string;
-  balance: number;
-  createTime: string;
-  maxSize: number;
-  name: string;
-  ownerId: string;
-  tagsUrl: string;
-  _id: string;
-};
-
-export type chatAppListSchema = {
-  apps: AppType[];
-  teamInfo: teamInfoSchema;
-  uid?: string;
-};
-
 export type ChatWithAppSchema = Omit<ChatSchema, 'appId'> & {
   appId: AppSchema;
 };
@@ -90,6 +73,13 @@ export type ChatSiteItemType = ChatItemType & {
   ttsBuffer?: Uint8Array;
 };
 
+/* --------- team chat --------- */
+export type ChatAppListSchema = {
+  apps: AppType[];
+  teamInfo: teamInfoSchema;
+  uid?: string;
+};
+
 /* ---------- history ------------- */
 export type HistoryItemType = {
   chatId: string;
@@ -111,7 +101,7 @@ export type moduleDispatchResType = {
   textOutput?: string;
 
   // bill
-  charsLength?: number;
+  tokens?: number;
   model?: string;
   contextTotalLen?: number;
   totalPoints?: number;
@@ -129,7 +119,7 @@ export type moduleDispatchResType = {
   searchUsingReRank?: boolean;
   extensionModel?: string;
   extensionResult?: string;
-  extensionCharsLength?: number;
+  extensionTokens?: number;
 
   // cq
   cqList?: ClassifyQuestionAgentItemType[];

packages/global/core/dataset/constants.ts

@@ -75,17 +75,25 @@ export const DatasetCollectionSyncResultMap = {
 /* ------------ training -------------- */
 export enum TrainingModeEnum {
   chunk = 'chunk',
+  auto = 'auto',
   qa = 'qa'
 }
 
 export const TrainingTypeMap = {
   [TrainingModeEnum.chunk]: {
     label: 'core.dataset.training.Chunk mode',
-    tooltip: 'core.dataset.import.Chunk Split Tip'
+    tooltip: 'core.dataset.import.Chunk Split Tip',
+    isPlus: true
+  },
+  [TrainingModeEnum.auto]: {
+    label: 'core.dataset.training.Auto mode',
+    tooltip: 'core.dataset.training.Auto mode Tip',
+    isPlus: true
   },
   [TrainingModeEnum.qa]: {
     label: 'core.dataset.training.QA mode',
-    tooltip: 'core.dataset.import.QA Import Tip'
+    tooltip: 'core.dataset.import.QA Import Tip',
+    isPlus: true
   }
 };
 

packages/global/core/dataset/utils.ts

@@ -48,5 +48,6 @@ export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: strin
 
 export const predictDataLimitLength = (mode: `${TrainingModeEnum}`, data: any[]) => {
   if (mode === TrainingModeEnum.qa) return data.length * 20;
+  if (mode === TrainingModeEnum.auto) return data.length * 5;
   return data.length;
 };

packages/global/core/module/type.d.ts

@@ -8,7 +8,7 @@ import {
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 import { UserModelSchema } from 'support/user/type';
 import { moduleDispatchResType } from '..//chat/type';
-import { ChatModuleBillType } from '../../support/wallet/bill/type';
+import { ChatModuleUsageType } from '../../support/wallet/bill/type';
 
 export type FlowModuleTemplateType = {
   id: string; // module id, unique
@@ -129,5 +129,5 @@ export type ModuleDispatchProps<T> = ChatDispatchProps & {
 };
 export type ModuleDispatchResponse<T> = T & {
   [ModuleOutputKeyEnum.responseData]?: moduleDispatchResType;
-  [ModuleOutputKeyEnum.moduleDispatchBills]?: ChatModuleBillType[];
+  [ModuleOutputKeyEnum.moduleDispatchBills]?: ChatModuleUsageType[];
 };

packages/global/support/outLink/type.d.ts

@@ -1,3 +1,4 @@
+import { AppSchema } from 'core/app/type';
 import { OutLinkTypeEnum } from './constant';
 
 export type OutLinkSchema = {
@@ -18,6 +19,9 @@ export type OutLinkSchema = {
     hookUrl?: string;
   };
 };
+export type OutLinkWithAppType = Omit<OutLinkSchema, 'appId'> & {
+  appId: AppSchema;
+};
 
 export type OutLinkEditType = {
   _id?: string;

packages/global/support/permission/chat.d.ts

@@ -0,0 +1,9 @@
+type ShareChatAuthProps = {
+  shareId?: string;
+  outLinkUid?: string;
+};
+type TeamChatAuthProps = {
+  teamId?: string;
+  teamToken?: string;
+};
+export type OutLinkChatAuthProps = ShareChatAuthProps & TeamChatAuthProps;

packages/global/support/permission/constant.ts

@@ -2,7 +2,8 @@ export enum AuthUserTypeEnum {
   token = 'token',
   root = 'root',
   apikey = 'apikey',
-  outLink = 'outLink'
+  outLink = 'outLink',
+  teamDomain = 'teamDomain'
 }
 
 export enum PermissionTypeEnum {

packages/global/support/user/api.d.ts

@@ -10,7 +10,11 @@ export type OauthLoginProps = {
   code: string;
   callbackUrl: string;
   inviterId?: string;
-  tmbId?: string;
+};
+
+export type WxLoginProps = {
+  inviterId?: string;
+  code: string;
 };
 
 export type FastLoginProps = {

packages/global/support/user/auth/constants.ts

@@ -0,0 +1,11 @@
+export enum UserAuthTypeEnum {
+  register = 'register',
+  findPassword = 'findPassword',
+  wxLogin = 'wxLogin'
+}
+
+export const userAuthTypeMap = {
+  [UserAuthTypeEnum.register]: 'register',
+  [UserAuthTypeEnum.findPassword]: 'findPassword',
+  [UserAuthTypeEnum.wxLogin]: 'wxLogin'
+};

packages/global/support/user/constant.ts

@@ -13,10 +13,6 @@ export const userStatusMap = {
 
 export enum OAuthEnum {
   github = 'github',
-  google = 'google'
-}
-
-export enum UserAuthTypeEnum {
-  register = 'register',
-  findPassword = 'findPassword'
+  google = 'google',
+  wechat = 'wechat'
 }

packages/global/support/user/login/api.d.ts

@@ -0,0 +1,4 @@
+export type GetWXLoginQRResponse = {
+  code: string;
+  codeUrl: string;
+};

packages/global/support/user/team/controller.d.ts

@@ -15,7 +15,7 @@ export type UpdateTeamProps = {
   teamId: string;
   name?: string;
   avatar?: string;
-  tagsUrl?: string;
+  teamDomain?: string;
 };
 
 /* ------------- member ----------- */

packages/global/support/user/team/tag.d.ts

@@ -0,0 +1,14 @@
+export type AuthTeamTagTokenProps = {
+  teamId: string;
+  teamToken: string;
+};
+
+export type AuthTokenFromTeamDomainResponse = {
+  success: boolean;
+  msg?: string;
+  message?: string;
+  data: {
+    uid: string;
+    tags: string[];
+  };
+};

packages/global/support/user/team/type.d.ts

@@ -8,22 +8,19 @@ export type TeamSchema = {
   avatar: string;
   createTime: Date;
   balance: number;
-  maxSize: number;
-  tagsUrl: string;
+  teamDomain: string;
   limit: {
     lastExportDatasetTime: Date;
     lastWebsiteSyncTime: Date;
   };
 };
 export type tagsType = {
-  label: string,
-  key: string
-}
-export type TeamTagsSchema = {
-  _id: string;
   label: string;
-  teamId: string;
   key: string;
+};
+export type TeamTagSchema = TeamTagItemType & {
+  _id: string;
+  teamId: string;
   createTime: Date;
 };
 
@@ -56,11 +53,11 @@ export type TeamItemType = {
   avatar: string;
   balance: number;
   tmbId: string;
+  teamDomain: string;
   defaultTeam: boolean;
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
   canWrite: boolean;
-  maxSize: number;
 };
 
 export type TeamMemberItemType = {
@@ -72,3 +69,8 @@ export type TeamMemberItemType = {
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
 };
+
+export type TeamTagItemType = {
+  label: string;
+  key: string;
+};

packages/global/support/wallet/bill/type.d.ts

@@ -21,9 +21,9 @@ export type BillSchemaType = {
   };
 };
 
-export type ChatModuleBillType = {
+export type ChatModuleUsageType = {
+  tokens?: number;
   totalPoints: number;
   moduleName: string;
   model?: string;
-  charsLength?: number;
 };

packages/global/support/wallet/usage/api.d.ts

@@ -20,7 +20,6 @@ export type CreateUsageProps = {
   appName: string;
   appId?: string;
   totalPoints: number;
-  // inputTokens: number;
   source: `${UsageSourceEnum}`;
   list: UsageListItemType[];
 };

packages/global/support/wallet/usage/type.d.ts

@@ -2,6 +2,7 @@ import { CreateUsageProps } from './api';
 import { UsageSourceEnum } from './constants';
 
 export type UsageListItemCountType = {
+  tokens?: number;
   charsLength?: number;
   duration?: number;
 };

packages/service/common/mongo/init.ts

@@ -34,6 +34,12 @@ export async function connectMongo({
       retryReads: true
     });
 
+    mongoose.connection.on('error', (error) => {
+      console.log('mongo error', error);
+      global.mongodb?.disconnect();
+      global.mongodb = undefined;
+    });
+
     console.log('mongo connected');
 
     afterHook && (await afterHook());

packages/service/common/system/log.ts

@@ -3,9 +3,12 @@ import dayjs from 'dayjs';
 /* add logger */
 export const addLog = {
   log(level: 'info' | 'warn' | 'error', msg: string, obj: Record<string, any> = {}) {
+    const stringifyObj = JSON.stringify(obj);
+    const isEmpty = Object.keys(obj).length === 0;
+
     console.log(
       `[${level.toLocaleUpperCase()}] ${dayjs().format('YYYY-MM-DD HH:mm:ss')} ${msg} ${
-        level !== 'error' ? JSON.stringify(obj) : ''
+        level !== 'error' && !isEmpty ? stringifyObj : ''
       }`
     );
 

packages/service/common/system/tools.ts

@@ -11,6 +11,7 @@ export const initFastGPTConfig = (config?: FastGPTConfigFileType) => {
   if (!config) return;
 
   global.feConfigs = config.feConfigs;
+  global.systemEnv = config.systemEnv;
   global.subPlans = config.subPlans;
 
   global.llmModels = config.llmModels;

packages/service/common/vectorStore/controller.ts

@@ -22,7 +22,7 @@ export const insertDatasetDataVector = async ({
   query: string;
   model: VectorModelItemType;
 }) => {
-  const { vectors, charsLength } = await getVectorsByText({
+  const { vectors, tokens } = await getVectorsByText({
     model,
     input: query
   });
@@ -32,27 +32,7 @@ export const insertDatasetDataVector = async ({
   });
 
   return {
-    charsLength,
+    tokens,
     insertId
   };
 };
-
-// export const updateDatasetDataVector = async ({
-//   id,
-//   ...props
-// }: InsertVectorProps & {
-//   id: string;
-//   query: string;
-//   model: VectorModelItemType;
-// }) => {
-//   // insert new vector
-//   const { charsLength, insertId } = await insertDatasetDataVector(props);
-
-//   // delete old vector
-//   await deleteDatasetDataVector({
-//     teamId: props.teamId,
-//     id
-//   });
-
-//   return { charsLength, insertId };
-// };

packages/service/core/ai/embedding/index.ts

@@ -1,6 +1,6 @@
 import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../config';
-import { replaceValidChars } from '../../chat/utils';
+import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 
 type GetVectorProps = {
   model: VectorModelItemType;
@@ -37,7 +37,7 @@ export async function getVectorsByText({ model, input }: GetVectorProps) {
         }
 
         return {
-          charsLength: replaceValidChars(input).length,
+          tokens: countPromptTokens(input),
           vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
         };
       });

packages/service/core/ai/functions/createQuestionGuide.ts

@@ -1,6 +1,6 @@
 import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
 import { getAIApi } from '../config';
-import { countGptMessagesChars } from '../../chat/utils';
+import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
 
 export const Prompt_QuestionGuide = `我不太清楚问你什么问题，请帮我生成 3 个问题，引导我继续提问。问题的长度应小于20个字符，按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
 
@@ -34,12 +34,12 @@ export async function createQuestionGuide({
   const start = answer.indexOf('[');
   const end = answer.lastIndexOf(']');
 
-  const charsLength = countGptMessagesChars(concatMessages);
+  const tokens = countGptMessagesTokens(concatMessages);
 
   if (start === -1 || end === -1) {
     return {
       result: [],
-      charsLength: 0
+      tokens: 0
     };
   }
 
@@ -51,12 +51,12 @@ export async function createQuestionGuide({
   try {
     return {
       result: JSON.parse(jsonStr),
-      charsLength
+      tokens
     };
   } catch (error) {
     return {
       result: [],
-      charsLength: 0
+      tokens: 0
     };
   }
 }

packages/service/core/ai/functions/queryExtension.ts

@@ -1,7 +1,7 @@
 import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { getAIApi } from '../config';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
-import { countGptMessagesChars } from '../../chat/utils';
+import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
 
 /* 
     query extension - 问题扩展
@@ -106,7 +106,7 @@ export const queryExtension = async ({
   rawQuery: string;
   extensionQueries: string[];
   model: string;
-  charsLength: number;
+  tokens: number;
 }> => {
   const systemFewShot = chatBg
     ? `Q: 对话背景。
@@ -148,7 +148,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: [],
       model,
-      charsLength: 0
+      tokens: 0
     };
   }
 
@@ -161,7 +161,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: queries,
       model,
-      charsLength: countGptMessagesChars(messages)
+      tokens: countGptMessagesTokens(messages)
     };
   } catch (error) {
     console.log(error);
@@ -169,7 +169,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: [],
       model,
-      charsLength: 0
+      tokens: 0
     };
   }
 };

packages/service/core/chat/utils.ts

@@ -1,11 +1,7 @@
 import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
 import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
-import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
-import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
-import type {
-  ChatCompletionContentPart,
-  ChatMessageItemType
-} from '@fastgpt/global/core/ai/type.d';
+import { countMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
+import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
 import axios from 'axios';
 
 /* slice chat context by tokens */
@@ -32,26 +28,34 @@ export function ChatContextFilter({
   const chatPrompts: ChatItemType[] = messages.slice(chatStartIndex);
 
   // reduce token of systemPrompt
-  maxTokens -= countMessagesTokens({
-    messages: systemPrompts
-  });
+  maxTokens -= countMessagesTokens(systemPrompts);
 
-  // 根据 tokens 截断内容
-  const chats: ChatItemType[] = [];
+  // Save the last chat prompt(question)
+  const question = chatPrompts.pop();
+  if (!question) {
+    return systemPrompts;
+  }
+  const chats: ChatItemType[] = [question];
 
-  // 从后往前截取对话内容
-  for (let i = chatPrompts.length - 1; i >= 0; i--) {
-    const item = chatPrompts[i];
-    chats.unshift(item);
+  // 从后往前截取对话内容, 每次需要截取2个
+  while (1) {
+    const assistant = chatPrompts.pop();
+    const user = chatPrompts.pop();
+    if (!assistant || !user) {
+      break;
+    }
 
-    const tokens = countPromptTokens(item.value, adaptRole_Chat2Message(item.obj));
+    const tokens = countMessagesTokens([assistant, user]);
     maxTokens -= tokens;
+    /* 整体 tokens 超出范围，截断  */
+    if (maxTokens < 0) {
+      break;
+    }
 
-    /* 整体 tokens 超出范围, system必须保留 */
-    if (maxTokens <= 0) {
-      if (chats.length > 1) {
-        chats.shift();
-      }
+    chats.unshift(assistant);
+    chats.unshift(user);
+
+    if (chatPrompts.length === 0) {
       break;
     }
   }
@@ -59,16 +63,6 @@ export function ChatContextFilter({
   return [...systemPrompts, ...chats];
 }
 
-export const replaceValidChars = (str: string) => {
-  const reg = /[\s\r\n]+/g;
-  return str.replace(reg, '');
-};
-export const countMessagesChars = (messages: ChatItemType[]) => {
-  return messages.reduce((sum, item) => sum + replaceValidChars(item.value).length, 0);
-};
-export const countGptMessagesChars = (messages: ChatMessageItemType[]) =>
-  messages.reduce((sum, item) => sum + replaceValidChars(item.content).length, 0);
-
 /**
     string to vision model. Follow the markdown code block rule for interception:
 

packages/service/core/dataset/schema.ts

@@ -4,6 +4,7 @@ import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import {
   DatasetStatusEnum,
   DatasetStatusMap,
+  DatasetTypeEnum,
   DatasetTypeMap
 } from '@fastgpt/global/core/dataset/constants';
 import {
@@ -39,7 +40,7 @@ const DatasetSchema = new Schema({
     type: String,
     enum: Object.keys(DatasetTypeMap),
     required: true,
-    default: 'dataset'
+    default: DatasetTypeEnum.dataset
   },
   status: {
     type: String,

packages/service/core/dataset/training/controller.ts

@@ -46,12 +46,16 @@ export async function pushDataListToTrainingQueue({
   } = await getCollectionWithDataset(collectionId);
 
   const checkModelValid = async () => {
-    if (trainingMode === TrainingModeEnum.chunk) {
-      const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
-      if (!vectorModelData) {
-        return Promise.reject(`File model ${vectorModel} is inValid`);
-      }
+    const agentModelData = datasetModelList?.find((item) => item.model === agentModel);
+    if (!agentModelData) {
+      return Promise.reject(`Vector model ${agentModel} is inValid`);
+    }
+    const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
+    if (!vectorModelData) {
+      return Promise.reject(`File model ${vectorModel} is inValid`);
+    }
 
+    if (trainingMode === TrainingModeEnum.chunk) {
       return {
         maxToken: vectorModelData.maxToken * 1.3,
         model: vectorModelData.model,
@@ -59,17 +63,14 @@ export async function pushDataListToTrainingQueue({
       };
     }
 
-    if (trainingMode === TrainingModeEnum.qa) {
-      const qaModelData = datasetModelList?.find((item) => item.model === agentModel);
-      if (!qaModelData) {
-        return Promise.reject(`Vector model ${agentModel} is inValid`);
-      }
+    if (trainingMode === TrainingModeEnum.qa || trainingMode === TrainingModeEnum.auto) {
       return {
-        maxToken: qaModelData.maxContext * 0.8,
-        model: qaModelData.model,
+        maxToken: agentModelData.maxContext * 0.8,
+        model: agentModelData.model,
         weight: 0
       };
     }
+
     return Promise.reject(`Training mode "${trainingMode}" is inValid`);
   };
 

packages/service/core/dataset/training/utils.ts

@@ -0,0 +1,41 @@
+import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
+import { addLog } from '../../../common/system/log';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { MongoDatasetTraining } from './schema';
+
+export const checkInvalidChunkAndLock = async ({
+  err,
+  errText,
+  data
+}: {
+  err: any;
+  errText: string;
+  data: DatasetTrainingSchemaType;
+}) => {
+  if (err?.response) {
+    addLog.error(`openai error: ${errText}`, {
+      status: err.response?.status,
+      statusText: err.response?.statusText,
+      data: err.response?.data
+    });
+  } else {
+    addLog.error(getErrText(err, errText), err);
+  }
+
+  if (
+    err?.message === 'invalid message format' ||
+    err?.type === 'invalid_request_error' ||
+    err?.code === 500
+  ) {
+    addLog.info('Lock training data');
+    console.log(err);
+
+    try {
+      await MongoDatasetTraining.findByIdAndUpdate(data._id, {
+        lockTime: new Date('2998/5/5')
+      });
+    } catch (error) {}
+    return true;
+  }
+  return false;
+};

packages/service/support/outLink/schema.ts

@@ -6,6 +6,7 @@ import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
+import { appCollectionName } from '../../core/app/schema';
 
 const OutLinkSchema = new Schema({
   shareId: {
@@ -24,7 +25,7 @@ const OutLinkSchema = new Schema({
   },
   appId: {
     type: Schema.Types.ObjectId,
-    ref: 'model',
+    ref: appCollectionName,
     required: true
   },
   type: {

packages/service/support/permission/auth/common.ts

@@ -1,7 +1,5 @@
-import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { parseHeaderCert } from '../controller';
 import { AuthModeType } from '../type';
-import { authOutLinkValid } from './outLink';
 import { SERVICE_LOCAL_HOST } from '../../../common/system/tools';
 
 export const authCert = async (props: AuthModeType) => {
@@ -13,25 +11,6 @@ export const authCert = async (props: AuthModeType) => {
     canWrite: true
   };
 };
-export async function authCertOrShareId({
-  shareId,
-  ...props
-}: AuthModeType & { shareId?: string }) {
-  if (!shareId) {
-    return authCert(props);
-  }
-
-  const { shareChat } = await authOutLinkValid({ shareId });
-
-  return {
-    teamId: String(shareChat.teamId),
-    tmbId: String(shareChat.tmbId),
-    authType: AuthUserTypeEnum.outLink,
-    apikey: '',
-    isOwner: false,
-    canWrite: false
-  };
-}
 
 /* auth the request from local service */
 export const authRequestFromLocal = ({ req }: AuthModeType) => {

packages/service/support/user/team/controller.ts

@@ -23,11 +23,11 @@ async function getTeamMember(match: Record<string, any>): Promise<TeamItemType>
     avatar: tmb.teamId.avatar,
     balance: tmb.teamId.balance,
     tmbId: String(tmb._id),
+    teamDomain: tmb.teamId?.teamDomain,
     role: tmb.role,
     status: tmb.status,
     defaultTeam: tmb.defaultTeam,
-    canWrite: tmb.role !== TeamMemberRoleEnum.visitor,
-    maxSize: tmb.teamId.maxSize
+    canWrite: tmb.role !== TeamMemberRoleEnum.visitor
   };
 }
 
@@ -55,14 +55,12 @@ export async function createDefaultTeam({
   teamName = 'My Team',
   avatar = '/icon/logo.svg',
   balance,
-  maxSize = 5,
   session
 }: {
   userId: string;
   teamName?: string;
   avatar?: string;
   balance?: number;
-  maxSize?: number;
   session: ClientSession;
 }) {
   // auth default team
@@ -82,7 +80,6 @@ export async function createDefaultTeam({
           name: teamName,
           avatar,
           balance,
-          maxSize,
           createTime: new Date()
         }
       ],
@@ -106,8 +103,7 @@ export async function createDefaultTeam({
     console.log('default team exist', userId);
     await MongoTeam.findByIdAndUpdate(tmb.teamId, {
       $set: {
-        ...(balance !== undefined && { balance }),
-        maxSize
+        ...(balance !== undefined && { balance })
       }
     });
   }

packages/service/support/user/team/teamSchema.ts

@@ -25,11 +25,7 @@ const TeamSchema = new Schema({
     type: Number,
     default: 0
   },
-  maxSize: {
-    type: Number,
-    default: 1
-  },
-  tagsUrl: {
+  teamDomain: {
     type: String
   },
   limit: {

packages/service/support/user/team/teamTagsSchema.ts

@@ -1,23 +1,24 @@
 import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
-import { TeamTagsSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
+import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
 import {
   TeamCollectionName,
   TeamTagsCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-const TeamTagsSchema = new Schema({
-  label: {
-    type: String,
-    required: true
-  },
+const TeamTagSchema = new Schema({
   teamId: {
     type: Schema.Types.ObjectId,
     ref: TeamCollectionName,
     required: true
   },
   key: {
-    type: String
+    type: String,
+    required: true
+  },
+  label: {
+    type: String,
+    required: true
   },
   createTime: {
     type: Date,
@@ -26,10 +27,10 @@ const TeamTagsSchema = new Schema({
 });
 
 try {
-  TeamTagsSchema.index({ teamId: 1 });
+  TeamTagSchema.index({ teamId: 1 });
 } catch (error) {
   console.log(error);
 }
 
 export const MongoTeamTags: Model<TeamTagsSchemaType> =
-  models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagsSchema);
+  models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagSchema);

packages/service/support/wallet/usage/controller.ts

@@ -31,13 +31,19 @@ export const createTrainingUsage = async ({
           {
             moduleName: 'support.wallet.moduleName.index',
             model: vectorModel,
-            charsLength: 0,
+            tokens: 0,
             amount: 0
           },
           {
             moduleName: 'support.wallet.moduleName.qa',
             model: agentModel,
-            charsLength: 0,
+            tokens: 0,
+            amount: 0
+          },
+          {
+            moduleName: 'core.dataset.training.Auto mode',
+            model: agentModel,
+            tokens: 0,
             amount: 0
           }
         ]

packages/service/support/wallet/usage/utils.ts

@@ -0,0 +1,28 @@
+import { ModelTypeEnum, getModelMap } from '../../../core/ai/model';
+
+export const formatModelChars2Points = ({
+  model,
+  tokens = 0,
+  modelType,
+  multiple = 1000
+}: {
+  model: string;
+  tokens: number;
+  modelType: `${ModelTypeEnum}`;
+  multiple?: number;
+}) => {
+  const modelData = getModelMap?.[modelType]?.(model);
+  if (!modelData) {
+    return {
+      totalPoints: 0,
+      modelName: ''
+    };
+  }
+
+  const totalPoints = (modelData.charsPointsPrice || 0) * (tokens / multiple);
+
+  return {
+    modelName: modelData.name,
+    totalPoints
+  };
+};

packages/service/type.d.ts

@@ -1,4 +1,4 @@
-import { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types';
+import { FastGPTFeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types';
 import {
   AudioSpeechModelType,
   ReRankModelItemType,
@@ -10,6 +10,7 @@ import { SubPlanType } from '@fastgpt/global/support/wallet/sub/type';
 
 declare global {
   var feConfigs: FastGPTFeConfigsType;
+  var systemEnv: SystemEnvType;
   var subPlans: SubPlanType | undefined;
 
   var llmModels: LLMModelItemType[];

packages/web/components/common/Icon/constants.ts

@@ -52,6 +52,7 @@ export const iconPaths = {
   'common/uploadFileFill': () => import('./icons/common/uploadFileFill.svg'),
   'common/viewLight': () => import('./icons/common/viewLight.svg'),
   'common/voiceLight': () => import('./icons/common/voiceLight.svg'),
+  'common/wechatFill': () => import('./icons/common/wechatFill.svg'),
   copy: () => import('./icons/copy.svg'),
   'core/app/aiFill': () => import('./icons/core/app/aiFill.svg'),
   'core/app/aiLight': () => import('./icons/core/app/aiLight.svg'),
@@ -144,6 +145,7 @@ export const iconPaths = {
   save: () => import('./icons/save.svg'),
   stop: () => import('./icons/stop.svg'),
   'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'),
+  'support/account/passwordLogin': () => import('./icons/support/account/passwordLogin.svg'),
   'support/account/plans': () => import('./icons/support/account/plans.svg'),
   'support/account/promotionLight': () => import('./icons/support/account/promotionLight.svg'),
   'support/bill/extraDatasetsize': () => import('./icons/support/bill/extraDatasetsize.svg'),

packages/web/components/common/Icon/icons/common/wechatFill.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1708675975750" class="icon" viewBox="0 0 1025 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4218" xmlns:xlink="http://www.w3.org/1999/xlink" width="200.1953125" height="200"><path d="M1024.16 694.816c0-149.92-143.104-271.392-319.584-271.392-176.576 0-319.68 121.504-319.68 271.392S528 966.208 704.576 966.208c55.456 0 107.648-12.096 153.184-33.248l125.984 54.528-14.592-140.544c34.784-43.392 55.04-95.808 55.04-152.128zM596.832 621.28c-25.152 0-45.472-20.352-45.472-45.472s20.32-45.472 45.472-45.472c25.12 0 45.44 20.384 45.44 45.472s-20.384 45.472-45.44 45.472z m215.392 0c-25.056 0-45.44-20.352-45.44-45.472s20.384-45.472 45.44-45.472c25.184 0 45.536 20.384 45.536 45.472s-20.352 45.472-45.536 45.472zM704.576 387.488c49.376 0 96.416 8.8 139.264 24.64 0.32-5.728 0.992-11.232 0.992-16.992 0-198.08-189.152-358.624-422.432-358.624C189.184 36.512 0.032 197.024 0.032 395.136c0 74.496 26.816 143.776 72.704 201.12L53.472 781.92l166.432-72.096c41.216 19.2 86.784 32.16 134.88 38.784-3.616-17.504-5.824-35.424-5.824-53.792 0.032-169.44 159.552-307.296 355.616-307.296z m-139.808-209.6c33.184 0 60 26.88 60 60 0 33.184-26.816 60.064-60 60.064s-60.032-26.88-60.032-60.064c0-33.152 26.88-60 60.032-60zM280.032 297.952c-33.184 0-60-26.88-60-60.064 0-33.152 26.848-60 60-60 33.184 0 60.032 26.88 60.032 60s-26.88 60.064-60.032 60.064z" fill="#51C332" p-id="4219"></path></svg>

packages/web/components/common/Icon/icons/support/account/passwordLogin.svg

@@ -0,0 +1,6 @@
+<svg t="1709471698048" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4242"
+    width="128" height="128">
+    <path
+        d="M855.158154 945.664H168.999385c-28.081231 0-50.845538-22.843077-50.845539-51.003077V486.833231C118.153846 458.673231 129.457231 433.230769 157.538462 433.230769h708.923076c28.081231 0 39.502769 25.442462 39.50277 53.602462v407.827692c0 28.16-22.764308 51.003077-50.806154 51.003077z m-340.913231-376.595692a99.761231 99.761231 0 0 0-99.603692 99.958154c0 40.251077 23.827692 74.712615 57.974154 90.54523V827.076923a39.384615 39.384615 0 0 0 78.76923 0v-65.417846a99.879385 99.879385 0 0 0 62.424616-92.632615 99.761231 99.761231 0 0 0-99.564308-99.958154z m0.551385-396.524308c-104.841846 0-189.794462 81.329231-197.159385 184.123077H217.718154C229.060923 201.334154 358.321231 78.769231 516.489846 78.769231s287.428923 122.564923 298.732308 277.897846h-103.266462c-7.364923-102.793846-92.317538-184.123077-197.159384-184.123077z"
+        fill="#3B9BF8" p-id="4243"></path>
+</svg>

packages/web/components/common/Input/HttpInput/Editor.tsx

@@ -64,14 +64,17 @@ export default function Editor({
 
   useEffect(() => {
     setKey(getNanoid(6));
+    setFocus(false);
   }, [updateTrigger]);
 
   const dropdownVariables = useMemo(
     () =>
       variables.filter((item) => {
-        return item.key.includes(currentValue || '') && item.key !== currentValue;
+        const key = item.key.toLowerCase();
+        const current = currentValue?.toLowerCase();
+        return key.includes(current || '') && item.key !== currentValue;
       }),
-    [currentValue]
+    [currentValue, variables]
   );
 
   return (
@@ -115,7 +118,7 @@ export default function Editor({
           }}
         />
         {hasVariablePlugin ? <VariablePickerPlugin variables={variables} /> : ''}
-        {hasVariablePlugin ? <VariablePlugin variables={variables} /> : ''}
+        <VariablePlugin variables={variables} />
         <OnBlurPlugin onBlur={onBlur} />
         <SingleLinePlugin />
       </LexicalComposer>

packages/web/components/common/Textarea/JsonEditor/index.tsx

@@ -21,6 +21,7 @@ type Props = Omit<BoxProps, 'resize' | 'onChange'> & {
   value?: string;
   onChange?: (e: string) => void;
   variables?: EditorVariablePickerType[];
+  defaultHeight?: number;
 };
 
 const options = {
@@ -52,11 +53,12 @@ const JSONEditor = ({
   resize,
   variables = [],
   placeholder,
+  defaultHeight = 100,
   ...props
 }: Props) => {
   const { toast } = useToast();
   const { t } = useTranslation();
-  const [height, setHeight] = useState(props.height || 100);
+  const [height, setHeight] = useState(defaultHeight);
   const [placeholderDisplay, setPlaceholderDisplay] = useState('block');
   const initialY = useRef(0);
   const completionRegisterRef = useRef<any>();
@@ -205,7 +207,15 @@ const JSONEditor = ({
   }, []);
 
   return (
-    <Box position={'relative'}>
+    <Box
+      borderWidth={'1px'}
+      borderRadius={'md'}
+      borderColor={'myGray.200'}
+      py={2}
+      height={height}
+      position={'relative'}
+      {...props}
+    >
       {resize && (
         <Box
           position={'absolute'}
@@ -219,54 +229,43 @@ const JSONEditor = ({
           <MyIcon name={'common/editor/resizer'} width={'16px'} height={'16px'} />
         </Box>
       )}
-
+      <Editor
+        height={'100%'}
+        defaultLanguage="json"
+        options={options as any}
+        theme="JSONEditorTheme"
+        beforeMount={beforeMount}
+        defaultValue={defaultValue}
+        value={value}
+        onChange={(e) => {
+          onChange?.(e || '');
+          if (!e) {
+            setPlaceholderDisplay('block');
+          } else {
+            setPlaceholderDisplay('none');
+          }
+        }}
+        wrapperProps={{
+          onBlur
+        }}
+        onMount={() => {
+          if (!value) {
+            setPlaceholderDisplay('block');
+          } else {
+            setPlaceholderDisplay('none');
+          }
+        }}
+      />
       <Box
-        borderWidth={'1px'}
-        borderRadius={'md'}
-        borderColor={'myGray.200'}
-        py={2}
-        height={'auto'}
-        position={'relative'}
-        {...props}
+        className="monaco-placeholder"
+        position={'absolute'}
+        top={2}
+        left={4}
+        fontSize={'xs'}
+        color={'myGray.500'}
+        display={placeholderDisplay}
       >
-        <Editor
-          height={height}
-          defaultLanguage="json"
-          options={options as any}
-          theme="JSONEditorTheme"
-          beforeMount={beforeMount}
-          defaultValue={defaultValue}
-          value={value}
-          onChange={(e) => {
-            onChange?.(e || '');
-            if (!e) {
-              setPlaceholderDisplay('block');
-            } else {
-              setPlaceholderDisplay('none');
-            }
-          }}
-          wrapperProps={{
-            onBlur
-          }}
-          onMount={() => {
-            if (!value) {
-              setPlaceholderDisplay('block');
-            } else {
-              setPlaceholderDisplay('none');
-            }
-          }}
-        />
-        <Box
-          className="monaco-placeholder"
-          position={'absolute'}
-          top={2}
-          left={4}
-          fontSize={'xs'}
-          color={'myGray.500'}
-          display={placeholderDisplay}
-        >
-          {placeholder}
-        </Box>
+        {placeholder}
       </Box>
     </Box>
   );

packages/web/components/common/Textarea/PromptEditor/modules/DropDownMenu/index.tsx

@@ -44,6 +44,9 @@ export default function DropDownMenu({
       position={'fixed'}
       w={'auto'}
       zIndex={99999}
+      maxH={'300px'}
+      overflow={'auto'}
+      className="nowheel"
     >
       {variables.map((item, index) => (
         <Flex

packages/web/package.json

@@ -14,13 +14,13 @@
     "@fingerprintjs/fingerprintjs": "^4.2.1",
     "@monaco-editor/react": "^4.6.0",
     "mammoth": "^1.6.0",
-    "i18next": "^22.5.1",
+    "i18next": "23.10.0",
     "joplin-turndown-plugin-gfm": "^1.0.12",
-    "next-i18next": "^13.3.0",
+    "next-i18next": "15.2.0",
     "pdfjs-dist": "4.0.269",
     "react": "18.2.0",
     "react-dom": "18.2.0",
-    "react-i18next": "^12.3.1",
+    "react-i18next": "13.5.0",
     "turndown": "^7.1.2",
     "lexical": "0.12.6",
     "@lexical/react": "0.12.6",