v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/file/api.d.ts

@@ -1,11 +1,11 @@
 import { MongoImageTypeEnum } from './image/constants';
+import { OutLinkChatAuthProps } from '../../support/permission/chat.d';
 
-export type preUploadImgProps = {
+export type preUploadImgProps = OutLinkChatAuthProps & {
   type: `${MongoImageTypeEnum}`;
 
   expiredTime?: Date;
   metadata?: Record<string, any>;
-  shareId?: string;
 };
 export type UploadImgProps = preUploadImgProps & {
   base64Img: string;

packages/global/common/string/tiktoken/index.ts

@@ -4,6 +4,7 @@ import { Tiktoken } from 'js-tiktoken/lite';
 import { adaptChat2GptMessages } from '../../../core/chat/adapt';
 import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constant';
 import encodingJson from './cl100k_base.json';
+import { ChatMessageItemType } from '../../../core/ai/type';
 
 /* init tikToken obj */
 export function getTikTokenEnc() {
@@ -29,32 +30,35 @@ export function getTikTokenEnc() {
 /* count one prompt tokens */
 export function countPromptTokens(
   prompt = '',
-  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = ''
+  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = '',
+  tools?: any
 ) {
   const enc = getTikTokenEnc();
-  const text = `${role}\n${prompt}`;
+  const toolText = tools
+    ? JSON.stringify(tools)
+        .replace('"', '')
+        .replace('\n', '')
+        .replace(/( ){2,}/g, ' ')
+    : '';
+  const text = `${role}\n${prompt}\n${toolText}`.trim();
 
   try {
     const encodeText = enc.encode(text);
-    return encodeText.length + role.length; // 补充 role 估算值
+    const supplementaryToken = role ? 4 : 0;
+    return encodeText.length + supplementaryToken;
   } catch (error) {
     return text.length;
   }
 }
 
 /* count messages tokens */
-export function countMessagesTokens({ messages }: { messages: ChatItemType[] }) {
+export const countMessagesTokens = (messages: ChatItemType[], tools?: any) => {
   const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
 
-  let totalTokens = 0;
-  for (let i = 0; i < adaptMessages.length; i++) {
-    const item = adaptMessages[i];
-    const tokens = countPromptTokens(item.content, item.role);
-    totalTokens += tokens;
-  }
-
-  return totalTokens;
-}
+  return countGptMessagesTokens(adaptMessages, tools);
+};
+export const countGptMessagesTokens = (messages: ChatMessageItemType[], tools?: any) =>
+  messages.reduce((sum, item) => sum + countPromptTokens(item.content, item.role, tools), 0);
 
 /* slice messages from top to bottom by maxTokens */
 export function sliceMessagesTB({

packages/global/common/system/types/index.d.ts

@@ -39,9 +39,12 @@ export type FastGPTFeConfigsType = {
   systemTitle?: string;
   googleClientVerKey?: string;
   isPlus?: boolean;
+  show_phoneLogin?: boolean;
+  show_emailLogin?: boolean;
   oauth?: {
     github?: string;
     google?: string;
+    wechat?: string;
   };
   limit?: {
     exportDatasetLimitMinutes?: number;

packages/global/core/app/type.d.ts

@@ -5,7 +5,7 @@ import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.
 import { VariableInputEnum } from '../module/constants';
 import { SelectedDatasetType } from '../module/api';
 import { DatasetSearchModeEnum } from '../dataset/constants';
-import { TeamTagsSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
+import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
 export interface AppSchema {
   _id: string;
   userId: string;
@@ -20,7 +20,7 @@ export interface AppSchema {
   modules: ModuleItemType[];
   permission: `${PermissionTypeEnum}`;
   inited?: boolean;
-  teamTags: [string];
+  teamTags: string[];
 }
 
 export type AppListItemType = {

packages/global/core/chat/type.d.ts

@@ -26,23 +26,6 @@ export type ChatSchema = {
   metadata?: Record<string, any>;
 };
 
-export type teamInfoType = {
-  avatar: string;
-  balance: number;
-  createTime: string;
-  maxSize: number;
-  name: string;
-  ownerId: string;
-  tagsUrl: string;
-  _id: string;
-};
-
-export type chatAppListSchema = {
-  apps: AppType[];
-  teamInfo: teamInfoSchema;
-  uid?: string;
-};
-
 export type ChatWithAppSchema = Omit<ChatSchema, 'appId'> & {
   appId: AppSchema;
 };
@@ -90,6 +73,13 @@ export type ChatSiteItemType = ChatItemType & {
   ttsBuffer?: Uint8Array;
 };
 
+/* --------- team chat --------- */
+export type ChatAppListSchema = {
+  apps: AppType[];
+  teamInfo: teamInfoSchema;
+  uid?: string;
+};
+
 /* ---------- history ------------- */
 export type HistoryItemType = {
   chatId: string;
@@ -111,7 +101,7 @@ export type moduleDispatchResType = {
   textOutput?: string;
 
   // bill
-  charsLength?: number;
+  tokens?: number;
   model?: string;
   contextTotalLen?: number;
   totalPoints?: number;
@@ -129,7 +119,7 @@ export type moduleDispatchResType = {
   searchUsingReRank?: boolean;
   extensionModel?: string;
   extensionResult?: string;
-  extensionCharsLength?: number;
+  extensionTokens?: number;
 
   // cq
   cqList?: ClassifyQuestionAgentItemType[];

packages/global/core/dataset/constants.ts

@@ -75,17 +75,25 @@ export const DatasetCollectionSyncResultMap = {
 /* ------------ training -------------- */
 export enum TrainingModeEnum {
   chunk = 'chunk',
+  auto = 'auto',
   qa = 'qa'
 }
 
 export const TrainingTypeMap = {
   [TrainingModeEnum.chunk]: {
     label: 'core.dataset.training.Chunk mode',
-    tooltip: 'core.dataset.import.Chunk Split Tip'
+    tooltip: 'core.dataset.import.Chunk Split Tip',
+    isPlus: true
+  },
+  [TrainingModeEnum.auto]: {
+    label: 'core.dataset.training.Auto mode',
+    tooltip: 'core.dataset.training.Auto mode Tip',
+    isPlus: true
   },
   [TrainingModeEnum.qa]: {
     label: 'core.dataset.training.QA mode',
-    tooltip: 'core.dataset.import.QA Import Tip'
+    tooltip: 'core.dataset.import.QA Import Tip',
+    isPlus: true
   }
 };
 

packages/global/core/dataset/utils.ts

@@ -48,5 +48,6 @@ export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: strin
 
 export const predictDataLimitLength = (mode: `${TrainingModeEnum}`, data: any[]) => {
   if (mode === TrainingModeEnum.qa) return data.length * 20;
+  if (mode === TrainingModeEnum.auto) return data.length * 5;
   return data.length;
 };

packages/global/core/module/type.d.ts

@@ -8,7 +8,7 @@ import {
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 import { UserModelSchema } from 'support/user/type';
 import { moduleDispatchResType } from '..//chat/type';
-import { ChatModuleBillType } from '../../support/wallet/bill/type';
+import { ChatModuleUsageType } from '../../support/wallet/bill/type';
 
 export type FlowModuleTemplateType = {
   id: string; // module id, unique
@@ -129,5 +129,5 @@ export type ModuleDispatchProps<T> = ChatDispatchProps & {
 };
 export type ModuleDispatchResponse<T> = T & {
   [ModuleOutputKeyEnum.responseData]?: moduleDispatchResType;
-  [ModuleOutputKeyEnum.moduleDispatchBills]?: ChatModuleBillType[];
+  [ModuleOutputKeyEnum.moduleDispatchBills]?: ChatModuleUsageType[];
 };

packages/global/support/outLink/type.d.ts

@@ -1,3 +1,4 @@
+import { AppSchema } from 'core/app/type';
 import { OutLinkTypeEnum } from './constant';
 
 export type OutLinkSchema = {
@@ -18,6 +19,9 @@ export type OutLinkSchema = {
     hookUrl?: string;
   };
 };
+export type OutLinkWithAppType = Omit<OutLinkSchema, 'appId'> & {
+  appId: AppSchema;
+};
 
 export type OutLinkEditType = {
   _id?: string;

packages/global/support/permission/chat.d.ts

@@ -0,0 +1,9 @@
+type ShareChatAuthProps = {
+  shareId?: string;
+  outLinkUid?: string;
+};
+type TeamChatAuthProps = {
+  teamId?: string;
+  teamToken?: string;
+};
+export type OutLinkChatAuthProps = ShareChatAuthProps & TeamChatAuthProps;

packages/global/support/permission/constant.ts

@@ -2,7 +2,8 @@ export enum AuthUserTypeEnum {
   token = 'token',
   root = 'root',
   apikey = 'apikey',
-  outLink = 'outLink'
+  outLink = 'outLink',
+  teamDomain = 'teamDomain'
 }
 
 export enum PermissionTypeEnum {

packages/global/support/user/api.d.ts

@@ -10,7 +10,11 @@ export type OauthLoginProps = {
   code: string;
   callbackUrl: string;
   inviterId?: string;
-  tmbId?: string;
+};
+
+export type WxLoginProps = {
+  inviterId?: string;
+  code: string;
 };
 
 export type FastLoginProps = {

packages/global/support/user/auth/constants.ts

@@ -0,0 +1,11 @@
+export enum UserAuthTypeEnum {
+  register = 'register',
+  findPassword = 'findPassword',
+  wxLogin = 'wxLogin'
+}
+
+export const userAuthTypeMap = {
+  [UserAuthTypeEnum.register]: 'register',
+  [UserAuthTypeEnum.findPassword]: 'findPassword',
+  [UserAuthTypeEnum.wxLogin]: 'wxLogin'
+};

packages/global/support/user/constant.ts

@@ -13,10 +13,6 @@ export const userStatusMap = {
 
 export enum OAuthEnum {
   github = 'github',
-  google = 'google'
-}
-
-export enum UserAuthTypeEnum {
-  register = 'register',
-  findPassword = 'findPassword'
+  google = 'google',
+  wechat = 'wechat'
 }

packages/global/support/user/login/api.d.ts

@@ -0,0 +1,4 @@
+export type GetWXLoginQRResponse = {
+  code: string;
+  codeUrl: string;
+};

packages/global/support/user/team/controller.d.ts

@@ -15,7 +15,7 @@ export type UpdateTeamProps = {
   teamId: string;
   name?: string;
   avatar?: string;
-  tagsUrl?: string;
+  teamDomain?: string;
 };
 
 /* ------------- member ----------- */

packages/global/support/user/team/tag.d.ts

@@ -0,0 +1,14 @@
+export type AuthTeamTagTokenProps = {
+  teamId: string;
+  teamToken: string;
+};
+
+export type AuthTokenFromTeamDomainResponse = {
+  success: boolean;
+  msg?: string;
+  message?: string;
+  data: {
+    uid: string;
+    tags: string[];
+  };
+};

packages/global/support/user/team/type.d.ts

@@ -8,22 +8,19 @@ export type TeamSchema = {
   avatar: string;
   createTime: Date;
   balance: number;
-  maxSize: number;
-  tagsUrl: string;
+  teamDomain: string;
   limit: {
     lastExportDatasetTime: Date;
     lastWebsiteSyncTime: Date;
   };
 };
 export type tagsType = {
-  label: string,
-  key: string
-}
-export type TeamTagsSchema = {
-  _id: string;
   label: string;
-  teamId: string;
   key: string;
+};
+export type TeamTagSchema = TeamTagItemType & {
+  _id: string;
+  teamId: string;
   createTime: Date;
 };
 
@@ -56,11 +53,11 @@ export type TeamItemType = {
   avatar: string;
   balance: number;
   tmbId: string;
+  teamDomain: string;
   defaultTeam: boolean;
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
   canWrite: boolean;
-  maxSize: number;
 };
 
 export type TeamMemberItemType = {
@@ -72,3 +69,8 @@ export type TeamMemberItemType = {
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
 };
+
+export type TeamTagItemType = {
+  label: string;
+  key: string;
+};

packages/global/support/wallet/bill/type.d.ts

@@ -21,9 +21,9 @@ export type BillSchemaType = {
   };
 };
 
-export type ChatModuleBillType = {
+export type ChatModuleUsageType = {
+  tokens?: number;
   totalPoints: number;
   moduleName: string;
   model?: string;
-  charsLength?: number;
 };

packages/global/support/wallet/usage/api.d.ts

@@ -20,7 +20,6 @@ export type CreateUsageProps = {
   appName: string;
   appId?: string;
   totalPoints: number;
-  // inputTokens: number;
   source: `${UsageSourceEnum}`;
   list: UsageListItemType[];
 };

packages/global/support/wallet/usage/type.d.ts

@@ -2,6 +2,7 @@ import { CreateUsageProps } from './api';
 import { UsageSourceEnum } from './constants';
 
 export type UsageListItemCountType = {
+  tokens?: number;
   charsLength?: number;
   duration?: number;
 };