v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/file/api.d.ts

@@ -1,11 +1,11 @@
 import { MongoImageTypeEnum } from './image/constants';
+import { OutLinkChatAuthProps } from '../../support/permission/chat.d';
 
-export type preUploadImgProps = {
+export type preUploadImgProps = OutLinkChatAuthProps & {
   type: `${MongoImageTypeEnum}`;
 
   expiredTime?: Date;
   metadata?: Record<string, any>;
-  shareId?: string;
 };
 export type UploadImgProps = preUploadImgProps & {
   base64Img: string;

packages/global/common/string/tiktoken/index.ts

@@ -4,6 +4,7 @@ import { Tiktoken } from 'js-tiktoken/lite';
 import { adaptChat2GptMessages } from '../../../core/chat/adapt';
 import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constant';
 import encodingJson from './cl100k_base.json';
+import { ChatMessageItemType } from '../../../core/ai/type';
 
 /* init tikToken obj */
 export function getTikTokenEnc() {
@@ -29,32 +30,35 @@ export function getTikTokenEnc() {
 /* count one prompt tokens */
 export function countPromptTokens(
   prompt = '',
-  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = ''
+  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = '',
+  tools?: any
 ) {
   const enc = getTikTokenEnc();
-  const text = `${role}\n${prompt}`;
+  const toolText = tools
+    ? JSON.stringify(tools)
+        .replace('"', '')
+        .replace('\n', '')
+        .replace(/( ){2,}/g, ' ')
+    : '';
+  const text = `${role}\n${prompt}\n${toolText}`.trim();
 
   try {
     const encodeText = enc.encode(text);
-    return encodeText.length + role.length; // 补充 role 估算值
+    const supplementaryToken = role ? 4 : 0;
+    return encodeText.length + supplementaryToken;
   } catch (error) {
     return text.length;
   }
 }
 
 /* count messages tokens */
-export function countMessagesTokens({ messages }: { messages: ChatItemType[] }) {
+export const countMessagesTokens = (messages: ChatItemType[], tools?: any) => {
   const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
 
-  let totalTokens = 0;
-  for (let i = 0; i < adaptMessages.length; i++) {
-    const item = adaptMessages[i];
-    const tokens = countPromptTokens(item.content, item.role);
-    totalTokens += tokens;
-  }
-
-  return totalTokens;
-}
+  return countGptMessagesTokens(adaptMessages, tools);
+};
+export const countGptMessagesTokens = (messages: ChatMessageItemType[], tools?: any) =>
+  messages.reduce((sum, item) => sum + countPromptTokens(item.content, item.role, tools), 0);
 
 /* slice messages from top to bottom by maxTokens */
 export function sliceMessagesTB({

packages/global/common/system/types/index.d.ts

@@ -39,9 +39,12 @@ export type FastGPTFeConfigsType = {
   systemTitle?: string;
   googleClientVerKey?: string;
   isPlus?: boolean;
+  show_phoneLogin?: boolean;
+  show_emailLogin?: boolean;
   oauth?: {
     github?: string;
     google?: string;
+    wechat?: string;
   };
   limit?: {
     exportDatasetLimitMinutes?: number;