v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/service/core/ai/embedding/index.ts

@@ -1,6 +1,6 @@
 import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../config';
-import { replaceValidChars } from '../../chat/utils';
+import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 
 type GetVectorProps = {
   model: VectorModelItemType;
@@ -37,7 +37,7 @@ export async function getVectorsByText({ model, input }: GetVectorProps) {
         }
 
         return {
-          charsLength: replaceValidChars(input).length,
+          tokens: countPromptTokens(input),
           vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
         };
       });

packages/service/core/ai/functions/createQuestionGuide.ts

@@ -1,6 +1,6 @@
 import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
 import { getAIApi } from '../config';
-import { countGptMessagesChars } from '../../chat/utils';
+import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
 
 export const Prompt_QuestionGuide = `我不太清楚问你什么问题，请帮我生成 3 个问题，引导我继续提问。问题的长度应小于20个字符，按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
 
@@ -34,12 +34,12 @@ export async function createQuestionGuide({
   const start = answer.indexOf('[');
   const end = answer.lastIndexOf(']');
 
-  const charsLength = countGptMessagesChars(concatMessages);
+  const tokens = countGptMessagesTokens(concatMessages);
 
   if (start === -1 || end === -1) {
     return {
       result: [],
-      charsLength: 0
+      tokens: 0
     };
   }
 
@@ -51,12 +51,12 @@ export async function createQuestionGuide({
   try {
     return {
       result: JSON.parse(jsonStr),
-      charsLength
+      tokens
     };
   } catch (error) {
     return {
       result: [],
-      charsLength: 0
+      tokens: 0
     };
   }
 }

packages/service/core/ai/functions/queryExtension.ts

@@ -1,7 +1,7 @@
 import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { getAIApi } from '../config';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
-import { countGptMessagesChars } from '../../chat/utils';
+import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
 
 /* 
     query extension - 问题扩展
@@ -106,7 +106,7 @@ export const queryExtension = async ({
   rawQuery: string;
   extensionQueries: string[];
   model: string;
-  charsLength: number;
+  tokens: number;
 }> => {
   const systemFewShot = chatBg
     ? `Q: 对话背景。
@@ -148,7 +148,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: [],
       model,
-      charsLength: 0
+      tokens: 0
     };
   }
 
@@ -161,7 +161,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: queries,
       model,
-      charsLength: countGptMessagesChars(messages)
+      tokens: countGptMessagesTokens(messages)
     };
   } catch (error) {
     console.log(error);
@@ -169,7 +169,7 @@ A: ${chatBg}
       rawQuery: query,
       extensionQueries: [],
       model,
-      charsLength: 0
+      tokens: 0
     };
   }
 };