ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 13:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

v4.6.9-alpha (#918)

Browse Source 
Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-03-04 00:05:25 +08:00
committed by GitHub
parent f9f0b4bffd
commit 42a8184ea0
153 changed files with 4906 additions and 4307 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
packages/global/common/string/tiktoken/index.ts
View File

@@ -4,6 +4,7 @@ import { Tiktoken } from 'js-tiktoken/lite';
import { adaptChat2GptMessages } from '../../../core/chat/adapt';
import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constant';
import encodingJson from './cl100k_base.json';
import { ChatMessageItemType } from '../../../core/ai/type';
/* init tikToken obj */
export function getTikTokenEnc() {
@@ -29,32 +30,35 @@ export function getTikTokenEnc() {
/* count one prompt tokens */
export function countPromptTokens(
prompt = '',
role: '' | `${ChatCompletionRequestMessageRoleEnum}` = ''
role: '' | `${ChatCompletionRequestMessageRoleEnum}` = '',
tools?: any
) {
const enc = getTikTokenEnc();
const text = `${role}\n${prompt}`;
const toolText = tools
? JSON.stringify(tools)
.replace('"', '')
.replace('\n', '')
.replace(/( ){2,}/g, ' ')
: '';
const text = `${role}\n${prompt}\n${toolText}`.trim();
try {
const encodeText = enc.encode(text);
return encodeText.length + role.length; // 补充 role 估算值
const supplementaryToken = role ? 4 : 0;
return encodeText.length + supplementaryToken;
} catch (error) {
return text.length;
}
}
/* count messages tokens */
export function countMessagesTokens({ messages }: { messages: ChatItemType[] }) {
export const countMessagesTokens = (messages: ChatItemType[], tools?: any) => {
const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
let totalTokens = 0;
for (let i = 0; i < adaptMessages.length; i++) {
const item = adaptMessages[i];
const tokens = countPromptTokens(item.content, item.role);
totalTokens += tokens;
}
return totalTokens;
}
return countGptMessagesTokens(adaptMessages, tools);
};
export const countGptMessagesTokens = (messages: ChatMessageItemType[], tools?: any) =>
messages.reduce((sum, item) => sum + countPromptTokens(item.content, item.role, tools), 0);
/* slice messages from top to bottom by maxTokens */
export function sliceMessagesTB({