ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-29 17:55:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: chat content use tiktoken count

Browse Source
This commit is contained in:
archer
2023-04-24 16:46:39 +08:00
parent adbaa8b37b
commit 1f112f7715
23 changed files with 182 additions and 836 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/utils/tools.ts
View File

@@ -2,6 +2,7 @@ import crypto from 'crypto';
import { useToast } from '@/hooks/useToast';
import { encoding_for_model, type Tiktoken } from '@dqbd/tiktoken';
import Graphemer from 'graphemer';
import { ChatModelEnum } from '@/constants/model';
const textDecoder = new TextDecoder();
const graphemer = new Graphemer();
@@ -124,7 +125,7 @@ export const countChatTokens = ({
model = 'gpt-3.5-turbo',
messages
}: {
model?: 'gpt-4' | 'gpt-4-32k' | 'gpt-3.5-turbo';
model?: `${ChatModelEnum}`;
messages: { role: 'system' | 'user' | 'assistant'; content: string }[];
}) => {
const text = getChatGPTEncodingText(messages, model);