ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-29 17:55:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: model share market

Browse Source
This commit is contained in:
archer
2023-04-27 23:41:42 +08:00
parent 46eb96c72e
commit 3b8e5d2738
34 changed files with 574 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/utils/tools.ts
View File

@@ -6,22 +6,27 @@ import { ChatModelEnum } from '@/constants/model';
const textDecoder = new TextDecoder();
const graphemer = new Graphemer();
const encMap = {
'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4': encoding_for_model('gpt-4', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4-32k': encoding_for_model('gpt-4-32k', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
})
let encMap: Record<string, Tiktoken>;
const getEncMap = () => {
if (encMap) return encMap;
encMap = {
'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4': encoding_for_model('gpt-4', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
'gpt-4-32k': encoding_for_model('gpt-4-32k', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
})
};
return encMap;
};
/**
@@ -129,5 +134,5 @@ export const countChatTokens = ({
messages: { role: 'system' | 'user' | 'assistant'; content: string }[];
}) => {
const text = getChatGPTEncodingText(messages, model);
return text2TokensLen(encMap[model], text);
return text2TokensLen(getEncMap()[model], text);
};