perf: token count;feat: chunk size

This commit is contained in:
archer
2023-06-23 15:08:30 +08:00
parent 9aace871ff
commit ae1f7a888e
5 changed files with 85 additions and 148 deletions

View File

@@ -29,7 +29,6 @@
"eventsource-parser": "^0.1.0",
"formidable": "^2.1.1",
"framer-motion": "^9.0.6",
"graphemer": "^1.4.0",
"hyperdown": "^2.4.29",
"immer": "^9.0.19",
"jsonwebtoken": "^9.0.0",

9
client/pnpm-lock.yaml generated
View File

@@ -65,9 +65,6 @@ dependencies:
framer-motion:
specifier: ^9.0.6
version: registry.npmmirror.com/framer-motion@9.0.6(react-dom@18.2.0)(react@18.2.0)
graphemer:
specifier: ^1.4.0
version: registry.npmmirror.com/graphemer@1.4.0
hyperdown:
specifier: ^2.4.29
version: registry.npmmirror.com/hyperdown@2.4.29
@@ -8013,12 +8010,6 @@ packages:
version: 1.0.4
dev: true
registry.npmmirror.com/graphemer@1.4.0:
resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/graphemer/-/graphemer-1.4.0.tgz}
name: graphemer
version: 1.4.0
dev: false
registry.npmmirror.com/has-bigints@1.0.2:
resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/has-bigints/-/has-bigints-1.0.2.tgz}
name: has-bigints

View File

@@ -1,4 +1,4 @@
import React, { useState, useCallback } from 'react';
import React, { useState, useCallback, useRef } from 'react';
import {
Box,
Flex,
@@ -24,24 +24,10 @@ import { TrainingModeEnum } from '@/constants/plugin';
import { getErrText } from '@/utils/tools';
import { ChatModelMap, OpenAiChatEnum, embeddingPrice } from '@/constants/model';
import { formatPrice } from '@/utils/user';
import MySlider from '@/components/Slider';
const fileExtension = '.txt,.doc,.docx,.pdf,.md';
const modeMap = {
[TrainingModeEnum.qa]: {
maxLen: 8000,
slideLen: 3000,
price: ChatModelMap[OpenAiChatEnum.GPT3516k].price,
isPrompt: true
},
[TrainingModeEnum.index]: {
maxLen: 1000,
slideLen: 500,
price: embeddingPrice,
isPrompt: false
}
};
const SelectFileModal = ({
onClose,
onSuccess,
@@ -51,6 +37,16 @@ const SelectFileModal = ({
onSuccess: () => void;
kbId: string;
}) => {
const [modeMap, setModeMap] = useState({
[TrainingModeEnum.qa]: {
maxLen: 8000,
price: ChatModelMap[OpenAiChatEnum.GPT3516k].price
},
[TrainingModeEnum.index]: {
maxLen: 600,
price: embeddingPrice
}
});
const [btnLoading, setBtnLoading] = useState(false);
const { toast } = useToast();
const [prompt, setPrompt] = useState('');
@@ -200,7 +196,7 @@ const SelectFileModal = ({
});
}
setBtnLoading(false);
}, [files, mode, mutate, openConfirm, toast]);
}, [files, mode, modeMap, mutate, openConfirm, toast]);
return (
<Modal isOpen={true} onClose={onClose} isCentered>
@@ -244,19 +240,52 @@ const SelectFileModal = ({
/>
</Flex>
{/* 内容介绍 */}
{modeMap[mode].isPrompt && (
<Flex w={'100%'} px={5} alignItems={'center'} mt={4}>
<Box flex={'0 0 70px'} mr={2}>
</Box>
<Input
placeholder="提示词,例如: Laf的介绍/关于gpt4的论文/一段长文本"
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
size={'sm'}
/>
</Flex>
)}
<Flex w={'100%'} px={5} alignItems={'center'} mt={4}>
{mode === TrainingModeEnum.qa && (
<>
<Box flex={'0 0 70px'} mr={2}>
</Box>
<Input
placeholder="提示词,例如: Laf的介绍/关于gpt4的论文/一段长文本"
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
size={'sm'}
/>
</>
)}
{/* chunk size */}
{mode === TrainingModeEnum.index && (
<Flex w={'100%'} px={5} alignItems={'center'} mt={4}>
<Box w={['70px']} flexShrink={0}>
</Box>
<Box flex={1} ml={'10px'}>
<MySlider
markList={[
{ label: '300', value: 300 },
{ label: '1000', value: 1000 }
]}
width={['100%', '260px']}
min={300}
max={1000}
step={50}
activeVal={modeMap[TrainingModeEnum.index].maxLen}
setVal={(val) => {
setModeMap((state) => ({
...state,
[TrainingModeEnum.index]: {
maxLen: val,
price: embeddingPrice
}
}));
}}
/>
</Box>
</Flex>
)}
</Flex>
{/* 文本内容 */}
<Box flex={'1 0 0'} px={5} h={0} w={'100%'} overflowY={'auto'} mt={4}>
{files.slice(0, 100).map((item, i) => (

View File

@@ -148,15 +148,9 @@ export const fileDownload = ({
* slideLen - The size of the before and after Text
* maxLen > slideLen
*/
export const splitText_token = ({
text,
maxLen,
slideLen
}: {
text: string;
maxLen: number;
slideLen: number;
}) => {
export const splitText_token = ({ text, maxLen }: { text: string; maxLen: number }) => {
const slideLen = Math.floor(maxLen * 0.3);
try {
const enc = getOpenAiEncMap()[OpenAiChatEnum.GPT35];
// filter empty text. encode sentence

View File

@@ -1,68 +1,20 @@
import { encoding_for_model, type Tiktoken } from '@dqbd/tiktoken';
import { encoding_for_model } from '@dqbd/tiktoken';
import type { ChatItemType } from '@/types/chat';
import { ChatRoleEnum } from '@/constants/chat';
import { type ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
import { ChatCompletionRequestMessageRoleEnum } from 'openai';
import { OpenAiChatEnum } from '@/constants/model';
import Graphemer from 'graphemer';
import axios from 'axios';
import dayjs from 'dayjs';
import type { MessageItemType } from '@/pages/api/openapi/v1/chat/completions';
const textDecoder = new TextDecoder();
const graphemer = new Graphemer();
export const getOpenAiEncMap = () => {
if (typeof window !== 'undefined') {
window.OpenAiEncMap = window.OpenAiEncMap || {
[OpenAiChatEnum.GPT35]: encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
[OpenAiChatEnum.GPT3516k]: encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
[OpenAiChatEnum.GPT4]: encoding_for_model('gpt-4', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
[OpenAiChatEnum.GPT432k]: encoding_for_model('gpt-4-32k', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
})
};
if (typeof window !== 'undefined' && window.OpenAiEncMap) {
return window.OpenAiEncMap;
}
if (typeof global !== 'undefined') {
global.OpenAiEncMap = global.OpenAiEncMap || {
[OpenAiChatEnum.GPT35]: encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
[OpenAiChatEnum.GPT3516k]: encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
[OpenAiChatEnum.GPT4]: encoding_for_model('gpt-4', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
}),
[OpenAiChatEnum.GPT432k]: encoding_for_model('gpt-4-32k', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
'<|im_sep|>': 100266
})
};
if (typeof global !== 'undefined' && global.OpenAiEncMap) {
return global.OpenAiEncMap;
}
return {
const enc = {
[OpenAiChatEnum.GPT35]: encoding_for_model('gpt-3.5-turbo', {
'<|im_start|>': 100264,
'<|im_end|>': 100265,
@@ -84,6 +36,15 @@ export const getOpenAiEncMap = () => {
'<|im_sep|>': 100266
})
};
if (typeof window !== 'undefined') {
window.OpenAiEncMap = enc;
}
if (typeof global !== 'undefined') {
global.OpenAiEncMap = enc;
}
return enc;
};
export const adaptChatItem_openAI = ({
@@ -112,55 +73,18 @@ export function countOpenAIToken({
messages: ChatItemType[];
model: `${OpenAiChatEnum}`;
}) {
function getChatGPTEncodingText(
messages: ChatCompletionRequestMessage[],
model: `${OpenAiChatEnum}`
) {
const isGpt3 = model.startsWith('gpt-3.5-turbo');
const msgSep = isGpt3 ? '\n' : '';
const roleSep = isGpt3 ? '\n' : '<|im_sep|>';
return [
messages
.map(({ name = '', role, content }) => {
return `<|im_start|>${name || role}${roleSep}${content}<|im_end|>`;
})
.join(msgSep),
`<|im_start|>assistant${roleSep}`
].join(msgSep);
}
function text2TokensLen(encoder: Tiktoken, inputText: string) {
const encoding = encoder.encode(inputText, 'all');
const segments: { text: string; tokens: { id: number; idx: number }[] }[] = [];
let byteAcc: number[] = [];
let tokenAcc: { id: number; idx: number }[] = [];
let inputGraphemes = graphemer.splitGraphemes(inputText);
for (let idx = 0; idx < encoding.length; idx++) {
const token = encoding[idx]!;
byteAcc.push(...encoder.decode_single_token_bytes(token));
tokenAcc.push({ id: token, idx });
const segmentText = textDecoder.decode(new Uint8Array(byteAcc));
const graphemes = graphemer.splitGraphemes(segmentText);
if (graphemes.every((item, idx) => inputGraphemes[idx] === item)) {
segments.push({ text: segmentText, tokens: tokenAcc });
byteAcc = [];
tokenAcc = [];
inputGraphemes = inputGraphemes.slice(graphemes.length);
}
}
return segments.reduce((memo, i) => memo + i.tokens.length, 0) ?? 0;
}
const diffVal = model.startsWith('gpt-3.5-turbo') ? 3 : 2;
const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true });
const token = adaptMessages.reduce((sum, item) => {
const text = `${item.role}\n${item.content}`;
const enc = getOpenAiEncMap()[model];
const encodeText = enc.encode(text);
const tokens = encodeText.length + diffVal;
return sum + tokens;
}, 0);
return text2TokensLen(getOpenAiEncMap()[model], getChatGPTEncodingText(adaptMessages, model));
return token;
}
export const openAiSliceTextByToken = ({