ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-30 18:48:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: token count

Browse Source
This commit is contained in:
archer
2023-06-14 20:02:43 +08:00
parent 8036ed6143
commit e4aeee7be3
5 changed files with 16 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
client/src/pages/api/openapi/kb/pushData.ts
View File

@@ -76,14 +76,17 @@ export async function pushDataToKb({
data.forEach((item) => {
const text = item.q + item.a;
// count token
const token = modelToolMap[OpenAiChatEnum.GPT35].countTokens({
messages: [{ obj: 'System', value: item.q }]
});
if (mode === TrainingModeEnum.qa) {
// count token
const token = modelToolMap[OpenAiChatEnum.GPT35].countTokens({
messages: [{ obj: 'System', value: item.q }]
});
if (token > modeMaxToken[TrainingModeEnum.qa]) {
return;
}
}
if (mode === TrainingModeEnum.qa && token > modeMaxToken[TrainingModeEnum.qa]) {
console.log('q is too long');
} else if (!set.has(text)) {
if (!set.has(text)) {
filterData.push(item);
set.add(text);
}

1
client/src/pages/kb/components/Detail.tsx
View File

@@ -40,6 +40,7 @@ const Detail = ({ kbId }: { kbId: string }) => {
onSuccess(res) {
kbId && setLastKbId(kbId);
if (res) {
setCurrentTab(TabEnum.data);
reset(res);
BasicInfo.current?.initInput?.(res.tags);
}