ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-01 11:58:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: token count

Browse Source
This commit is contained in:
archer
2023-06-14 20:02:43 +08:00
parent 8036ed6143
commit e4aeee7be3
5 changed files with 16 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
client/src/pages/api/openapi/kb/pushData.ts
View File

@@ -76,14 +76,17 @@ export async function pushDataToKb({
data.forEach((item) => {
const text = item.q + item.a;
// count token
const token = modelToolMap[OpenAiChatEnum.GPT35].countTokens({
messages: [{ obj: 'System', value: item.q }]
});
if (mode === TrainingModeEnum.qa) {
// count token
const token = modelToolMap[OpenAiChatEnum.GPT35].countTokens({
messages: [{ obj: 'System', value: item.q }]
});
if (token > modeMaxToken[TrainingModeEnum.qa]) {
return;
}
}
if (mode === TrainingModeEnum.qa && token > modeMaxToken[TrainingModeEnum.qa]) {
console.log('q is too long');
} else if (!set.has(text)) {
if (!set.has(text)) {
filterData.push(item);
set.add(text);
}