perf: ui

client/src/pages/api/openapi/kb/pushData.ts

@@ -7,6 +7,7 @@ import { withNextCors } from '@/service/utils/tools';
 import { TrainingModeEnum } from '@/constants/plugin';
 import { startQueue } from '@/service/utils/tools';
 import { PgClient } from '@/service/pg';
+import { modelToolMap } from '@/utils/plugin';
 
 type DateItemType = { a: string; q: string; source?: string };
 
@@ -21,6 +22,11 @@ export type Response = {
   insertLen: number;
 };
 
+const modeMaxToken = {
+  [TrainingModeEnum.index]: 700,
+  [TrainingModeEnum.qa]: 3300
+};
+
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     const { kbId, data, mode, prompt } = req.body as Props;
@@ -68,7 +74,15 @@ export async function pushDataToKb({
 
   data.forEach((item) => {
     const text = item.q + item.a;
-    if (!set.has(text)) {
+
+    // count token
+    const token = modelToolMap['gpt-3.5-turbo'].countTokens({
+      messages: [{ obj: 'System', value: item.q }]
+    });
+
+    if (mode === TrainingModeEnum.qa && token > modeMaxToken[TrainingModeEnum.qa]) {
+      console.log('q is too long');
+    } else if (!set.has(text)) {
       filterData.push(item);
       set.add(text);
     }