perf: 文本截取

src/pages/api/chat/vectorGpt.ts

@@ -116,8 +116,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       });
     } else {
       // 有匹配情况下，添加知识库内容。
-      // 系统提示词过滤，最多 2800 tokens
-      const systemPrompt = systemPromptFilter(formatRedisPrompt, 2800);
+      // 系统提示词过滤，最多 2000 tokens
+      const systemPrompt = systemPromptFilter(formatRedisPrompt, 2000);
 
       prompts.unshift({
         obj: 'SYSTEM',

src/pages/api/model/data/splitData.ts

@@ -28,10 +28,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       throw new Error('无权操作该模型');
     }
 
-    const replaceText = text.replace(/(\\n|\n)+/g, ' ');
+    const replaceText = text.replace(/\\n/g, '\n');
 
     // 文本拆分成 chunk
-    const chunks = replaceText.match(/[^!?.。]+[!?.。]/g) || [];
+    const chunks = replaceText.split('\n').filter((item) => item.trim());
 
     const textList: string[] = [];
     let splitText = '';

src/pages/api/user/payRes.ts

@@ -1,18 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import axios from 'axios';
-import { connectToDatabase, User, Pay } from '@/service/mongo';
-import { authToken } from '@/service/utils/tools';
-import { PaySchema } from '@/types/mongoSchema';
-import dayjs from 'dayjs';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    res.send('');
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

src/pages/chat/index.tsx

@@ -204,17 +204,6 @@ const Chat = ({ chatId }: { chatId: string }) => {
       return;
     }
 
-    // 长度校验
-    const model = modelList.find((item) => item.model === chatData.modelName);
-
-    if (model && val.length >= model.maxToken) {
-      toast({
-        title: '单次输入超出 4000 字符',
-        status: 'warning'
-      });
-      return;
-    }
-
     const newChatList: ChatSiteItemType[] = [
       ...chatData.history,
       {

src/pages/model/detail/components/SelectFileModal.tsx

@@ -85,7 +85,7 @@ const SelectFileModal = ({
       if (!fileText) return;
       await postModelDataSplitData({
         modelId,
-        text: fileText,
+        text: fileText.replace(/\\n/g, '\n').replace(/\n+/g, '\n'),
         prompt: `下面是${prompt || '一段长文本'}`
       });
       toast({

src/service/utils/tools.ts

@@ -110,12 +110,13 @@ export const openaiChatFilter = (prompts: ChatItemType[], maxTokens: number) =>
   // 从后往前截取
   for (let i = formatPrompts.length - 1; i >= 0; i--) {
     const tokens = encode(formatPrompts[i].value).length;
-    if (maxTokens >= tokens) {
-      res.unshift(formatPrompts[i]);
-      maxTokens -= tokens;
-    } else {
+    res.unshift(formatPrompts[i]);
+    /* 整体 tokens 超出范围 */
+    if (tokens >= maxTokens) {
       break;
     }
+
+    maxTokens -= tokens;
   }
 
   return systemPrompt ? [systemPrompt, ...res] : res;