System optimize (#303)

client/src/pages/api/core/dataset/file/list.ts

@@ -44,6 +44,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         })
         .skip((pageNum - 1) * pageSize)
         .limit(pageSize)
+        .sort({ uploadDate: -1 })
         .toArray(),
       collection.countDocuments(mongoWhere)
     ]);

client/src/pages/api/openapi/kb/pushData.ts

@@ -7,9 +7,9 @@ import { withNextCors } from '@/service/utils/tools';
 import { PgDatasetTableName, TrainingModeEnum } from '@/constants/plugin';
 import { startQueue } from '@/service/utils/tools';
 import { PgClient } from '@/service/pg';
-import { modelToolMap } from '@/utils/plugin';
 import { getVectorModel } from '@/service/utils/data';
 import { DatasetItemType } from '@/types/plugin';
+import { countPromptTokens } from '@/utils/common/tiktoken';
 
 export type Props = {
   kbId: string;
@@ -102,9 +102,7 @@ export async function pushDataToKb({
     const text = item.q + item.a;
 
     // count q token
-    const token = modelToolMap.countTokens({
-      messages: [{ obj: 'System', value: item.q }]
-    });
+    const token = countPromptTokens(item.q, 'system');
 
     if (token > modeMaxToken[mode]) {
       return;

client/src/pages/api/openapi/text/gptMessagesSlice.ts

@@ -1,61 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
-import type { ChatItemType } from '@/types/chat';
-import { countOpenAIToken } from '@/utils/plugin/openai';
-
-type Props = {
-  messages: ChatItemType[];
-  model: string;
-  maxLen: number;
-};
-type Response = ChatItemType[];
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await authUser({ req });
-
-    const { messages, model, maxLen } = req.body as Props;
-
-    if (!Array.isArray(messages) || !model || !maxLen) {
-      throw new Error('params is error');
-    }
-
-    return jsonRes<Response>(res, {
-      data: gpt_chatItemTokenSlice({
-        messages,
-        maxToken: maxLen
-      })
-    });
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}
-
-export function gpt_chatItemTokenSlice({
-  messages,
-  maxToken
-}: {
-  messages: ChatItemType[];
-  maxToken: number;
-}) {
-  let result: ChatItemType[] = [];
-
-  for (let i = 0; i < messages.length; i++) {
-    const msgs = [...result, messages[i]];
-
-    const tokens = countOpenAIToken({ messages: msgs });
-
-    if (tokens < maxToken) {
-      result = msgs;
-    } else {
-      break;
-    }
-  }
-
-  return result.length === 0 && messages[0] ? [messages[0]] : result;
-}

client/src/pages/api/openapi/v1/chat/completions.ts

@@ -79,6 +79,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     if (!Array.isArray(messages)) {
       throw new Error('messages is not array');
     }
+    if (messages.length === 0) {
+      throw new Error('messages is empty');
+    }
 
     await connectToDatabase();
     let startTime = Date.now();
@@ -120,7 +123,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     responseDetail = isOwner || responseDetail;
 
     const prompts = history.concat(gptMessage2ChatType(messages));
-    if (prompts[prompts.length - 1].obj === 'AI') {
+    if (prompts[prompts.length - 1]?.obj === 'AI') {
       prompts.pop();
     }
     // user question

client/src/pages/api/plugins/kb/data/insertData.ts

@@ -5,10 +5,10 @@ import { authKb, authUser } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { insertKbItem, PgClient } from '@/service/pg';
-import { modelToolMap } from '@/utils/plugin';
 import { getVectorModel } from '@/service/utils/data';
 import { getVector } from '@/pages/api/openapi/plugin/vector';
 import { DatasetItemType } from '@/types/plugin';
+import { countPromptTokens } from '@/utils/common/tiktoken';
 
 export type Props = {
   kbId: string;
@@ -35,9 +35,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     const a = data?.a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
 
     // token check
-    const token = modelToolMap.countTokens({
-      messages: [{ obj: 'System', value: q }]
-    });
+    const token = countPromptTokens(q, 'system');
 
     if (token > getVectorModel(kb.vectorModel).maxToken) {
       throw new Error('Over Tokens');