v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/service/events/generateQA.ts

@@ -10,8 +10,10 @@ import { Prompt_AgentQA } from '@/global/core/prompt/agent';
 import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
 import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
 import { getLLMModel } from '@fastgpt/service/core/ai/model';
-import { checkInvalidChunkAndLock, checkTeamAiPointsAndLock } from './utils';
-import { countGptMessagesChars } from '@fastgpt/service/core/chat/utils';
+import { checkTeamAiPointsAndLock } from './utils';
+import { checkInvalidChunkAndLock } from '@fastgpt/service/core/dataset/training/utils';
+import { addMinutes } from 'date-fns';
+import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
 
 const reduceQueue = () => {
   global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -20,9 +22,11 @@ const reduceQueue = () => {
 };
 
 export async function generateQA(): Promise<any> {
-  if (global.qaQueueLen >= global.systemEnv.qaMaxProcess) return;
+  const max = global.systemEnv?.qaMaxProcess || 10;
+  if (global.qaQueueLen >= max) return;
   global.qaQueueLen++;
 
+  const startTime = Date.now();
   // get training data
   const {
     data,
@@ -33,7 +37,7 @@ export async function generateQA(): Promise<any> {
     try {
       const data = await MongoDatasetTraining.findOneAndUpdate(
         {
-          lockTime: { $lte: new Date(Date.now() - 6 * 60 * 1000) },
+          lockTime: { $lte: addMinutes(new Date(), -6) },
           mode: TrainingModeEnum.qa
         },
         {
@@ -66,7 +70,7 @@ export async function generateQA(): Promise<any> {
         text: data.q
       };
     } catch (error) {
-      console.log(`Get Training Data error`, error);
+      addLog.error(`[QA Queue] Error`, error);
       return {
         error: true
       };
@@ -75,7 +79,7 @@ export async function generateQA(): Promise<any> {
 
   if (done || !data) {
     if (reduceQueue()) {
-      console.log(`【QA】Task Done`);
+      addLog.info(`[QA Queue] Done`);
     }
     return;
   }
@@ -83,17 +87,15 @@ export async function generateQA(): Promise<any> {
     reduceQueue();
     return generateQA();
   }
-  console.log('Start QA Training');
 
   // auth balance
   if (!(await checkTeamAiPointsAndLock(data.teamId, data.tmbId))) {
-    console.log('balance not enough');
     reduceQueue();
     return generateQA();
   }
+  addLog.info(`[QA Queue] Start`);
 
   try {
-    const startTime = Date.now();
     const model = getLLMModel(data.model)?.model;
     const prompt = `${data.prompt || Prompt_AgentQA.description}
 ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
@@ -119,8 +121,8 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
 
     const qaArr = formatSplitText(answer, text); // 格式化后的QA对
 
-    addLog.info(`QA Training Finish`, {
-      time: `${(Date.now() - startTime) / 1000}s`,
+    addLog.info(`[QA Queue] Finish`, {
+      time: Date.now() - startTime,
       splitLength: qaArr.length,
       usage: chatResponse.usage
     });
@@ -146,7 +148,7 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
       pushQAUsage({
         teamId: data.teamId,
         tmbId: data.tmbId,
-        charsLength: countGptMessagesChars(messages).length,
+        tokens: countGptMessagesTokens(messages),
         billId: data.billId,
         model
       });

projects/app/src/service/events/generateVector.ts

@@ -2,8 +2,10 @@ import { insertData2Dataset } from '@/service/core/dataset/data/controller';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
-import { checkInvalidChunkAndLock, checkTeamAiPointsAndLock } from './utils';
-import { delay } from '@fastgpt/global/common/system/utils';
+import { checkTeamAiPointsAndLock } from './utils';
+import { checkInvalidChunkAndLock } from '@fastgpt/service/core/dataset/training/utils';
+import { addMinutes } from 'date-fns';
+import { addLog } from '@fastgpt/service/common/system/log';
 
 const reduceQueue = () => {
   global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0;
@@ -13,7 +15,8 @@ const reduceQueue = () => {
 
 /* 索引生成队列。每导入一次，就是一个单独的线程 */
 export async function generateVector(): Promise<any> {
-  if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return;
+  const max = global.systemEnv?.vectorMaxProcess || 10;
+  if (global.vectorQueueLen >= max) return;
   global.vectorQueueLen++;
   const start = Date.now();
 
@@ -27,7 +30,7 @@ export async function generateVector(): Promise<any> {
     try {
       const data = await MongoDatasetTraining.findOneAndUpdate(
         {
-          lockTime: { $lte: new Date(Date.now() - 1 * 60 * 1000) },
+          lockTime: { $lte: addMinutes(new Date(), -1) },
           mode: TrainingModeEnum.chunk
         },
         {
@@ -68,7 +71,7 @@ export async function generateVector(): Promise<any> {
         }
       };
     } catch (error) {
-      console.log(`Get Training Data error`, error);
+      addLog.error(`Get Training Data error`, error);
       return {
         error: true
       };
@@ -77,11 +80,12 @@ export async function generateVector(): Promise<any> {
 
   if (done || !data) {
     if (reduceQueue()) {
-      console.log(`【index】Task done`);
+      addLog.info(`[Vector Queue] Done`);
     }
     return;
   }
   if (error) {
+    addLog.error(`[Vector Queue] Error`, { error });
     reduceQueue();
     return generateVector();
   }
@@ -92,6 +96,8 @@ export async function generateVector(): Promise<any> {
     return generateVector();
   }
 
+  addLog.info(`[Vector Queue] Start`);
+
   // create vector and insert
   try {
     // invalid data
@@ -103,7 +109,7 @@ export async function generateVector(): Promise<any> {
     }
 
     // insert to dataset
-    const { charsLength } = await insertData2Dataset({
+    const { tokens } = await insertData2Dataset({
       teamId: data.teamId,
       tmbId: data.tmbId,
       datasetId: data.datasetId,
@@ -119,7 +125,7 @@ export async function generateVector(): Promise<any> {
     pushGenerateVectorUsage({
       teamId: data.teamId,
       tmbId: data.tmbId,
-      charsLength,
+      tokens,
       model: data.model,
       billId: data.billId
     });
@@ -129,7 +135,9 @@ export async function generateVector(): Promise<any> {
     reduceQueue();
     generateVector();
 
-    console.log(`embedding finished, time: ${Date.now() - start}ms`);
+    addLog.info(`[Vector Queue] Finish`, {
+      time: Date.now() - start
+    });
   } catch (err: any) {
     reduceQueue();
 

projects/app/src/service/events/utils.ts

@@ -2,10 +2,6 @@ import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
 import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
 import { sendOneInform } from '../support/user/inform/api';
 import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
-import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
-import { addLog } from '@fastgpt/service/common/system/log';
-import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
-import { getErrText } from '@fastgpt/global/common/error/utils';
 
 export const checkTeamAiPointsAndLock = async (teamId: string, tmbId: string) => {
   try {
@@ -29,41 +25,3 @@ export const checkTeamAiPointsAndLock = async (teamId: string, tmbId: string) =>
     return false;
   }
 };
-
-export const checkInvalidChunkAndLock = async ({
-  err,
-  errText,
-  data
-}: {
-  err: any;
-  errText: string;
-  data: DatasetTrainingSchemaType;
-}) => {
-  if (err?.response) {
-    addLog.info(`openai error: ${errText}`, {
-      status: err.response?.status,
-      stateusText: err.response?.statusText,
-      data: err.response?.data
-    });
-  } else {
-    console.log(err);
-    addLog.error(getErrText(err, errText));
-  }
-
-  if (
-    err?.message === 'invalid message format' ||
-    err?.type === 'invalid_request_error' ||
-    err?.code === 500
-  ) {
-    addLog.info('Lock training data');
-    console.log(err);
-
-    try {
-      await MongoDatasetTraining.findByIdAndUpdate(data._id, {
-        lockTime: new Date('2998/5/5')
-      });
-    } catch (error) {}
-    return true;
-  }
-  return false;
-};