perf: queue link

client/.env.template

@@ -1,4 +1,5 @@
 # 运行端口，如果不是 3000 口运行，需要改成其他的。注意：不是改了这个变量就会变成其他端口，而是因为改成其他端口，才用这个变量。
+DB_MAX_LINK=15 # database max link
 PORT=3000
 # 代理
 # AXIOS_PROXY_HOST=127.0.0.1

client/src/pages/api/openapi/kb/pushData.ts

@@ -73,9 +73,6 @@ export async function pushDataToKb({
   const set = new Set();
   const filterData: DateItemType[] = [];
 
-  const time = Date.now();
-  console.log('push data', data.length);
-
   data.forEach((item) => {
     const text = item.q + item.a;
 
@@ -156,8 +153,6 @@ export async function pushDataToKb({
 
   insertData.length > 0 && startQueue();
 
-  console.log('push data finish', Date.now() - time);
-
   return {
     insertLen: insertData.length
   };

client/src/pages/chat/components/ModelList.tsx

@@ -38,9 +38,6 @@ const ModelList = ({ models, modelId }: { models: ModelListItemType[]; modelId:
               <Box className="textEllipsis" color={'myGray.1000'}>
                 {item.name}
               </Box>
-              <Box className="textEllipsis" color={'myGray.400'} fontSize={'sm'}>
-                {item.systemPrompt || '这个 应用 没有设置提示词~'}
-              </Box>
             </Box>
           </Flex>
         </Box>

client/src/service/events/generateQA.ts

@@ -23,34 +23,10 @@ export async function generateQA(): Promise<any> {
   let userId = '';
 
   try {
-    const match = {
-      mode: TrainingModeEnum.qa,
-      lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) }
-    };
-    // random get task
-    const agree = await TrainingData.aggregate([
-      {
-        $match: match
-      },
-      { $sample: { size: 1 } },
-      {
-        $project: {
-          _id: 1
-        }
-      }
-    ]);
-
-    // no task
-    if (agree.length === 0) {
-      reduceQueue();
-      global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`);
-      return;
-    }
-
     const data = await TrainingData.findOneAndUpdate(
       {
-        _id: agree[0]._id,
+        mode: TrainingModeEnum.qa,
-        ...match
+        lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) }
       },
       {
         lockTime: new Date()
@@ -67,7 +43,8 @@ export async function generateQA(): Promise<any> {
     // task preemption
     if (!data) {
       reduceQueue();
-      return generateQA();
+      global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`);
+      return;
     }
 
     trainingId = data._id;

client/src/service/events/generateVector.ts

@@ -19,34 +19,10 @@ export async function generateVector(): Promise<any> {
   let userId = '';
 
   try {
-    const match = {
-      mode: TrainingModeEnum.index,
-      lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
-    };
-    // random get task
-    const agree = await TrainingData.aggregate([
-      {
-        $match: match
-      },
-      { $sample: { size: 1 } },
-      {
-        $project: {
-          _id: 1
-        }
-      }
-    ]);
-
-    // no task
-    if (agree.length === 0) {
-      reduceQueue();
-      global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`);
-      return;
-    }
-
     const data = await TrainingData.findOneAndUpdate(
       {
-        _id: agree[0]._id,
+        mode: TrainingModeEnum.index,
-        ...match
+        lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
       },
       {
         lockTime: new Date()
@@ -63,7 +39,8 @@ export async function generateVector(): Promise<any> {
     // task preemption
     if (!data) {
       reduceQueue();
-      return generateVector();
+      global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`);
+      return;
     }
 
     trainingId = data._id;

client/src/service/mongo.ts

@@ -39,9 +39,9 @@ export async function connectToDatabase(): Promise<void> {
     global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
       bufferCommands: true,
       dbName: process.env.MONGODB_NAME,
-      maxConnecting: 30,
+      maxConnecting: Number(process.env.DB_MAX_LINK || 10),
-      maxPoolSize: 30,
+      maxPoolSize: Number(process.env.DB_MAX_LINK || 10),
-      minPoolSize: 10
+      minPoolSize: 5
     });
     console.log('mongo connected');
   } catch (error) {

client/src/service/pg.ts

@@ -12,7 +12,7 @@ export const connectPg = async () => {
     user: process.env.PG_USER,
     password: process.env.PG_PASSWORD,
     database: process.env.PG_DB_NAME,
-    max: global.systemEnv.vectorMaxProcess + 10,
+    max: Number(process.env.DB_MAX_LINK || 10),
     idleTimeoutMillis: 30000,
     connectionTimeoutMillis: 5000
   });

docs/deploy/fastgpt/docker-compose.yml

@@ -41,6 +41,7 @@ services:
     container_name: fastgpt
     environment: # 可选的变量，不需要的话需要去掉
       - PORT=3000 # 运行的端口地址，如果不是 3000，需要修改成实际地址。
+      - DB_MAX_LINK=15 # database max link
       # proxy（可选）
       - AXIOS_PROXY_HOST=127.0.0.1
       - AXIOS_PROXY_PORT=7890