From 00c93a63cd5ac4d71713c491eff8862c9a419665 Mon Sep 17 00:00:00 2001 From: archer <545436317@qq.com> Date: Sat, 17 Jun 2023 21:27:44 +0800 Subject: [PATCH] perf: queue link --- client/.env.template | 1 + client/src/pages/api/openapi/kb/pushData.ts | 5 --- .../src/pages/chat/components/ModelList.tsx | 3 -- client/src/service/events/generateQA.ts | 31 +++---------------- client/src/service/events/generateVector.ts | 31 +++---------------- client/src/service/mongo.ts | 6 ++-- client/src/service/pg.ts | 2 +- docs/deploy/fastgpt/docker-compose.yml | 1 + 8 files changed, 14 insertions(+), 66 deletions(-) diff --git a/client/.env.template b/client/.env.template index 54fc1f659..0ea936abb 100644 --- a/client/.env.template +++ b/client/.env.template @@ -1,4 +1,5 @@ # 运行端口，如果不是 3000 口运行，需要改成其他的。注意：不是改了这个变量就会变成其他端口，而是因为改成其他端口，才用这个变量。 +DB_MAX_LINK=15 # database max link PORT=3000 # 代理 # AXIOS_PROXY_HOST=127.0.0.1 diff --git a/client/src/pages/api/openapi/kb/pushData.ts b/client/src/pages/api/openapi/kb/pushData.ts index 799824ff4..7ca3b8e4f 100644 --- a/client/src/pages/api/openapi/kb/pushData.ts +++ b/client/src/pages/api/openapi/kb/pushData.ts @@ -73,9 +73,6 @@ export async function pushDataToKb({ const set = new Set(); const filterData: DateItemType[] = []; - const time = Date.now(); - console.log('push data', data.length); - data.forEach((item) => { const text = item.q + item.a; @@ -156,8 +153,6 @@ export async function pushDataToKb({ insertData.length > 0 && startQueue(); - console.log('push data finish', Date.now() - time); - return { insertLen: insertData.length }; diff --git a/client/src/pages/chat/components/ModelList.tsx b/client/src/pages/chat/components/ModelList.tsx index e71108f2b..74fdc00e0 100644 --- a/client/src/pages/chat/components/ModelList.tsx +++ b/client/src/pages/chat/components/ModelList.tsx @@ -38,9 +38,6 @@ const ModelList = ({ models, modelId }: { models: ModelListItemType[]; modelId: {item.name} - - {item.systemPrompt || '这个 应用 没有设置提示词~'} - diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts index 2c7fcc9ee..d5d00f9bc 100644 --- a/client/src/service/events/generateQA.ts +++ b/client/src/service/events/generateQA.ts @@ -23,34 +23,10 @@ export async function generateQA(): Promise { let userId = ''; try { - const match = { - mode: TrainingModeEnum.qa, - lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) } - }; - // random get task - const agree = await TrainingData.aggregate([ - { - $match: match - }, - { $sample: { size: 1 } }, - { - $project: { - _id: 1 - } - } - ]); - - // no task - if (agree.length === 0) { - reduceQueue(); - global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`); - return; - } - const data = await TrainingData.findOneAndUpdate( { - _id: agree[0]._id, - ...match + mode: TrainingModeEnum.qa, + lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) } }, { lockTime: new Date() @@ -67,7 +43,8 @@ export async function generateQA(): Promise { // task preemption if (!data) { reduceQueue(); - return generateQA(); + global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`); + return; } trainingId = data._id; diff --git a/client/src/service/events/generateVector.ts b/client/src/service/events/generateVector.ts index 937d3f20e..eafd1ccf2 100644 --- a/client/src/service/events/generateVector.ts +++ b/client/src/service/events/generateVector.ts @@ -19,34 +19,10 @@ export async function generateVector(): Promise { let userId = ''; try { - const match = { - mode: TrainingModeEnum.index, - lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) } - }; - // random get task - const agree = await TrainingData.aggregate([ - { - $match: match - }, - { $sample: { size: 1 } }, - { - $project: { - _id: 1 - } - } - ]); - - // no task - if (agree.length === 0) { - reduceQueue(); - global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`); - return; - } - const data = await TrainingData.findOneAndUpdate( { - _id: agree[0]._id, - ...match + mode: TrainingModeEnum.index, + lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) } }, { lockTime: new Date() @@ -63,7 +39,8 @@ export async function generateVector(): Promise { // task preemption if (!data) { reduceQueue(); - return generateVector(); + global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`); + return; } trainingId = data._id; diff --git a/client/src/service/mongo.ts b/client/src/service/mongo.ts index df32a068b..b95dc2c23 100644 --- a/client/src/service/mongo.ts +++ b/client/src/service/mongo.ts @@ -39,9 +39,9 @@ export async function connectToDatabase(): Promise { global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, { bufferCommands: true, dbName: process.env.MONGODB_NAME, - maxConnecting: 30, - maxPoolSize: 30, - minPoolSize: 10 + maxConnecting: Number(process.env.DB_MAX_LINK || 10), + maxPoolSize: Number(process.env.DB_MAX_LINK || 10), + minPoolSize: 5 }); console.log('mongo connected'); } catch (error) { diff --git a/client/src/service/pg.ts b/client/src/service/pg.ts index e3179552a..36d88c8a8 100644 --- a/client/src/service/pg.ts +++ b/client/src/service/pg.ts @@ -12,7 +12,7 @@ export const connectPg = async () => { user: process.env.PG_USER, password: process.env.PG_PASSWORD, database: process.env.PG_DB_NAME, - max: global.systemEnv.vectorMaxProcess + 10, + max: Number(process.env.DB_MAX_LINK || 10), idleTimeoutMillis: 30000, connectionTimeoutMillis: 5000 }); diff --git a/docs/deploy/fastgpt/docker-compose.yml b/docs/deploy/fastgpt/docker-compose.yml index 672b6b5a5..d16ae066e 100644 --- a/docs/deploy/fastgpt/docker-compose.yml +++ b/docs/deploy/fastgpt/docker-compose.yml @@ -41,6 +41,7 @@ services: container_name: fastgpt environment: # 可选的变量，不需要的话需要去掉 - PORT=3000 # 运行的端口地址，如果不是 3000，需要修改成实际地址。 + - DB_MAX_LINK=15 # database max link # proxy（可选） - AXIOS_PROXY_HOST=127.0.0.1 - AXIOS_PROXY_PORT=7890