ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-28 00:56:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: queue link

Browse Source
This commit is contained in:
archer
2023-06-17 21:27:44 +08:00
parent 61447c60ac
commit 00c93a63cd
8 changed files with 14 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
client/src/service/events/generateQA.ts
View File

@@ -23,34 +23,10 @@ export async function generateQA(): Promise<any> {
let userId = '';
try {
const match = {
mode: TrainingModeEnum.qa,
lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) }
};
// random get task
const agree = await TrainingData.aggregate([
{
$match: match
},
{ $sample: { size: 1 } },
{
$project: {
_id: 1
}
}
]);
// no task
if (agree.length === 0) {
reduceQueue();
global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`);
return;
}
const data = await TrainingData.findOneAndUpdate(
{
_id: agree[0]._id,
...match
mode: TrainingModeEnum.qa,
lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) }
},
{
lockTime: new Date()
@@ -67,7 +43,8 @@ export async function generateQA(): Promise<any> {
// task preemption
if (!data) {
reduceQueue();
return generateQA();
global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`);
return;
}
trainingId = data._id;

31
client/src/service/events/generateVector.ts
View File

@@ -19,34 +19,10 @@ export async function generateVector(): Promise<any> {
let userId = '';
try {
const match = {
mode: TrainingModeEnum.index,
lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
};
// random get task
const agree = await TrainingData.aggregate([
{
$match: match
},
{ $sample: { size: 1 } },
{
$project: {
_id: 1
}
}
]);
// no task
if (agree.length === 0) {
reduceQueue();
global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`);
return;
}
const data = await TrainingData.findOneAndUpdate(
{
_id: agree[0]._id,
...match
mode: TrainingModeEnum.index,
lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
},
{
lockTime: new Date()
@@ -63,7 +39,8 @@ export async function generateVector(): Promise<any> {
// task preemption
if (!data) {
reduceQueue();
return generateVector();
global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`);
return;
}
trainingId = data._id;

6
client/src/service/mongo.ts
View File

@@ -39,9 +39,9 @@ export async function connectToDatabase(): Promise<void> {
global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
bufferCommands: true,
dbName: process.env.MONGODB_NAME,
maxConnecting: 30,
maxPoolSize: 30,
minPoolSize: 10
maxConnecting: Number(process.env.DB_MAX_LINK || 10),
maxPoolSize: Number(process.env.DB_MAX_LINK || 10),
minPoolSize: 5
});
console.log('mongo connected');
} catch (error) {

2
client/src/service/pg.ts
View File

@@ -12,7 +12,7 @@ export const connectPg = async () => {
user: process.env.PG_USER,
password: process.env.PG_PASSWORD,
database: process.env.PG_DB_NAME,
max: global.systemEnv.vectorMaxProcess + 10,
max: Number(process.env.DB_MAX_LINK || 10),
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 5000
});