perf: insert kb data

This commit is contained in:
archer
2023-05-22 13:16:34 +08:00
parent 51a5d450b7
commit 5128bbcce4
4 changed files with 44 additions and 27 deletions

View File

@@ -4,7 +4,7 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { generateVector } from '@/service/events/generateVector';
import { PgClient } from '@/service/pg';
import { PgClient, insertKbItem } from '@/service/pg';
import { authKb } from '@/service/utils/auth';
import { withNextCors } from '@/service/utils/tools';
@@ -68,14 +68,10 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
.map<{ q: string; a: string }>((item: any) => item.value);
// 插入记录
const insertRes = await PgClient.insert('modelData', {
values: filterData.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'kb_id', value: kbId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: 'waiting' }
])
const insertRes = await insertKbItem({
userId,
kbId,
data: filterData
});
generateVector();

View File

@@ -4,7 +4,7 @@ import { connectToDatabase, SplitData } from '@/service/mongo';
import { authKb, authUser } from '@/service/utils/auth';
import { generateVector } from '@/service/events/generateVector';
import { generateQA } from '@/service/events/generateQA';
import { PgClient } from '@/service/pg';
import { insertKbItem } from '@/service/pg';
import { SplitTextTypEnum } from '@/constants/plugin';
import { withNextCors } from '@/service/utils/tools';
@@ -43,14 +43,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
} else if (mode === SplitTextTypEnum.subsection) {
// 待优化,直接调用另一个接口
// 插入记录
await PgClient.insert('modelData', {
values: chunks.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'kb_id', value: kbId },
{ key: 'q', value: item },
{ key: 'a', value: '' },
{ key: 'status', value: 'waiting' }
])
await insertKbItem({
userId,
kbId,
data: chunks.map((item) => ({
q: item,
a: ''
}))
});
generateVector();

View File

@@ -4,7 +4,7 @@ import { OpenAiChatEnum } from '@/constants/model';
import { pushSplitDataBill } from '@/service/events/pushBill';
import { generateVector } from './generateVector';
import { openaiError2 } from '../errorCode';
import { PgClient } from '@/service/pg';
import { insertKbItem } from '@/service/pg';
import { SplitDataSchema } from '@/types/mongoSchema';
import { modelServiceToolMap } from '../utils/chat';
import { ChatRoleEnum } from '@/constants/chat';
@@ -133,14 +133,10 @@ A2:
textList: dataItem.textList.slice(0, -5)
}),
// 生成的内容插入 pg
PgClient.insert('modelData', {
values: resultList.map((item) => [
{ key: 'user_id', value: dataItem.userId },
{ key: 'kb_id', value: dataItem.kbId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: 'waiting' }
])
insertKbItem({
userId: dataItem.userId,
kbId: dataItem.kbId,
data: resultList
})
]);
console.log('生成QA成功time:', `${(Date.now() - startTime) / 1000}s`);

View File

@@ -156,3 +156,29 @@ class Pg {
}
export const PgClient = new Pg();
/**
* data insert kb
*/
export const insertKbItem = ({
userId,
kbId,
data
}: {
userId: string;
kbId: string;
data: {
q: string;
a: string;
}[];
}) => {
return PgClient.insert('modelData', {
values: data.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'kb_id', value: kbId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: 'waiting' }
])
});
};