ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-24 13:53:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

pg table name

Browse Source
This commit is contained in:
archer
2023-08-14 10:36:15 +08:00
parent c5fd5706a1
commit 3a49efd46d
12 changed files with 37 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/constants/plugin.ts
View File

@@ -6,3 +6,5 @@ export const TrainingTypeMap = {
[TrainingModeEnum.qa]: 'qa',
[TrainingModeEnum.index]: 'index'
};
export const PgTrainingTableName = 'modeldata';

3
client/src/pages/api/openapi/kb/delDataById.ts
View File

@@ -3,6 +3,7 @@ import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools';
import { PgTrainingTableName } from '@/constants/plugin';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -17,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// 凭证校验
const { userId } = await authUser({ req });
await PgClient.delete('modelData', {
await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['id', dataId]]
});

4
client/src/pages/api/openapi/kb/pushData.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authKb } from '@/service/utils/auth';
import { withNextCors } from '@/service/utils/tools';
import { TrainingModeEnum } from '@/constants/plugin';
import { PgTrainingTableName, TrainingModeEnum } from '@/constants/plugin';
import { startQueue } from '@/service/utils/tools';
import { PgClient } from '@/service/pg';
import { modelToolMap } from '@/utils/plugin';
@@ -129,7 +129,7 @@ export async function pushDataToKb({
try {
const { rows } = await PgClient.query(`
SELECT COUNT(*) > 0 AS exists
FROM modelData
FROM ${PgTrainingTableName}
WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}'
`);
const exists = rows[0]?.exists || false;

3
client/src/pages/api/openapi/kb/searchTest.ts
View File

@@ -5,6 +5,7 @@ import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools';
import { getVector } from '../plugin/vector';
import type { KbTestItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
export type Props = {
model: string;
@@ -39,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
select id,q,a,source,(vector <#> '[${
vectors[0]
}]') * -1 AS score from modelData where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
}]') * -1 AS score from ${PgTrainingTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
vectors[0]
}]' limit 12;
COMMIT;`

3
client/src/pages/api/openapi/kb/updateData.ts
View File

@@ -5,6 +5,7 @@ import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools';
import { KB, connectToDatabase } from '@/service/mongo';
import { getVector } from '../plugin/vector';
import { PgTrainingTableName } from '@/constants/plugin';
export type Props = {
dataId: string;
@@ -46,7 +47,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
})();
// 更新 pg 内容.仅修改a，不需要更新向量。
await PgClient.update('modelData', {
await PgClient.update(PgTrainingTableName, {
where: [['id', dataId], 'AND', ['user_id', userId]],
values: [
{ key: 'source', value: '手动修改' },

18
client/src/pages/api/plugins/kb/data/exportModelData.ts
View File

@@ -3,6 +3,7 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -38,16 +39,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 统计数据
const count = await PgClient.count('modelData', {
const count = await PgClient.count(PgTrainingTableName, {
where: [['kb_id', kbId], 'AND', ['user_id', userId]]
});
// 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string; source: string }>('modelData', {
where: [['kb_id', kbId], 'AND', ['user_id', userId]],
fields: ['q', 'a', 'source'],
order: [{ field: 'id', mode: 'DESC' }],
limit: count
});
const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
PgTrainingTableName,
{
where: [['kb_id', kbId], 'AND', ['user_id', userId]],
fields: ['q', 'a', 'source'],
order: [{ field: 'id', mode: 'DESC' }],
limit: count
}
);
const data: [string, string, string][] = pgData.rows.map((item) => [
item.q.replace(/\n/g, '\\n'),

3
client/src/pages/api/plugins/kb/data/getDataById.ts
View File

@@ -4,6 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
export type Response = {
id: string;
@@ -28,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const where: any = [['user_id', userId], 'AND', ['id', dataId]];
const searchRes = await PgClient.select<KbDataItemType>('modelData', {
const searchRes = await PgClient.select<KbDataItemType>(PgTrainingTableName, {
fields: ['kb_id', 'id', 'q', 'a', 'source'],
where,
limit: 1

5
client/src/pages/api/plugins/kb/data/getDataList.ts
View File

@@ -4,6 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -41,14 +42,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
];
const [searchRes, total] = await Promise.all([
PgClient.select<KbDataItemType>('modelData', {
PgClient.select<KbDataItemType>(PgTrainingTableName, {
fields: ['id', 'q', 'a', 'source'],
where,
order: [{ field: 'id', mode: 'DESC' }],
limit: pageSize,
offset: pageSize * (pageNum - 1)
}),
PgClient.count('modelData', {
PgClient.count(PgTrainingTableName, {
fields: ['id'],
where
})

3
client/src/pages/api/plugins/kb/delete.ts
View File

@@ -4,6 +4,7 @@ import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { Types } from 'mongoose';
import { PgTrainingTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -21,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase();
// delete all pg data
await PgClient.delete('modelData', {
await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', id]]
});

3
client/src/service/moduleDispatch/kb/search.ts
View File

@@ -5,6 +5,7 @@ import { getVector } from '@/pages/api/openapi/plugin/vector';
import { countModelPrice } from '@/service/events/pushBill';
import type { SelectedKbType } from '@/types/plugin';
import type { QuoteItemType } from '@/types/chat';
import { PgTrainingTableName } from '@/constants/plugin';
type KBSearchProps = {
kbList: SelectedKbType;
@@ -48,7 +49,7 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
const res: any = await PgClient.query(
`BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
select kb_id,id,q,a,source from modelData where kb_id IN (${kbList
select kb_id,id,q,a,source from ${PgTrainingTableName} where kb_id IN (${kbList
.map((item) => `'${item.kbId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0]

9
client/src/service/mongo.ts
View File

@@ -6,6 +6,7 @@ import { User } from './models/user';
import { PRICE_SCALE } from '@/constants/common';
import { connectPg, PgClient } from './pg';
import { createHashPassword } from '@/utils/tools';
import { PgTrainingTableName } from '@/constants/plugin';
/**
* connect MongoDB and init data
@@ -92,7 +93,7 @@ async function initPg() {
await connectPg();
await PgClient.query(`
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE IF NOT EXISTS modelData (
CREATE TABLE IF NOT EXISTS ${PgTrainingTableName} (
id BIGSERIAL PRIMARY KEY,
vector VECTOR(1536) NOT NULL,
user_id VARCHAR(50) NOT NULL,
@@ -101,9 +102,9 @@ async function initPg() {
q TEXT NOT NULL,
a TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS modelData_userId_index ON modelData USING HASH (user_id);
CREATE INDEX IF NOT EXISTS modelData_kbId_index ON modelData USING HASH (kb_id);
CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON modelData (md5(q), md5(a), user_id, kb_id);
CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id);
CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id);
CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgTrainingTableName} (md5(q), md5(a), user_id, kb_id);
`);
console.log('init pg successful');
} catch (error) {

3
client/src/service/pg.ts
View File

@@ -1,5 +1,6 @@
import { Pool } from 'pg';
import type { QueryResultRow } from 'pg';
import { PgTrainingTableName } from '@/constants/plugin';
export const connectPg = async () => {
if (global.pgClient) {
@@ -173,7 +174,7 @@ export const insertKbItem = ({
source?: string;
}[];
}) => {
return PgClient.insert('modelData', {
return PgClient.insert(PgTrainingTableName, {
values: data.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'kb_id', value: kbId },