ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-10-20 10:45:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

pg table name

Browse Source
This commit is contained in:
archer
2023-08-14 10:36:15 +08:00
parent c5fd5706a1
commit 3a49efd46d
12 changed files with 37 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client/src/constants/plugin.ts
View File

@@ -6,3 +6,5 @@ export const TrainingTypeMap = {
[TrainingModeEnum.qa]: 'qa', [TrainingModeEnum.qa]: 'qa',
[TrainingModeEnum.index]: 'index' [TrainingModeEnum.index]: 'index'
}; };
export const PgTrainingTableName = 'modeldata';

3
client/src/pages/api/openapi/kb/delDataById.ts
View File

@@ -3,6 +3,7 @@ import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { PgTrainingTableName } from '@/constants/plugin';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
@@ -17,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req }); const { userId } = await authUser({ req });
await PgClient.delete('modelData', { await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['id', dataId]] where: [['user_id', userId], 'AND', ['id', dataId]]
}); });

4
client/src/pages/api/openapi/kb/pushData.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { authKb } from '@/service/utils/auth'; import { authKb } from '@/service/utils/auth';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { TrainingModeEnum } from '@/constants/plugin'; import { PgTrainingTableName, TrainingModeEnum } from '@/constants/plugin';
import { startQueue } from '@/service/utils/tools'; import { startQueue } from '@/service/utils/tools';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { modelToolMap } from '@/utils/plugin'; import { modelToolMap } from '@/utils/plugin';
@@ -129,7 +129,7 @@ export async function pushDataToKb({
try { try {
const { rows } = await PgClient.query(` const { rows } = await PgClient.query(`
SELECT COUNT(*) > 0 AS exists SELECT COUNT(*) > 0 AS exists
FROM modelData FROM ${PgTrainingTableName}
WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}' WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}'
`); `);
const exists = rows[0]?.exists || false; const exists = rows[0]?.exists || false;

3
client/src/pages/api/openapi/kb/searchTest.ts
View File

@@ -5,6 +5,7 @@ import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { getVector } from '../plugin/vector'; import { getVector } from '../plugin/vector';
import type { KbTestItemType } from '@/types/plugin'; import type { KbTestItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
export type Props = { export type Props = {
model: string; model: string;
@@ -39,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10}; SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
select id,q,a,source,(vector <#> '[${ select id,q,a,source,(vector <#> '[${
vectors[0] vectors[0]
}]') * -1 AS score from modelData where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${ }]') * -1 AS score from ${PgTrainingTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
vectors[0] vectors[0]
}]' limit 12; }]' limit 12;
COMMIT;` COMMIT;`

3
client/src/pages/api/openapi/kb/updateData.ts
View File

@@ -5,6 +5,7 @@ import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { KB, connectToDatabase } from '@/service/mongo'; import { KB, connectToDatabase } from '@/service/mongo';
import { getVector } from '../plugin/vector'; import { getVector } from '../plugin/vector';
import { PgTrainingTableName } from '@/constants/plugin';
export type Props = { export type Props = {
dataId: string; dataId: string;
@@ -46,7 +47,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
})(); })();
// 更新 pg 内容.仅修改a，不需要更新向量。 // 更新 pg 内容.仅修改a，不需要更新向量。
await PgClient.update('modelData', { await PgClient.update(PgTrainingTableName, {
where: [['id', dataId], 'AND', ['user_id', userId]], where: [['id', dataId], 'AND', ['user_id', userId]],
values: [ values: [
{ key: 'source', value: '手动修改' }, { key: 'source', value: '手动修改' },

18
client/src/pages/api/plugins/kb/data/exportModelData.ts
View File

@@ -3,6 +3,7 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, User } from '@/service/mongo'; import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
@@ -38,16 +39,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
} }
// 统计数据 // 统计数据
const count = await PgClient.count('modelData', { const count = await PgClient.count(PgTrainingTableName, {
where: [['kb_id', kbId], 'AND', ['user_id', userId]] where: [['kb_id', kbId], 'AND', ['user_id', userId]]
}); });
// 从 pg 中获取所有数据 // 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string; source: string }>('modelData', { const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
where: [['kb_id', kbId], 'AND', ['user_id', userId]], PgTrainingTableName,
fields: ['q', 'a', 'source'], {
order: [{ field: 'id', mode: 'DESC' }], where: [['kb_id', kbId], 'AND', ['user_id', userId]],
limit: count fields: ['q', 'a', 'source'],
}); order: [{ field: 'id', mode: 'DESC' }],
limit: count
}
);
const data: [string, string, string][] = pgData.rows.map((item) => [ const data: [string, string, string][] = pgData.rows.map((item) => [
item.q.replace(/\n/g, '\\n'), item.q.replace(/\n/g, '\\n'),

3
client/src/pages/api/plugins/kb/data/getDataById.ts
View File

@@ -4,6 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin'; import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
export type Response = { export type Response = {
id: string; id: string;
@@ -28,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const where: any = [['user_id', userId], 'AND', ['id', dataId]]; const where: any = [['user_id', userId], 'AND', ['id', dataId]];
const searchRes = await PgClient.select<KbDataItemType>('modelData', { const searchRes = await PgClient.select<KbDataItemType>(PgTrainingTableName, {
fields: ['kb_id', 'id', 'q', 'a', 'source'], fields: ['kb_id', 'id', 'q', 'a', 'source'],
where, where,
limit: 1 limit: 1

5
client/src/pages/api/plugins/kb/data/getDataList.ts
View File

@@ -4,6 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin'; import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
@@ -41,14 +42,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
]; ];
const [searchRes, total] = await Promise.all([ const [searchRes, total] = await Promise.all([
PgClient.select<KbDataItemType>('modelData', { PgClient.select<KbDataItemType>(PgTrainingTableName, {
fields: ['id', 'q', 'a', 'source'], fields: ['id', 'q', 'a', 'source'],
where, where,
order: [{ field: 'id', mode: 'DESC' }], order: [{ field: 'id', mode: 'DESC' }],
limit: pageSize, limit: pageSize,
offset: pageSize * (pageNum - 1) offset: pageSize * (pageNum - 1)
}), }),
PgClient.count('modelData', { PgClient.count(PgTrainingTableName, {
fields: ['id'], fields: ['id'],
where where
}) })

3
client/src/pages/api/plugins/kb/delete.ts
View File

@@ -4,6 +4,7 @@ import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { Types } from 'mongoose'; import { Types } from 'mongoose';
import { PgTrainingTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
@@ -21,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase(); await connectToDatabase();
// delete all pg data // delete all pg data
await PgClient.delete('modelData', { await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', id]] where: [['user_id', userId], 'AND', ['kb_id', id]]
}); });

3
client/src/service/moduleDispatch/kb/search.ts
View File

@@ -5,6 +5,7 @@ import { getVector } from '@/pages/api/openapi/plugin/vector';
import { countModelPrice } from '@/service/events/pushBill'; import { countModelPrice } from '@/service/events/pushBill';
import type { SelectedKbType } from '@/types/plugin'; import type { SelectedKbType } from '@/types/plugin';
import type { QuoteItemType } from '@/types/chat'; import type { QuoteItemType } from '@/types/chat';
import { PgTrainingTableName } from '@/constants/plugin';
type KBSearchProps = { type KBSearchProps = {
kbList: SelectedKbType; kbList: SelectedKbType;
@@ -48,7 +49,7 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
const res: any = await PgClient.query( const res: any = await PgClient.query(
`BEGIN; `BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10}; SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
select kb_id,id,q,a,source from modelData where kb_id IN (${kbList select kb_id,id,q,a,source from ${PgTrainingTableName} where kb_id IN (${kbList
.map((item) => `'${item.kbId}'`) .map((item) => `'${item.kbId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${ .join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0] vectors[0]

9
client/src/service/mongo.ts
View File

@@ -6,6 +6,7 @@ import { User } from './models/user';
import { PRICE_SCALE } from '@/constants/common'; import { PRICE_SCALE } from '@/constants/common';
import { connectPg, PgClient } from './pg'; import { connectPg, PgClient } from './pg';
import { createHashPassword } from '@/utils/tools'; import { createHashPassword } from '@/utils/tools';
import { PgTrainingTableName } from '@/constants/plugin';
/** /**
* connect MongoDB and init data * connect MongoDB and init data
@@ -92,7 +93,7 @@ async function initPg() {
await connectPg(); await connectPg();
await PgClient.query(` await PgClient.query(`
CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE IF NOT EXISTS modelData ( CREATE TABLE IF NOT EXISTS ${PgTrainingTableName} (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
vector VECTOR(1536) NOT NULL, vector VECTOR(1536) NOT NULL,
user_id VARCHAR(50) NOT NULL, user_id VARCHAR(50) NOT NULL,
@@ -101,9 +102,9 @@ async function initPg() {
q TEXT NOT NULL, q TEXT NOT NULL,
a TEXT NOT NULL a TEXT NOT NULL
); );
CREATE INDEX IF NOT EXISTS modelData_userId_index ON modelData USING HASH (user_id); CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id);
CREATE INDEX IF NOT EXISTS modelData_kbId_index ON modelData USING HASH (kb_id); CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id);
CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON modelData (md5(q), md5(a), user_id, kb_id); CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgTrainingTableName} (md5(q), md5(a), user_id, kb_id);
`); `);
console.log('init pg successful'); console.log('init pg successful');
} catch (error) { } catch (error) {

3
client/src/service/pg.ts
View File

@@ -1,5 +1,6 @@
import { Pool } from 'pg'; import { Pool } from 'pg';
import type { QueryResultRow } from 'pg'; import type { QueryResultRow } from 'pg';
import { PgTrainingTableName } from '@/constants/plugin';
export const connectPg = async () => { export const connectPg = async () => {
if (global.pgClient) { if (global.pgClient) {
@@ -173,7 +174,7 @@ export const insertKbItem = ({
source?: string; source?: string;
}[]; }[];
}) => { }) => {
return PgClient.insert('modelData', { return PgClient.insert(PgTrainingTableName, {
values: data.map((item) => [ values: data.map((item) => [
{ key: 'user_id', value: userId }, { key: 'user_id', value: userId },
{ key: 'kb_id', value: kbId }, { key: 'kb_id', value: kbId },