mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 13:03:50 +00:00
pg index
This commit is contained in:
@@ -16,6 +16,6 @@ CREATE TABLE IF NOT EXISTS modelData (
|
|||||||
-- CREATE INDEX IF NOT EXISTS modelData_userId_index ON modelData USING HASH (user_id);
|
-- CREATE INDEX IF NOT EXISTS modelData_userId_index ON modelData USING HASH (user_id);
|
||||||
-- CREATE INDEX IF NOT EXISTS modelData_kbId_index ON modelData USING HASH (kb_id);
|
-- CREATE INDEX IF NOT EXISTS modelData_kbId_index ON modelData USING HASH (kb_id);
|
||||||
-- CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON modelData (md5(q), md5(a), user_id, kb_id);
|
-- CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON modelData (md5(q), md5(a), user_id, kb_id);
|
||||||
-- CREATE INDEX IF NOT EXISTS vector_index ON modelData USING ivfflat (vector vector_cosine_ops) WITH (lists = 1000);
|
-- CREATE INDEX modeldata_id_desc_idx ON modeldata (id DESC);
|
||||||
-- vector 索引,可以参考 [pg vector](https://github.com/pgvector/pgvector) 去配置,根据数据量去配置
|
-- vector 索引,可以参考 [pg vector](https://github.com/pgvector/pgvector) 去配置,根据数据量去配置
|
||||||
EOSQL
|
EOSQL
|
||||||
|
@@ -113,15 +113,15 @@ export const ModelVectorSearchModeMap: Record<
|
|||||||
> = {
|
> = {
|
||||||
[appVectorSearchModeEnum.hightSimilarity]: {
|
[appVectorSearchModeEnum.hightSimilarity]: {
|
||||||
text: '高相似度, 无匹配时拒绝回复',
|
text: '高相似度, 无匹配时拒绝回复',
|
||||||
similarity: 0.18
|
similarity: 0.8
|
||||||
},
|
},
|
||||||
[appVectorSearchModeEnum.noContext]: {
|
[appVectorSearchModeEnum.noContext]: {
|
||||||
text: '高相似度,无匹配时直接回复',
|
text: '高相似度,无匹配时直接回复',
|
||||||
similarity: 0.18
|
similarity: 0.8
|
||||||
},
|
},
|
||||||
[appVectorSearchModeEnum.lowSimilarity]: {
|
[appVectorSearchModeEnum.lowSimilarity]: {
|
||||||
text: '低相似度匹配',
|
text: '低相似度匹配',
|
||||||
similarity: 0.7
|
similarity: 0.3
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@@ -93,15 +93,15 @@ export async function appKbSearch({
|
|||||||
// search kb
|
// search kb
|
||||||
const res: any = await PgClient.query(
|
const res: any = await PgClient.query(
|
||||||
`BEGIN;
|
`BEGIN;
|
||||||
SET LOCAL ivfflat.probes = ${process.env.PG_IVFFLAT_PROBE || 100};
|
|
||||||
select id,q,a,source from modelData where kb_id IN (${model.chat.relatedKbs
|
select id,q,a,source from modelData where kb_id IN (${model.chat.relatedKbs
|
||||||
.map((item) => `'${item}'`)
|
.map((item) => `'${item}'`)
|
||||||
.join(',')}) AND vector <#> '[${promptVector[0]}]' < ${similarity} order by vector <#> '[${
|
.join(',')}) AND vector <#> '[${promptVector[0]}]' < -${similarity} order by vector <#> '[${
|
||||||
promptVector[0]
|
promptVector[0]
|
||||||
}]' limit 8;
|
}]' limit 8;
|
||||||
COMMIT;`
|
COMMIT;`
|
||||||
);
|
);
|
||||||
const searchRes: QuoteItemType[] = res?.[2]?.rows || [];
|
|
||||||
|
const searchRes: QuoteItemType[] = res?.[1]?.rows || [];
|
||||||
|
|
||||||
// filter same search result
|
// filter same search result
|
||||||
const idSet = new Set<string>();
|
const idSet = new Set<string>();
|
||||||
|
@@ -39,22 +39,25 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
|
|||||||
: [])
|
: [])
|
||||||
];
|
];
|
||||||
|
|
||||||
const searchRes = await PgClient.select<KbDataItemType>('modelData', {
|
const [searchRes, total] = await Promise.all([
|
||||||
fields: ['id', 'q', 'a', 'source'],
|
PgClient.select<KbDataItemType>('modelData', {
|
||||||
where,
|
fields: ['id', 'q', 'a', 'source'],
|
||||||
order: [{ field: 'id', mode: 'DESC' }],
|
where,
|
||||||
limit: pageSize,
|
limit: pageSize,
|
||||||
offset: pageSize * (pageNum - 1)
|
offset: pageSize * (pageNum - 1)
|
||||||
});
|
}),
|
||||||
|
PgClient.count('modelData', {
|
||||||
|
fields: ['id'],
|
||||||
|
where
|
||||||
|
})
|
||||||
|
]);
|
||||||
|
|
||||||
jsonRes(res, {
|
jsonRes(res, {
|
||||||
data: {
|
data: {
|
||||||
pageNum,
|
pageNum,
|
||||||
pageSize,
|
pageSize,
|
||||||
data: searchRes.rows,
|
data: searchRes.rows,
|
||||||
total: await PgClient.count('modelData', {
|
total
|
||||||
where
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
@@ -113,7 +113,7 @@ class Pg {
|
|||||||
return pg.query<T>(sql);
|
return pg.query<T>(sql);
|
||||||
}
|
}
|
||||||
async count(table: string, props: GetProps) {
|
async count(table: string, props: GetProps) {
|
||||||
const sql = `SELECT COUNT(*)
|
const sql = `SELECT COUNT(${props?.fields?.[0] || '*'})
|
||||||
FROM ${table}
|
FROM ${table}
|
||||||
${this.getWhereStr(props.where)}
|
${this.getWhereStr(props.where)}
|
||||||
`;
|
`;
|
||||||
|
@@ -27,8 +27,8 @@ export const authGoogleToken = async (data: {
|
|||||||
`https://www.recaptcha.net/recaptcha/api/siteverify?${Obj2Query(data)}`
|
`https://www.recaptcha.net/recaptcha/api/siteverify?${Obj2Query(data)}`
|
||||||
);
|
);
|
||||||
|
|
||||||
if (res.data.success && res.data.score && res.data.score >= 0.7) {
|
if (res.data.success) {
|
||||||
return Promise.resolve('');
|
return Promise.resolve('');
|
||||||
}
|
}
|
||||||
return Promise.reject(res.data['error-codes'][0] || '非法环境');
|
return Promise.reject(res?.data?.['error-codes']?.[0] || '非法环境');
|
||||||
};
|
};
|
||||||
|
Reference in New Issue
Block a user