ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-03 13:38:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

v4.6 -1 (#459)

Browse Source
This commit is contained in:
Archer
2023-11-09 09:46:57 +08:00
committed by GitHub
parent 661ee79943
commit 8bb5588305
402 changed files with 9899 additions and 5967 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
projects/app/src/service/moduleDispatch/dataset/search.ts
View File

@@ -1,17 +1,11 @@
import { PgClient } from '@/service/pg';
import type { moduleDispatchResType } from '@/types/chat';
import { TaskResponseKeyEnum } from '@/constants/chat';
import { getVector } from '@/pages/api/openapi/plugin/vector';
import { countModelPrice } from '@/service/common/bill/push';
import type { moduleDispatchResType } from '@fastgpt/global/core/chat/type.d';
import { TaskResponseKeyEnum } from '@fastgpt/global/core/chat/constants';
import { countModelPrice } from '@/service/support/wallet/bill/utils';
import type { SelectedDatasetType } from '@/types/core/dataset';
import type {
SearchDataResponseItemType,
SearchDataResultItemType
} from '@fastgpt/global/core/dataset/type';
import { PgDatasetTableName } from '@/constants/plugin';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import type { ModuleDispatchProps } from '@/types/core/chat/type';
import { ModelTypeEnum } from '@/service/core/ai/model';
import { getDatasetDataItemInfo } from '@/pages/api/core/dataset/data/getDataById';
import { searchDatasetData } from '@/service/core/dataset/data/utils';
type DatasetSearchProps = ModuleDispatchProps<{
datasets: SelectedDatasetType;
@@ -28,7 +22,8 @@ export type KBSearchResponse = {
export async function dispatchDatasetSearch(props: Record<string, any>): Promise<KBSearchResponse> {
const {
user,
teamId,
tmbId,
inputs: { datasets = [], similarity = 0.4, limit = 5, userChatInput }
} = props as DatasetSearchProps;
@@ -42,34 +37,15 @@ export async function dispatchDatasetSearch(props: Record<string, any>): Promise
// get vector
const vectorModel = datasets[0]?.vectorModel || global.vectorModels[0];
const { vectors, tokenLen } = await getVector({
const { searchRes, tokenLen } = await searchDatasetData({
text: userChatInput,
model: vectorModel.model,
input: [userChatInput]
similarity,
limit,
datasetIds: datasets.map((item) => item.datasetId)
});
// search kb
const results: any = await PgClient.query(
`BEGIN;
SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 100};
select id, q, a, dataset_id, collection_id, (vector <#> '[${
vectors[0]
}]') * -1 AS score from ${PgDatasetTableName} where user_id='${
user._id
}' AND dataset_id IN (${datasets
.map((item) => `'${item.datasetId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0]
}]' limit ${limit};
COMMIT;`
);
const rows = results?.[2]?.rows as SearchDataResultItemType[];
const collectionsData = await getDatasetDataItemInfo({ pgDataList: rows });
const searchRes: SearchDataResponseItemType[] = collectionsData.map((item, index) => ({
...item,
score: rows[index].score
}));
return {
isEmpty: searchRes.length === 0 ? true : undefined,
unEmpty: searchRes.length > 0 ? true : undefined,