ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-22 20:37:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: ery extension and fix filter same embedding result (#3833)

Browse Source 
* perf: ery extension and fix filter same embedding result

* fix: extract node too long

* perf: ui

* perf: not chatId will auto save

* fix: laf

* fix: member load

* feat: add completions unstream error response

* feat: add completions unstream error response

* updat emodel provider
This commit is contained in:
Archer
2025-02-19 22:16:43 +08:00
committed by GitHub
parent 8604cbd021
commit 6762723b10
26 changed files with 317 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
packages/service/core/dataset/search/controller.ts
View File

@@ -383,6 +383,7 @@ export async function searchDatasetData(
).lean()
]);
const set = new Map<string, number>();
const formatResult = results
.map((item, index) => {
const collection = collections.find((col) => String(col._id) === String(item.collectionId));
@@ -398,8 +399,6 @@ export async function searchDatasetData(
return;
}
const score = item?.score || 0;
const result: SearchDataResponseItemType = {
id: String(data._id),
updateTime: data.updateTime,
@@ -409,12 +408,24 @@ export async function searchDatasetData(
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
...getCollectionSourceData(collection),
score: [{ type: SearchScoreTypeEnum.embedding, value: score, index }]
score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }]
};
return result;
})
.filter(Boolean) as SearchDataResponseItemType[];
.filter((item) => {
if (!item) return false;
if (set.has(item.id)) return false;
set.set(item.id, 1);
return true;
})
.map((item, index) => {
if (!item) return;
return {
...item,
score: item.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[];
return {
embeddingRecallResults: formatResult,
@@ -717,11 +728,12 @@ export const defaultSearchDatasetData = async ({
? getLLMModel(datasetSearchExtensionModel)
: undefined;
const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
query,
extensionModel,
extensionBg: datasetSearchExtensionBg
});
const { concatQueries, extensionQueries, rewriteQuery, aiExtensionResult } =
await datasetSearchQueryExtension({
query,
extensionModel,
extensionBg: datasetSearchExtensionBg
});
const result = await searchDatasetData({
...props,
@@ -736,7 +748,7 @@ export const defaultSearchDatasetData = async ({
model: aiExtensionResult.model,
inputTokens: aiExtensionResult.inputTokens,
outputTokens: aiExtensionResult.outputTokens,
query: concatQueries.join('\n')
query: extensionQueries.join('\n')
}
: undefined
};