perf: ery extension and fix filter same embedding result (#3833)

* perf: ery extension and fix filter same embedding result

* fix: extract node too long

* perf: ui

* perf: not chatId will auto save

* fix: laf

* fix: member load

* feat: add completions unstream error response

* feat: add completions unstream error response

* updat emodel provider
This commit is contained in:
Archer
2025-02-19 22:16:43 +08:00
committed by GitHub
parent 8604cbd021
commit 6762723b10
26 changed files with 317 additions and 257 deletions

View File

@@ -72,11 +72,6 @@ export const ModelProviderList: ModelProviderType[] = [
name: 'Groq',
avatar: 'model/groq'
},
{
id: 'AliCloud',
name: i18nT('common:model_alicloud'),
avatar: 'model/alicloud'
},
{
id: 'Qwen',
name: i18nT('common:model_qwen'),
@@ -87,6 +82,11 @@ export const ModelProviderList: ModelProviderType[] = [
name: i18nT('common:model_doubao'),
avatar: 'model/doubao'
},
{
id: 'DeepSeek',
name: 'DeepSeek',
avatar: 'model/deepseek'
},
{
id: 'ChatGLM',
name: i18nT('common:model_chatglm'),
@@ -97,11 +97,6 @@ export const ModelProviderList: ModelProviderType[] = [
name: i18nT('common:model_ernie'),
avatar: 'model/ernie'
},
{
id: 'DeepSeek',
name: 'DeepSeek',
avatar: 'model/deepseek'
},
{
id: 'Moonshot',
name: i18nT('common:model_moonshot'),
@@ -163,6 +158,11 @@ export const ModelProviderList: ModelProviderType[] = [
name: i18nT('common:model_moka'),
avatar: 'model/moka'
},
{
id: 'AliCloud',
name: i18nT('common:model_alicloud'),
avatar: 'model/alicloud'
},
{
id: 'Siliconflow',
name: i18nT('common:model_siliconflow'),

View File

@@ -1,7 +1,7 @@
import { connectionMongo, getMongoModel } from '../../common/mongo';
const { Schema } = connectionMongo;
import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
import { ChatSourceMap } from '@fastgpt/global/core/chat/constants';
import { ChatSourceEnum, ChatSourceMap } from '@fastgpt/global/core/chat/constants';
import {
TeamCollectionName,
TeamMemberCollectionName
@@ -52,11 +52,10 @@ const ChatSchema = new Schema({
},
source: {
type: String,
required: true
},
sourceName: {
type: String
required: true,
enum: Object.values(ChatSourceEnum)
},
sourceName: String,
shareId: {
type: String
},

View File

@@ -1,6 +1,10 @@
import type { AIChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type.d';
import { MongoApp } from '../app/schema';
import { ChatItemValueTypeEnum, ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import {
ChatItemValueTypeEnum,
ChatRoleEnum,
ChatSourceEnum
} from '@fastgpt/global/core/chat/constants';
import { MongoChatItem } from './chatItemSchema';
import { MongoChat } from './chatSchema';
import { addLog } from '../../common/system/log';
@@ -22,8 +26,8 @@ type Props = {
variables?: Record<string, any>;
isUpdateUseTime: boolean;
newTitle: string;
source: string;
sourceName: string;
source: `${ChatSourceEnum}`;
sourceName?: string;
shareId?: string;
outLinkUid?: string;
content: [UserChatItemType & { dataId?: string }, AIChatItemType & { dataId?: string }];

View File

@@ -383,6 +383,7 @@ export async function searchDatasetData(
).lean()
]);
const set = new Map<string, number>();
const formatResult = results
.map((item, index) => {
const collection = collections.find((col) => String(col._id) === String(item.collectionId));
@@ -398,8 +399,6 @@ export async function searchDatasetData(
return;
}
const score = item?.score || 0;
const result: SearchDataResponseItemType = {
id: String(data._id),
updateTime: data.updateTime,
@@ -409,12 +408,24 @@ export async function searchDatasetData(
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
...getCollectionSourceData(collection),
score: [{ type: SearchScoreTypeEnum.embedding, value: score, index }]
score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }]
};
return result;
})
.filter(Boolean) as SearchDataResponseItemType[];
.filter((item) => {
if (!item) return false;
if (set.has(item.id)) return false;
set.set(item.id, 1);
return true;
})
.map((item, index) => {
if (!item) return;
return {
...item,
score: item.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[];
return {
embeddingRecallResults: formatResult,
@@ -717,11 +728,12 @@ export const defaultSearchDatasetData = async ({
? getLLMModel(datasetSearchExtensionModel)
: undefined;
const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
query,
extensionModel,
extensionBg: datasetSearchExtensionBg
});
const { concatQueries, extensionQueries, rewriteQuery, aiExtensionResult } =
await datasetSearchQueryExtension({
query,
extensionModel,
extensionBg: datasetSearchExtensionBg
});
const result = await searchDatasetData({
...props,
@@ -736,7 +748,7 @@ export const defaultSearchDatasetData = async ({
model: aiExtensionResult.model,
inputTokens: aiExtensionResult.inputTokens,
outputTokens: aiExtensionResult.outputTokens,
query: concatQueries.join('\n')
query: extensionQueries.join('\n')
}
: undefined
};

View File

@@ -72,12 +72,15 @@ Human: ${query}
if (result.extensionQueries?.length === 0) return;
return result;
})();
const extensionQueries = filterSamQuery(aiExtensionResult?.extensionQueries || []);
if (aiExtensionResult) {
queries = filterSamQuery(queries.concat(aiExtensionResult.extensionQueries));
queries = filterSamQuery(queries.concat(extensionQueries));
rewriteQuery = queries.join('\n');
}
return {
extensionQueries,
concatQueries: queries,
rewriteQuery,
aiExtensionResult

View File

@@ -1182,7 +1182,6 @@
"support.wallet.usage.Audio Speech": "Voice Playback",
"support.wallet.usage.Bill Module": "Billing Module",
"support.wallet.usage.Duration": "Duration (seconds)",
"support.wallet.usage.Extension result": "Question Optimization Result",
"support.wallet.usage.Module name": "Module Name",
"support.wallet.usage.Source": "Source",
"support.wallet.usage.Text Length": "Text Length",

View File

@@ -1185,7 +1185,6 @@
"support.wallet.usage.Audio Speech": "语音播放",
"support.wallet.usage.Bill Module": "扣费模块",
"support.wallet.usage.Duration": "时长(秒)",
"support.wallet.usage.Extension result": "问题优化结果",
"support.wallet.usage.Module name": "模块名",
"support.wallet.usage.Source": "来源",
"support.wallet.usage.Text Length": "文本长度",

View File

@@ -1181,7 +1181,6 @@
"support.wallet.usage.Audio Speech": "語音播放",
"support.wallet.usage.Bill Module": "計費模組",
"support.wallet.usage.Duration": "時長(秒)",
"support.wallet.usage.Extension result": "問題最佳化結果",
"support.wallet.usage.Module name": "模組名稱",
"support.wallet.usage.Source": "來源",
"support.wallet.usage.Text Length": "文字長度",