V4.9.1 feature (#4206)

* fix: remove DefaultTeam (#4037) * fix ：Get application bound knowledge base information logical rewrite (#4057) * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * update package * fix: import dataset step error;perf: ai proxy avatar (#4074) * perf: pg config params * perf: ai proxy avatar * fix: import dataset step error * feat: data input ux * perf: app dataset rewite * fix: 文本提取不支持arrayString,arrayNumber等jsonSchema (#4079) * update doc ;perf: model test (#4098) * perf: extract array * update doc * perf: model test * perf: model test * perf: think tag parse (#4102) * chat quote reader (#3912) * init chat quote full text reader * linked structure * dataset data linked * optimize code * fix ts build * test finish * delete log * fix * fix ts * fix ts * remove nextId * initial scroll * fix * fix * perf: chunk read (#4109) * package * perf: chunk read * feat: api dataset support pdf parse;fix: chunk reader auth (#4117) * feat: api dataset support pdf parse * fix: chunk reader auth * feat: invitation link (#3979) * feat: invitation link schema and apis * feat: add invitation link * feat: member status: active, leave, forbidden * fix: expires show hours and minutes * feat: invalid invitation link hint * fix: typo * chore: fix typo & i18n * fix * pref: fe * feat: add ttl index for 30-day-clean-up * perf: invite member code (#4118) * perf: invite member code * fix: ts * fix: model test channel id;fix: quote reader (#4123) * fix: model test channel id * fix: quote reader * fix chat quote reader (#4125) * perf: model test;perf: sidebar trigger (#4127) * fix: import dataset step error;perf: ai proxy avatar (#4074) * perf: pg config params * perf: ai proxy avatar * fix: import dataset step error * feat: data input ux * perf: app dataset rewite * perf: model test * perf: sidebar trigger * lock * update nanoid version * fix: select component ux * fix: ts * fix: vitest * remove test * fix: prompt toolcall ui (#4139) * load log error adapt * fix: prompt toolcall ui * perf: commercial function tip * update package * pref: copy link (#4147) * fix(i18n): namespace (#4143) * hiden dataset source (#4152) * hiden dataset source * perf: reader * chore: move all tests into a single folder (#4160) * fix modal close scroll (#4162) * fix modal close scroll * update refresh * feat: rerank modal select and weight (#4164) * fix loadInitData refresh (#4169) * fix * fix * form input number default & api dataset max token * feat: mix search weight (#4170) * feat: mix search weight * feat: svg render * fix: avatar error remove (#4173) * fix: avatar error remove * fix: index * fix: guide * fix: auth * update package;fix: input data model ui (#4181) * update package * fix: ts * update config * update jieba package * add type sign * fix: input data ui * fix: page title refresh (#4186) * fix: ts * update jieba package * fix: page title refresh * fix: remove member length check when opening invite create modal (#4193) * add env to check internal ip (#4187) * fix: ts * update jieba package * add env to check internal ip * package * fix: jieba * reset package * update config * fix: jieba package * init shell * init version * change team reload * update jieba package (#4200) * update jieba package * package * update package * remove invalid code * action * package (#4201) * package * update package * remove invalid code * package * remove i18n tip (#4202) * doc (#4205) * fix: i18n (#4208) * fix: next config (#4207) * reset package * i18n * update config * i18n * remove log --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> Co-authored-by: shilin <39396378+shilin66@users.noreply.github.com> Co-authored-by: heheer <heheer@sealos.io>
2025-07-23 05:12:39 +00:00 · 2025-03-18 14:40:41 +08:00
parent 56793114d8
commit e75d81d05a
316 changed files with 10626 additions and 8464 deletions
--- a/packages/service/core/dataset/search/controller.ts
+++ b/packages/service/core/dataset/search/controller.ts
@@ -16,7 +16,7 @@ import { reRankRecall } from '../../../core/ai/rerank';
 import { countPromptTokens } from '../../../common/string/tiktoken/index';
 import { datasetSearchResultConcat } from '@fastgpt/global/core/dataset/search/utils';
 import { hashStr } from '@fastgpt/global/common/string/tools';
-import { jiebaSplit } from '../../../common/string/jieba';
+import { jiebaSplit } from '../../../common/string/jieba/index';
 import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
 import { Types } from '../../../common/mongo';
 import json5 from 'json5';
@@ -27,6 +27,7 @@ import { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { POST } from '../../../common/api/plusRequest';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { datasetSearchQueryExtension } from './utils';
+import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';

 export type SearchDatasetDataProps = {
  histories: ChatItemType[];
@@ -39,7 +40,11 @@ export type SearchDatasetDataProps = {
  [NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
  [NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
  [NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
+  [NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
+
  [NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
+  [NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
+  [NodeInputKeyEnum.datasetSearchRerankWeight]?: number;

  /* 
    {
@@ -75,13 +80,16 @@ export type SearchDatasetDataResponse = {
 };

 export const datasetDataReRank = async ({
+  rerankModel,
  data,
  query
 }: {
+  rerankModel?: RerankModelItemType;
  data: SearchDataResponseItemType[];
  query: string;
 }): Promise<SearchDataResponseItemType[]> => {
  const results = await reRankRecall({
+    model: rerankModel,
    query,
    documents: data.map((item) => ({
      id: item.id,
@@ -154,7 +162,10 @@ export async function searchDatasetData(
    similarity = 0,
    limit: maxTokens,
    searchMode = DatasetSearchModeEnum.embedding,
+    embeddingWeight = 0.5,
    usingReRank = false,
+    rerankModel,
+    rerankWeight = 0.5,
    datasetIds = [],
    collectionFilterMatch
  } = props;
@@ -526,7 +537,7 @@ export async function searchDatasetData(
                $match: {
                  teamId: new Types.ObjectId(teamId),
                  datasetId: new Types.ObjectId(id),
-                  $text: { $search: jiebaSplit({ text: query }) },
+                  $text: { $search: await jiebaSplit({ text: query }) },
                  ...(filterCollectionIdList
                    ? {
                        collectionId: {
@@ -711,6 +722,7 @@ export async function searchDatasetData(
    });
    try {
      return await datasetDataReRank({
+        rerankModel,
        query: reRankQuery,
        data: filterSameDataResults
      });
@@ -721,11 +733,26 @@ export async function searchDatasetData(
  })();

  // embedding recall and fullText recall rrf concat
-  const rrfConcatResults = datasetSearchResultConcat([
-    { k: 60, list: embeddingRecallResults },
-    { k: 60, list: fullTextRecallResults },
-    { k: 58, list: reRankResults }
+  const baseK = 120;
+  const embK = Math.round(baseK * (1 - embeddingWeight)); // 搜索结果的 k 值
+  const fullTextK = Math.round(baseK * embeddingWeight); // rerank 结果的 k 值
+
+  const rrfSearchResult = datasetSearchResultConcat([
+    { k: embK, list: embeddingRecallResults },
+    { k: fullTextK, list: fullTextRecallResults }
  ]);
+  const rrfConcatResults = (() => {
+    if (reRankResults.length === 0) return rrfSearchResult;
+    if (rerankWeight === 1) return reRankResults;
+
+    const searchK = Math.round(baseK * rerankWeight); // 搜索结果的 k 值
+    const rerankK = Math.round(baseK * (1 - rerankWeight)); // rerank 结果的 k 值
+
+    return datasetSearchResultConcat([
+      { k: searchK, list: rrfSearchResult },
+      { k: rerankK, list: reRankResults }
+    ]);
+  })();

  // remove same q and a data
  set = new Set<string>();