4.6.8-alpha (#804)

* perf: redirect request and err log replace perf: dataset openapi feat: session fix: retry input error feat: 468 doc sub page feat: standard sub perf: rerank tip perf: rerank tip perf: api sdk perf: openapi sub plan perf: sub ui fix: ts * perf: init log * fix: variable select * sub page * icon * perf: llm model config * perf: menu ux * perf: system store * perf: publish app name * fix: init data * perf: flow edit ux * fix: value type format and ux * fix prompt editor default value (#13) * fix prompt editor default value * fix prompt editor update when not focus * add key with variable --------- Co-authored-by: Archer <545436317@qq.com> * fix: value type * doc * i18n * import path * home page * perf: mongo session running * fix: ts * perf: use toast * perf: flow edit * perf: sse response * slider ui * fetch error * fix prompt editor rerender when not focus by key defaultvalue (#14) * perf: prompt editor * feat: dataset search concat * perf: doc * fix:ts * perf: doc * fix json editor onblur value (#15) * faq * vector model default config * ipv6 --------- Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
2025-07-23 05:12:39 +00:00 · 2024-02-01 21:57:41 +08:00
parent fc19c4cf09
commit 34602b25df
285 changed files with 10345 additions and 11223 deletions
--- a/packages/global/core/dataset/search/utils.ts
+++ b/packages/global/core/dataset/search/utils.ts
@@ -0,0 +1,95 @@
+import { countPromptTokens } from '../../../common/string/tiktoken';
+import { SearchScoreTypeEnum } from '../constants';
+import { SearchDataResponseItemType } from '../type';
+
+/* dataset search result concat */
+export const datasetSearchResultConcat = (
+  arr: { k: number; list: SearchDataResponseItemType[] }[]
+): SearchDataResponseItemType[] => {
+  arr = arr.filter((item) => item.list.length > 0);
+
+  if (arr.length === 0) return [];
+  if (arr.length === 1) return arr[0].list;
+
+  const map = new Map<string, SearchDataResponseItemType & { rrfScore: number }>();
+
+  // rrf
+  arr.forEach((item) => {
+    const k = item.k;
+
+    item.list.forEach((data, index) => {
+      const rank = index + 1;
+      const score = 1 / (k + rank);
+
+      const record = map.get(data.id);
+      if (record) {
+        // 合并两个score,有相同type的score,取最大值
+        const concatScore = [...record.score];
+        for (const dataItem of data.score) {
+          const sameScore = concatScore.find((item) => item.type === dataItem.type);
+          if (sameScore) {
+            sameScore.value = Math.max(sameScore.value, dataItem.value);
+          } else {
+            concatScore.push(dataItem);
+          }
+        }
+
+        map.set(data.id, {
+          ...record,
+          score: concatScore,
+          rrfScore: record.rrfScore + score
+        });
+      } else {
+        map.set(data.id, {
+          ...data,
+          rrfScore: score
+        });
+      }
+    });
+  });
+
+  // sort
+  const mapArray = Array.from(map.values());
+  const results = mapArray.sort((a, b) => b.rrfScore - a.rrfScore);
+
+  return results.map((item, index) => {
+    // if SearchScoreTypeEnum.rrf exist, reset score
+    const rrfScore = item.score.find((item) => item.type === SearchScoreTypeEnum.rrf);
+    if (rrfScore) {
+      rrfScore.value = item.rrfScore;
+      rrfScore.index = index;
+    } else {
+      item.score.push({
+        type: SearchScoreTypeEnum.rrf,
+        value: item.rrfScore,
+        index
+      });
+    }
+
+    // @ts-ignore
+    delete item.rrfScore;
+    return item;
+  });
+};
+
+export const filterSearchResultsByMaxChars = (
+  list: SearchDataResponseItemType[],
+  maxTokens: number
+) => {
+  const results: SearchDataResponseItemType[] = [];
+  let totalTokens = 0;
+
+  for (let i = 0; i < list.length; i++) {
+    const item = list[i];
+    totalTokens += countPromptTokens(item.q + item.a);
+    if (totalTokens > maxTokens + 500) {
+      break;
+    }
+    results.push(item);
+    if (totalTokens > maxTokens) {
+      break;
+    }
+  }
+
+  return results.length === 0 ? list.slice(0, 1) : results;
+};