v4.6.2 (#523)

projects/app/src/service/common/api/request.ts

@@ -0,0 +1,113 @@
+import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios';
+
+interface ConfigType {
+  headers?: { [key: string]: string };
+  hold?: boolean;
+  timeout?: number;
+}
+interface ResponseDataType {
+  code: number;
+  message: string;
+  data: any;
+}
+
+/**
+ * 请求开始
+ */
+function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig {
+  return config;
+}
+
+/**
+ * 请求成功,检查请求头
+ */
+function responseSuccess(response: AxiosResponse<ResponseDataType>) {
+  return response;
+}
+/**
+ * 响应数据检查
+ */
+function checkRes(data: ResponseDataType) {
+  if (data === undefined) {
+    console.log('error->', data, 'data is empty');
+    return Promise.reject('服务器异常');
+  } else if (data?.code && (data.code < 200 || data.code >= 400)) {
+    return Promise.reject(data);
+  }
+  return data.data;
+}
+
+/**
+ * 响应错误
+ */
+function responseError(err: any) {
+  if (!err) {
+    return Promise.reject({ message: '未知错误' });
+  }
+  if (typeof err === 'string') {
+    return Promise.reject({ message: err });
+  }
+
+  if (err?.response?.data) {
+    return Promise.reject(err?.response?.data);
+  }
+  return Promise.reject(err);
+}
+
+/* 创建请求实例 */
+const instance = axios.create({
+  timeout: 60000, // 超时时间
+  headers: {
+    'content-type': 'application/json',
+    'Cache-Control': 'no-cache'
+  }
+});
+
+/* 请求拦截 */
+instance.interceptors.request.use(requestStart, (err) => Promise.reject(err));
+/* 响应拦截 */
+instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err));
+
+export function request(url: string, data: any, config: ConfigType, method: Method): any {
+  /* 去空 */
+  for (const key in data) {
+    if (data[key] === null || data[key] === undefined) {
+      delete data[key];
+    }
+  }
+
+  return instance
+    .request({
+      baseURL: `http://localhost:${process.env.PORT || 3000}`,
+      url,
+      method,
+      data: ['POST', 'PUT'].includes(method) ? data : null,
+      params: !['POST', 'PUT'].includes(method) ? data : null,
+      ...config // 用户自定义配置，可以覆盖前面的配置
+    })
+    .then((res) => checkRes(res.data))
+    .catch((err) => responseError(err));
+}
+
+/**
+ * api请求方式
+ * @param {String} url
+ * @param {Any} params
+ * @param {Object} config
+ * @returns
+ */
+export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, params, config, 'GET');
+}
+
+export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'POST');
+}
+
+export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'PUT');
+}
+
+export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'DELETE');
+}

projects/app/src/service/core/ai/rerank.ts

@@ -0,0 +1,26 @@
+import { PostReRankProps, PostReRankResponse } from '@fastgpt/global/core/ai/api';
+import { POST } from '@/service/common/api/request';
+
+export function reRankRecall({ query, inputs }: PostReRankProps) {
+  const model = global.reRankModels[0];
+
+  if (!model || !model?.requestUrl) {
+    return Promise.reject('no rerank model');
+  }
+
+  let start = Date.now();
+  return POST<PostReRankResponse>(
+    model.requestUrl,
+    {
+      query,
+      inputs
+    },
+    {
+      headers: {
+        Authorization: `Bearer ${model.requestAuth}`
+      }
+    }
+  ).finally(() => {
+    console.log('rerank time:', Date.now() - start);
+  });
+}

projects/app/src/service/core/dataset/data/controller.ts

@@ -72,7 +72,7 @@ export async function insertData2Dataset({
     collectionId,
     q,
     a,
-    fullTextToken: jiebaSplit({ text: q + a }),
+    fullTextToken: jiebaSplit({ text: qaStr }),
     indexes: indexes.map((item, i) => ({
       ...item,
       dataId: result[i].insertId

projects/app/src/service/core/dataset/data/pg.ts

@@ -1,4 +1,4 @@
-import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
+import { DatasetSearchModeEnum, PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
 import type {
   DatasetDataSchemaType,
   SearchDataResponseItemType
@@ -9,9 +9,8 @@ import { delay } from '@/utils/tools';
 import { PgSearchRawType } from '@fastgpt/global/core/dataset/api';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { POST } from '@fastgpt/service/common/api/plusRequest';
-import { PostReRankResponse } from '@fastgpt/global/core/ai/api';
 import { jiebaSplit } from '../utils';
+import { reRankRecall } from '../../ai/rerank';
 
 export async function insertData2Pg({
   mongoDataId,
@@ -136,31 +135,60 @@ type SearchProps = {
   similarity?: number; // min distance
   limit: number;
   datasetIds: string[];
-  rerank?: boolean;
+  searchMode?: `${DatasetSearchModeEnum}`;
 };
 export async function searchDatasetData(props: SearchProps) {
-  const { text, similarity = 0, limit, rerank = false } = props;
+  let { text, similarity = 0, limit, searchMode = DatasetSearchModeEnum.embedding } = props;
+  searchMode = global.systemEnv.pluginBaseUrl ? searchMode : DatasetSearchModeEnum.embedding;
+
+  const rerank =
+    searchMode === DatasetSearchModeEnum.embeddingReRank ||
+    searchMode === DatasetSearchModeEnum.embFullTextReRank;
+
+  const { embeddingLimit, fullTextLimit } = (() => {
+    // Increase search range, reduce hnsw loss
+    if (searchMode === DatasetSearchModeEnum.embedding) {
+      return {
+        embeddingLimit: limit * 2,
+        fullTextLimit: 0
+      };
+    }
+    // 50 < 2*limit < value < 100
+    if (searchMode === DatasetSearchModeEnum.embeddingReRank) {
+      return {
+        embeddingLimit: Math.min(100, Math.max(50, limit * 2)),
+        fullTextLimit: 0
+      };
+    }
+    // 50 < 3*limit < embedding < 80
+    // 20 < limit < fullTextLimit < 40
+    return {
+      embeddingLimit: Math.min(80, Math.max(50, limit * 2)),
+      fullTextLimit: Math.min(40, Math.max(20, limit))
+    };
+  })();
 
   const [{ tokenLen, embeddingRecallResults }, { fullTextRecallResults }] = await Promise.all([
     embeddingRecall({
       ...props,
-      limit: rerank ? Math.max(50, limit * 3) : limit * 2
+      rerank,
+      limit: embeddingLimit
     }),
     fullTextRecall({
       ...props,
-      limit: 40
+      limit: fullTextLimit
     })
   ]);
 
-  // concat recall result
-  let set = new Set<string>();
+  // concat embedding and fullText recall result
+  let set = new Set<string>(embeddingRecallResults.map((item) => item.id));
   const concatRecallResults = embeddingRecallResults;
-  for (const item of fullTextRecallResults) {
-    if (!set.has(item.id)) {
+  fullTextRecallResults.forEach((item) => {
+    if (!set.has(item.id) && item.score >= similarity) {
       concatRecallResults.push(item);
       set.add(item.id);
     }
-  }
+  });
 
   // remove same q and a data
   set = new Set<string>();
@@ -173,32 +201,30 @@ export async function searchDatasetData(props: SearchProps) {
 
   if (!rerank) {
     return {
-      searchRes: filterSameDataResults.slice(0, limit),
+      searchRes: filterSameDataResults.filter((item) => item.score >= similarity).slice(0, limit),
       tokenLen
     };
   }
 
   // ReRank result
-  const reRankResults = await reRankSearchResult({
-    query: text,
-    data: filterSameDataResults
+  const reRankResults = (
+    await reRankSearchResult({
+      query: text,
+      data: filterSameDataResults
+    })
+  ).filter((item) => item.score > similarity);
+
+  // (It's possible that rerank failed) concat rerank results and search results
+  set = new Set<string>(reRankResults.map((item) => item.id));
+  embeddingRecallResults.forEach((item) => {
+    if (!set.has(item.id) && item.score >= similarity) {
+      reRankResults.push(item);
+      set.add(item.id);
+    }
   });
 
-  // similarity filter
-  const filterReRankResults = reRankResults.filter((item) => item.score > similarity);
-
-  // concat rerank and embedding data
-  set = new Set<string>(filterReRankResults.map((item) => item.id));
-  const concatResult = filterReRankResults.concat(
-    filterSameDataResults.filter((item) => {
-      if (set.has(item.id)) return false;
-      set.add(item.id);
-      return true;
-    })
-  );
-
   return {
-    searchRes: concatResult.slice(0, limit),
+    searchRes: reRankResults.slice(0, limit),
     tokenLen
   };
 }
@@ -209,7 +235,7 @@ export async function embeddingRecall({
   limit,
   datasetIds = [],
   rerank = false
-}: SearchProps) {
+}: SearchProps & { rerank: boolean }) {
   const { vectors, tokenLen } = await getVectorsByText({
     model,
     input: [text]
@@ -282,16 +308,11 @@ export async function embeddingRecall({
     tokenLen
   };
 }
-export async function fullTextRecall({
-  text,
-  limit,
-  datasetIds = [],
-  rerank = false
-}: SearchProps): Promise<{
+export async function fullTextRecall({ text, limit, datasetIds = [] }: SearchProps): Promise<{
   fullTextRecallResults: SearchDataResponseItemType[];
   tokenLen: number;
 }> {
-  if (!rerank) {
+  if (limit === 0) {
     return {
       fullTextRecallResults: [],
       tokenLen: 0
@@ -361,22 +382,23 @@ export async function reRankSearchResult({
   data: SearchDataResponseItemType[];
   query: string;
 }): Promise<SearchDataResponseItemType[]> {
-  if (!global.systemEnv.pluginBaseUrl) return data;
   try {
-    const result = await POST<PostReRankResponse>('/core/ai/retrival/rerank', {
+    const results = await reRankRecall({
       query,
       inputs: data.map((item) => ({
         id: item.id,
-        text: `${item.q}\n${item.a}`.trim()
+        text: `${item.q}\n${item.a}`
       }))
     });
-    const mergeResult = result
+
+    // add new score to data
+    const mergeResult = results
       .map((item) => {
         const target = data.find((dataItem) => dataItem.id === item.id);
         if (!target) return null;
         return {
           ...target,
-          score: item.score ?? target.score
+          score: item.score || 0
         };
       })
       .filter(Boolean) as SearchDataResponseItemType[];
@@ -385,7 +407,7 @@ export async function reRankSearchResult({
   } catch (error) {
     console.log(error);
 
-    return data;
+    return [];
   }
 }
 // ------------------ search end ------------------

projects/app/src/service/core/dataset/utils.ts

@@ -1,5 +1,5 @@
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { cut, extract } from '@node-rs/jieba';
+import { cut } from '@node-rs/jieba';
 
 /**
  * Same value judgment
@@ -27,8 +27,10 @@ export async function hasSameValue({
 export function jiebaSplit({ text }: { text: string }) {
   const tokens = cut(text, true);
 
-  return tokens
-    .map((item) => item.replace(/[^\u4e00-\u9fa5a-zA-Z0-9\s]/g, '').trim())
-    .filter(Boolean)
-    .join(' ');
+  return (
+    tokens
+      .map((item) => item.replace(/[^\u4e00-\u9fa5a-zA-Z0-9\s]/g, '').trim())
+      .filter(Boolean)
+      .join(' ') || ''
+  );
 }

projects/app/src/service/moduleDispatch/dataset/search.ts

@@ -6,12 +6,13 @@ import type { ModuleDispatchProps } from '@/types/core/chat/type';
 import { ModelTypeEnum } from '@/service/core/ai/model';
 import { searchDatasetData } from '@/service/core/dataset/data/pg';
 import { ModuleInputKeyEnum, ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants';
+import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
 
 type DatasetSearchProps = ModuleDispatchProps<{
   [ModuleInputKeyEnum.datasetSelectList]: SelectedDatasetType;
   [ModuleInputKeyEnum.datasetSimilarity]: number;
   [ModuleInputKeyEnum.datasetLimit]: number;
-  [ModuleInputKeyEnum.datasetStartReRank]: boolean;
+  [ModuleInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
   [ModuleInputKeyEnum.userChatInput]: string;
 }>;
 export type DatasetSearchResponse = {
@@ -27,7 +28,7 @@ export async function dispatchDatasetSearch(
   const {
     teamId,
     tmbId,
-    inputs: { datasets = [], similarity = 0.4, limit = 5, rerank, userChatInput }
+    inputs: { datasets = [], similarity = 0.4, limit = 5, searchMode, userChatInput }
   } = props as DatasetSearchProps;
 
   if (datasets.length === 0) {
@@ -47,7 +48,7 @@ export async function dispatchDatasetSearch(
     similarity,
     limit,
     datasetIds: datasets.map((item) => item.datasetId),
-    rerank
+    searchMode
   });
 
   return {

projects/app/src/service/support/wallet/bill/push.ts

@@ -239,3 +239,35 @@ export function pushWhisperBill({
     ]
   });
 }
+
+export function pushReRankBill({
+  teamId,
+  tmbId,
+  source
+}: {
+  teamId: string;
+  tmbId: string;
+  source: `${BillSourceEnum}`;
+}) {
+  const model = global.reRankModels[0];
+  if (!model) return;
+
+  const total = model.price * PRICE_SCALE;
+  const name = 'wallet.bill.ReRank';
+
+  createBill({
+    teamId,
+    tmbId,
+    appName: name,
+    total,
+    source,
+    list: [
+      {
+        moduleName: name,
+        amount: total,
+        model: model.name,
+        tokenLen: 1
+      }
+    ]
+  });
+}