doc and config rerank (#475)

projects/app/src/service/core/dataset/data/pg.ts

@@ -131,13 +131,15 @@ export async function searchDatasetData({
   model,
   similarity = 0,
   limit,
-  datasetIds = []
+  datasetIds = [],
+  rerank = false
 }: {
   text: string;
   model: string;
   similarity?: number; // min distance
   limit: number;
   datasetIds: string[];
+  rerank?: boolean;
 }) {
   const { vectors, tokenLen } = await getVectorsByText({
     model,
@@ -219,6 +221,13 @@ export async function searchDatasetData({
     return true;
   });
 
+  if (!rerank) {
+    return {
+      searchRes: filterData.slice(0, limit),
+      tokenLen
+    };
+  }
+
   // ReRank result
   const reRankResult = await reRankSearchResult({
     query: text,

projects/app/src/service/moduleDispatch/chat/oneapi.ts

@@ -208,7 +208,7 @@ function filterQuote({
       source: item.sourceName,
       sourceId: String(item.sourceId || 'UnKnow'),
       index: index + 1,
-      score: item.score.toFixed(4)
+      score: item.score?.toFixed(4)
     });
   }
   const sliceResult = sliceMessagesTB({

projects/app/src/service/moduleDispatch/dataset/search.ts

@@ -11,6 +11,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
   datasets: SelectedDatasetType;
   similarity: number;
   limit: number;
+  rerank: boolean;
   userChatInput: string;
 }>;
 export type KBSearchResponse = {
@@ -20,9 +21,9 @@ export type KBSearchResponse = {
   quoteQA: SearchDataResponseItemType[];
 };
 
-export async function dispatchDatasetSearch(props: Record<string, any>): Promise<KBSearchResponse> {
+export async function dispatchDatasetSearch(props: DatasetSearchProps): Promise<KBSearchResponse> {
   const {
-    inputs: { datasets = [], similarity = 0.4, limit = 5, userChatInput }
+    inputs: { datasets = [], similarity = 0.4, limit = 5, rerank, userChatInput }
   } = props as DatasetSearchProps;
 
   if (datasets.length === 0) {
@@ -41,7 +42,8 @@ export async function dispatchDatasetSearch(props: Record<string, any>): Promise
     model: vectorModel.model,
     similarity,
     limit,
-    datasetIds: datasets.map((item) => item.datasetId)
+    datasetIds: datasets.map((item) => item.datasetId),
+    rerank
   });
 
   return {