V4.6.6-2 (#673)

packages/global/core/ai/model.d.ts

@@ -3,7 +3,8 @@ export type LLMModelItemType = {
   name: string;
   maxContext: number;
   maxResponse: number;
-  price: number;
+  inputPrice: number;
+  outputPrice: number;
 };
 export type ChatModelItemType = LLMModelItemType & {
   quoteMaxToken: number;
@@ -22,7 +23,8 @@ export type VectorModelItemType = {
   model: string;
   name: string;
   defaultToken: number;
-  price: number;
+  inputPrice: number;
+  outputPrice: number;
   maxToken: number;
   weight: number;
 };
@@ -30,7 +32,8 @@ export type VectorModelItemType = {
 export type ReRankModelItemType = {
   model: string;
   name: string;
-  price: number;
+  inputPrice: number;
+  outputPrice?: number;
   requestUrl?: string;
   requestAuth?: string;
 };
@@ -38,12 +41,14 @@ export type ReRankModelItemType = {
 export type AudioSpeechModelType = {
   model: string;
   name: string;
-  price: number;
+  inputPrice: number;
+  outputPrice?: number;
   voices: { label: string; value: string; bufferId: string }[];
 };
 
 export type WhisperModelType = {
   model: string;
   name: string;
-  price: number;
+  inputPrice: number;
+  outputPrice?: number;
 };

packages/global/core/ai/model.ts

@@ -6,7 +6,8 @@ export const defaultQAModels: LLMModelItemType[] = [
     name: 'GPT35-16k',
     maxContext: 16000,
     maxResponse: 16000,
-    price: 0
+    inputPrice: 0,
+    outputPrice: 0
   }
 ];
 
@@ -14,7 +15,8 @@ export const defaultVectorModels: VectorModelItemType[] = [
   {
     model: 'text-embedding-ada-002',
     name: 'Embedding-2',
-    price: 0,
+    inputPrice: 0,
+    outputPrice: 0,
     defaultToken: 500,
     maxToken: 3000,
     weight: 100

packages/global/core/app/type.d.ts

@@ -65,6 +65,7 @@ export type AppSimpleEditFormType = {
     similarity: number;
     limit: number;
     searchMode: `${DatasetSearchModeEnum}`;
+    usingReRank: boolean;
     searchEmptyText: string;
   };
   cfr: {
@@ -112,6 +113,7 @@ export type AppSimpleEditConfigTemplateType = {
       similarity?: boolean;
       limit?: boolean;
       searchMode: `${DatasetSearchModeEnum}`;
+      usingReRank: boolean;
       searchEmptyText?: boolean;
     };
     cfr?: {

packages/global/core/app/utils.ts

@@ -26,7 +26,8 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd
       similarity: 0.4,
       limit: 1500,
       searchEmptyText: '',
-      searchMode: DatasetSearchModeEnum.embedding
+      searchMode: DatasetSearchModeEnum.embedding,
+      usingReRank: false
     },
     userGuide: {
       welcomeText: '',
@@ -95,6 +96,10 @@ export const appModules2Form = ({
       defaultAppForm.dataset.searchMode =
         findInputValueByKey(module.inputs, ModuleInputKeyEnum.datasetSearchMode) ||
         DatasetSearchModeEnum.embedding;
+      defaultAppForm.dataset.usingReRank = !!findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetSearchUsingReRank
+      );
 
       // empty text
       const emptyOutputs =

packages/global/core/chat/type.d.ts

@@ -89,7 +89,8 @@ export type moduleDispatchResType = {
   moduleLogo?: string;
   price?: number;
   runningTime?: number;
-  tokens?: number;
+  inputTokens?: number;
+  outputTokens?: number;
   model?: string;
   query?: string;
   contextTotalLen?: number;
@@ -105,6 +106,7 @@ export type moduleDispatchResType = {
   similarity?: number;
   limit?: number;
   searchMode?: `${DatasetSearchModeEnum}`;
+  searchUsingReRank?: boolean;
 
   // cq
   cqList?: ClassifyQuestionAgentItemType[];
@@ -124,6 +126,9 @@ export type moduleDispatchResType = {
 
   // tf switch
   tfSwitchResult?: boolean;
+
+  // abandon
+  tokens?: number;
 };
 
 export type ChatHistoryItemResType = moduleDispatchResType & {

packages/global/core/dataset/constant.ts

@@ -1,5 +1,3 @@
-export const PgDatasetTableName = 'modeldata';
-
 /* ------------ dataset -------------- */
 export enum DatasetTypeEnum {
   folder = 'folder',
@@ -119,8 +117,8 @@ export const TrainingTypeMap = {
 /* ------------ search -------------- */
 export enum DatasetSearchModeEnum {
   embedding = 'embedding',
-  embeddingReRank = 'embeddingReRank',
-  embFullTextReRank = 'embFullTextReRank'
+  fullTextRecall = 'fullTextRecall',
+  mixedRecall = 'mixedRecall'
 }
 
 export const DatasetSearchModeMap = {
@@ -130,18 +128,25 @@ export const DatasetSearchModeMap = {
     desc: 'core.dataset.search.mode.embedding desc',
     value: DatasetSearchModeEnum.embedding
   },
-  [DatasetSearchModeEnum.embeddingReRank]: {
-    icon: 'core/dataset/modeEmbeddingRerank',
-    title: 'core.dataset.search.mode.embeddingReRank',
-    desc: 'core.dataset.search.mode.embeddingReRank desc',
-    value: DatasetSearchModeEnum.embeddingReRank
+  [DatasetSearchModeEnum.fullTextRecall]: {
+    icon: 'core/dataset/fullTextRecall',
+    title: 'core.dataset.search.mode.fullTextRecall',
+    desc: 'core.dataset.search.mode.fullTextRecall desc',
+    value: DatasetSearchModeEnum.fullTextRecall
   },
-  [DatasetSearchModeEnum.embFullTextReRank]: {
-    icon: 'core/dataset/modeEmbFTRerank',
-    title: 'core.dataset.search.mode.embFullTextReRank',
-    desc: 'core.dataset.search.mode.embFullTextReRank desc',
-    value: DatasetSearchModeEnum.embFullTextReRank
+  [DatasetSearchModeEnum.mixedRecall]: {
+    icon: 'core/dataset/mixedRecall',
+    title: 'core.dataset.search.mode.mixedRecall',
+    desc: 'core.dataset.search.mode.mixedRecall desc',
+    value: DatasetSearchModeEnum.mixedRecall
   }
 };
 
+export enum SearchScoreTypeEnum {
+  embedding = 'embedding',
+  fullText = 'fullText',
+  reRank = 'reRank',
+  rrf = 'rrf'
+}
+
 export const FolderAvatarSrc = '/imgs/files/folder.svg';

packages/global/core/dataset/type.d.ts

@@ -6,6 +6,7 @@ import {
   DatasetDataIndexTypeEnum,
   DatasetStatusEnum,
   DatasetTypeEnum,
+  SearchScoreTypeEnum,
   TrainingModeEnum
 } from './constant';
 
@@ -161,5 +162,6 @@ export type DatasetFileSchema = {
 
 /* ============= search =============== */
 export type SearchDataResponseItemType = Omit<DatasetDataItemType, 'isOwner' | 'canWrite'> & {
-  score: number;
+  score: { type: `${SearchScoreTypeEnum}`; value: number; index: number }[];
+  // score: number;
 };

packages/global/core/module/constants.ts

@@ -63,6 +63,7 @@ export enum ModuleInputKeyEnum {
   datasetSimilarity = 'similarity',
   datasetLimit = 'limit',
   datasetSearchMode = 'searchMode',
+  datasetSearchUsingReRank = 'usingReRank',
   datasetParamsModal = 'datasetParamsModal',
 
   // context extract

packages/global/core/module/template/system/datasetSearch.ts

@@ -64,12 +64,21 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.datasetSearchMode,
       type: FlowNodeInputTypeEnum.hidden,
-      label: 'core.dataset.search.Mode',
+      label: '',
       valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false,
       value: DatasetSearchModeEnum.embedding
     },
+    {
+      key: ModuleInputKeyEnum.datasetSearchUsingReRank,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '',
+      valueType: ModuleIOValueTypeEnum.boolean,
+      showTargetInApp: false,
+      showTargetInPlugin: false,
+      value: false
+    },
     {
       key: ModuleInputKeyEnum.datasetParamsModal,
       type: FlowNodeInputTypeEnum.selectDatasetParamsModal,