V4.6.6-2 (#673)

packages/global/common/system/types/index.d.ts

@@ -48,6 +48,8 @@ export type FastGPTFeConfigsType = {
   };
   scripts?: { [key: string]: string }[];
   favicon?: string;
+  customApiDomain?: string;
+  customSharePageDomain?: string;
 };
 
 export type SystemEnvType = {

packages/global/common/vectorStore/constants.ts

@@ -0,0 +1 @@
+export const PgDatasetTableName = 'modeldata';

packages/global/core/ai/model.d.ts

@@ -3,7 +3,8 @@ export type LLMModelItemType = {
   name: string;
   maxContext: number;
   maxResponse: number;
-  price: number;
+  inputPrice: number;
+  outputPrice: number;
 };
 export type ChatModelItemType = LLMModelItemType & {
   quoteMaxToken: number;
@@ -22,7 +23,8 @@ export type VectorModelItemType = {
   model: string;
   name: string;
   defaultToken: number;
-  price: number;
+  inputPrice: number;
+  outputPrice: number;
   maxToken: number;
   weight: number;
 };
@@ -30,7 +32,8 @@ export type VectorModelItemType = {
 export type ReRankModelItemType = {
   model: string;
   name: string;
-  price: number;
+  inputPrice: number;
+  outputPrice?: number;
   requestUrl?: string;
   requestAuth?: string;
 };
@@ -38,12 +41,14 @@ export type ReRankModelItemType = {
 export type AudioSpeechModelType = {
   model: string;
   name: string;
-  price: number;
+  inputPrice: number;
+  outputPrice?: number;
   voices: { label: string; value: string; bufferId: string }[];
 };
 
 export type WhisperModelType = {
   model: string;
   name: string;
-  price: number;
+  inputPrice: number;
+  outputPrice?: number;
 };

packages/global/core/ai/model.ts

@@ -6,7 +6,8 @@ export const defaultQAModels: LLMModelItemType[] = [
     name: 'GPT35-16k',
     maxContext: 16000,
     maxResponse: 16000,
-    price: 0
+    inputPrice: 0,
+    outputPrice: 0
   }
 ];
 
@@ -14,7 +15,8 @@ export const defaultVectorModels: VectorModelItemType[] = [
   {
     model: 'text-embedding-ada-002',
     name: 'Embedding-2',
-    price: 0,
+    inputPrice: 0,
+    outputPrice: 0,
     defaultToken: 500,
     maxToken: 3000,
     weight: 100

packages/global/core/app/type.d.ts

@@ -65,6 +65,7 @@ export type AppSimpleEditFormType = {
     similarity: number;
     limit: number;
     searchMode: `${DatasetSearchModeEnum}`;
+    usingReRank: boolean;
     searchEmptyText: string;
   };
   cfr: {
@@ -112,6 +113,7 @@ export type AppSimpleEditConfigTemplateType = {
       similarity?: boolean;
       limit?: boolean;
       searchMode: `${DatasetSearchModeEnum}`;
+      usingReRank: boolean;
       searchEmptyText?: boolean;
     };
     cfr?: {

packages/global/core/app/utils.ts

@@ -26,7 +26,8 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd
       similarity: 0.4,
       limit: 1500,
       searchEmptyText: '',
-      searchMode: DatasetSearchModeEnum.embedding
+      searchMode: DatasetSearchModeEnum.embedding,
+      usingReRank: false
     },
     userGuide: {
       welcomeText: '',
@@ -95,6 +96,10 @@ export const appModules2Form = ({
       defaultAppForm.dataset.searchMode =
         findInputValueByKey(module.inputs, ModuleInputKeyEnum.datasetSearchMode) ||
         DatasetSearchModeEnum.embedding;
+      defaultAppForm.dataset.usingReRank = !!findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetSearchUsingReRank
+      );
 
       // empty text
       const emptyOutputs =

packages/global/core/chat/type.d.ts

@@ -89,7 +89,8 @@ export type moduleDispatchResType = {
   moduleLogo?: string;
   price?: number;
   runningTime?: number;
-  tokens?: number;
+  inputTokens?: number;
+  outputTokens?: number;
   model?: string;
   query?: string;
   contextTotalLen?: number;
@@ -105,6 +106,7 @@ export type moduleDispatchResType = {
   similarity?: number;
   limit?: number;
   searchMode?: `${DatasetSearchModeEnum}`;
+  searchUsingReRank?: boolean;
 
   // cq
   cqList?: ClassifyQuestionAgentItemType[];
@@ -124,6 +126,9 @@ export type moduleDispatchResType = {
 
   // tf switch
   tfSwitchResult?: boolean;
+
+  // abandon
+  tokens?: number;
 };
 
 export type ChatHistoryItemResType = moduleDispatchResType & {

packages/global/core/dataset/constant.ts

@@ -1,5 +1,3 @@
-export const PgDatasetTableName = 'modeldata';
-
 /* ------------ dataset -------------- */
 export enum DatasetTypeEnum {
   folder = 'folder',
@@ -119,8 +117,8 @@ export const TrainingTypeMap = {
 /* ------------ search -------------- */
 export enum DatasetSearchModeEnum {
   embedding = 'embedding',
-  embeddingReRank = 'embeddingReRank',
-  embFullTextReRank = 'embFullTextReRank'
+  fullTextRecall = 'fullTextRecall',
+  mixedRecall = 'mixedRecall'
 }
 
 export const DatasetSearchModeMap = {
@@ -130,18 +128,25 @@ export const DatasetSearchModeMap = {
     desc: 'core.dataset.search.mode.embedding desc',
     value: DatasetSearchModeEnum.embedding
   },
-  [DatasetSearchModeEnum.embeddingReRank]: {
-    icon: 'core/dataset/modeEmbeddingRerank',
-    title: 'core.dataset.search.mode.embeddingReRank',
-    desc: 'core.dataset.search.mode.embeddingReRank desc',
-    value: DatasetSearchModeEnum.embeddingReRank
+  [DatasetSearchModeEnum.fullTextRecall]: {
+    icon: 'core/dataset/fullTextRecall',
+    title: 'core.dataset.search.mode.fullTextRecall',
+    desc: 'core.dataset.search.mode.fullTextRecall desc',
+    value: DatasetSearchModeEnum.fullTextRecall
   },
-  [DatasetSearchModeEnum.embFullTextReRank]: {
-    icon: 'core/dataset/modeEmbFTRerank',
-    title: 'core.dataset.search.mode.embFullTextReRank',
-    desc: 'core.dataset.search.mode.embFullTextReRank desc',
-    value: DatasetSearchModeEnum.embFullTextReRank
+  [DatasetSearchModeEnum.mixedRecall]: {
+    icon: 'core/dataset/mixedRecall',
+    title: 'core.dataset.search.mode.mixedRecall',
+    desc: 'core.dataset.search.mode.mixedRecall desc',
+    value: DatasetSearchModeEnum.mixedRecall
   }
 };
 
+export enum SearchScoreTypeEnum {
+  embedding = 'embedding',
+  fullText = 'fullText',
+  reRank = 'reRank',
+  rrf = 'rrf'
+}
+
 export const FolderAvatarSrc = '/imgs/files/folder.svg';

packages/global/core/dataset/type.d.ts

@@ -6,6 +6,7 @@ import {
   DatasetDataIndexTypeEnum,
   DatasetStatusEnum,
   DatasetTypeEnum,
+  SearchScoreTypeEnum,
   TrainingModeEnum
 } from './constant';
 
@@ -161,5 +162,6 @@ export type DatasetFileSchema = {
 
 /* ============= search =============== */
 export type SearchDataResponseItemType = Omit<DatasetDataItemType, 'isOwner' | 'canWrite'> & {
-  score: number;
+  score: { type: `${SearchScoreTypeEnum}`; value: number; index: number }[];
+  // score: number;
 };

packages/global/core/module/constants.ts

@@ -63,6 +63,7 @@ export enum ModuleInputKeyEnum {
   datasetSimilarity = 'similarity',
   datasetLimit = 'limit',
   datasetSearchMode = 'searchMode',
+  datasetSearchUsingReRank = 'usingReRank',
   datasetParamsModal = 'datasetParamsModal',
 
   // context extract

packages/global/core/module/template/system/datasetSearch.ts

@@ -64,12 +64,21 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.datasetSearchMode,
       type: FlowNodeInputTypeEnum.hidden,
-      label: 'core.dataset.search.Mode',
+      label: '',
       valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false,
       value: DatasetSearchModeEnum.embedding
     },
+    {
+      key: ModuleInputKeyEnum.datasetSearchUsingReRank,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '',
+      valueType: ModuleIOValueTypeEnum.boolean,
+      showTargetInApp: false,
+      showTargetInPlugin: false,
+      value: false
+    },
     {
       key: ModuleInputKeyEnum.datasetParamsModal,
       type: FlowNodeInputTypeEnum.selectDatasetParamsModal,

packages/global/support/user/team/type.d.ts

@@ -9,6 +9,7 @@ export type TeamSchema = {
   createTime: Date;
   balance: number;
   maxSize: number;
+  lastDatasetBillTime: Date;
 };
 
 export type TeamMemberSchema = {

packages/global/support/wallet/bill/api.d.ts

@@ -1,5 +1,5 @@
 import { BillSourceEnum } from './constants';
-import { BillListItemType } from './type';
+import { BillListItemCountType, BillListItemType } from './type';
 
 export type CreateTrainingBillProps = {
   name: string;
@@ -7,13 +7,12 @@ export type CreateTrainingBillProps = {
   agentModel?: string;
 };
 
-export type ConcatBillProps = {
+export type ConcatBillProps = BillListItemCountType & {
   teamId: string;
   tmbId: string;
   billId?: string;
   total: number;
   listIndex?: number;
-  tokens?: number;
 };
 
 export type CreateBillProps = {

packages/global/support/wallet/bill/constants.ts

@@ -1,16 +1,19 @@
-// ￥1 = 100000
+// model price: xxx/1k tokens
+// ￥1 = 100000.
 export const PRICE_SCALE = 100000;
 
 export enum BillSourceEnum {
   fastgpt = 'fastgpt',
   api = 'api',
   shareLink = 'shareLink',
-  training = 'training'
+  training = 'training',
+  datasetStore = 'datasetStore'
 }
 
 export const BillSourceMap: Record<`${BillSourceEnum}`, string> = {
   [BillSourceEnum.fastgpt]: '在线使用',
   [BillSourceEnum.api]: 'Api',
   [BillSourceEnum.shareLink]: '免登录链接',
-  [BillSourceEnum.training]: '数据训练'
+  [BillSourceEnum.training]: '数据训练',
+  [BillSourceEnum.datasetStore]: '知识库存储'
 };

packages/global/support/wallet/bill/tools.ts

@@ -6,9 +6,12 @@ import { AuthUserTypeEnum } from '../../permission/constant';
 /**
  * dataset price / PRICE_SCALE = real price
  */
-export const formatPrice = (val = 0, multiple = 1) => {
+export const formatStorePrice2Read = (val = 0, multiple = 1) => {
   return Number(((val / PRICE_SCALE) * multiple).toFixed(10));
 };
+export const formatModelPrice2Read = (val = 0) => {
+  return Number((val / 1000).toFixed(10));
+};
 
 export const getBillSourceByAuthType = ({
   shareId,

packages/global/support/wallet/bill/type.d.ts

@@ -1,11 +1,20 @@
 import { CreateBillProps } from './api';
 import { BillSourceEnum } from './constants';
 
-export type BillListItemType = {
+export type BillListItemCountType = {
+  inputTokens?: number;
+  outputTokens?: number;
+  textLen?: number;
+  duration?: number;
+  dataLen?: number;
+
+  // abandon
+  tokenLen?: number;
+};
+export type BillListItemType = BillListItemCountType & {
   moduleName: string;
   amount: number;
   model?: string;
-  tokenLen?: number;
 };
 
 export type BillSchema = CreateBillProps & {