v4.6.2 (#523)

packages/global/common/system/types/index.d.ts

@@ -2,16 +2,15 @@ export type FeConfigsType = {
   show_emptyChat?: boolean;
   show_register?: boolean;
   show_appStore?: boolean;
-  show_contact?: boolean;
   show_git?: boolean;
   show_pay?: boolean;
   show_openai_account?: boolean;
   show_promotion?: boolean;
   hide_app_flow?: boolean;
+  concatMd?: string;
   docUrl?: string;
   openAPIDocUrl?: string;
   systemTitle?: string;
-  authorText?: string;
   googleClientVerKey?: string;
   isPlus?: boolean;
   oauth?: {

packages/global/core/ai/model.d.ts

@@ -26,6 +26,14 @@ export type VectorModelItemType = {
   maxToken: number;
 };
 
+export type ReRankModelItemType = {
+  model: string;
+  name: string;
+  price: number;
+  requestUrl?: string;
+  requestAuth?: string;
+};
+
 export type AudioSpeechModelType = {
   model: string;
   name: string;

packages/global/core/ai/model.ts

@@ -4,7 +4,8 @@ import type {
   FunctionModelItemType,
   VectorModelItemType,
   AudioSpeechModelType,
-  WhisperModelType
+  WhisperModelType,
+  ReRankModelItemType
 } from './model.d';
 
 export const defaultChatModels: ChatModelItemType[] = [
@@ -117,6 +118,8 @@ export const defaultVectorModels: VectorModelItemType[] = [
   }
 ];
 
+export const defaultReRankModels: ReRankModelItemType[] = [];
+
 export const defaultAudioSpeechModels: AudioSpeechModelType[] = [
   {
     model: 'tts-1',

packages/global/core/app/type.d.ts

@@ -4,6 +4,7 @@ import { PermissionTypeEnum } from '../../support/permission/constant';
 import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.d';
 import { VariableInputEnum } from '../module/constants';
 import { SelectedDatasetType } from '../module/api';
+import { DatasetSearchModeEnum } from '../dataset/constant';
 
 export interface AppSchema {
   _id: string;
@@ -18,6 +19,7 @@ export interface AppSchema {
   updateTime: number;
   modules: ModuleItemType[];
   permission: `${PermissionTypeEnum}`;
+  inited?: boolean;
 }
 
 export type AppListItemType = {
@@ -62,7 +64,7 @@ export type AppSimpleEditFormType = {
     datasets: SelectedDatasetType;
     similarity: number;
     limit: number;
-    rerank: boolean;
+    searchMode: `${DatasetSearchModeEnum}`;
     searchEmptyText: string;
   };
   userGuide: {
@@ -106,7 +108,7 @@ export type AppSimpleEditConfigTemplateType = {
       datasets?: boolean;
       similarity?: boolean;
       limit?: boolean;
-      rerank?: boolean;
+      searchMode: `${DatasetSearchModeEnum}`;
       searchEmptyText?: boolean;
     };
     userGuide?: {

packages/global/core/app/utils.ts

@@ -5,6 +5,7 @@ import type { FlowNodeInputItemType } from '../module/node/type.d';
 import { getGuideModule, splitGuideModule } from '../module/utils';
 import { defaultChatModels } from '../ai/model';
 import { ModuleItemType } from '../module/type.d';
+import { DatasetSearchModeEnum } from '../dataset/constant';
 
 export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEditFormType => {
   const defaultChatModel = defaultChatModels[0];
@@ -25,7 +26,7 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd
       similarity: 0.4,
       limit: 5,
       searchEmptyText: '',
-      rerank: false
+      searchMode: DatasetSearchModeEnum.embedding
     },
     userGuide: {
       welcomeText: '',
@@ -91,10 +92,9 @@ export const appModules2Form = ({
         module.inputs,
         ModuleInputKeyEnum.datasetLimit
       );
-      defaultAppForm.dataset.rerank = findInputValueByKey(
-        module.inputs,
-        ModuleInputKeyEnum.datasetStartReRank
-      );
+      defaultAppForm.dataset.searchMode =
+        findInputValueByKey(module.inputs, ModuleInputKeyEnum.datasetSearchMode) ||
+        DatasetSearchModeEnum.embedding;
 
       // empty text
       const emptyOutputs =

packages/global/core/dataset/constant.ts

@@ -86,4 +86,31 @@ export const TrainingTypeMap = {
   // }
 };
 
+export enum DatasetSearchModeEnum {
+  embedding = 'embedding',
+  embeddingReRank = 'embeddingReRank',
+  embFullTextReRank = 'embFullTextReRank'
+}
+
+export const DatasetSearchModeMap = {
+  [DatasetSearchModeEnum.embedding]: {
+    icon: 'core/dataset/modeEmbedding',
+    title: 'core.dataset.search.mode.embedding',
+    desc: 'core.dataset.search.mode.embedding desc',
+    value: DatasetSearchModeEnum.embedding
+  },
+  [DatasetSearchModeEnum.embeddingReRank]: {
+    icon: 'core/dataset/modeEmbeddingRerank',
+    title: 'core.dataset.search.mode.embeddingReRank',
+    desc: 'core.dataset.search.mode.embeddingReRank desc',
+    value: DatasetSearchModeEnum.embeddingReRank
+  },
+  [DatasetSearchModeEnum.embFullTextReRank]: {
+    icon: 'core/dataset/modeEmbFTRerank',
+    title: 'core.dataset.search.mode.embFullTextReRank',
+    desc: 'core.dataset.search.mode.embFullTextReRank desc',
+    value: DatasetSearchModeEnum.embFullTextReRank
+  }
+};
+
 export const FolderAvatarSrc = '/imgs/files/folder.svg';

packages/global/core/module/constants.ts

@@ -61,7 +61,8 @@ export enum ModuleInputKeyEnum {
   datasetSelectList = 'datasets',
   datasetSimilarity = 'similarity',
   datasetLimit = 'limit',
-  datasetStartReRank = 'rerank',
+  datasetSearchMode = 'searchMode',
+  datasetParamsModal = 'datasetParamsModal',
 
   // context extract
   contextExtractInput = 'content',
@@ -98,5 +99,6 @@ export enum ModuleOutputKeyEnum {
 
 export enum VariableInputEnum {
   input = 'input',
+  textarea = 'textarea',
   select = 'select'
 }

packages/global/core/module/node/constant.ts

@@ -16,6 +16,8 @@ export enum FlowNodeInputTypeEnum {
   selectChatModel = 'selectChatModel',
   // dataset special input
   selectDataset = 'selectDataset',
+  selectDatasetParamsModal = 'selectDatasetParamsModal',
+
   hidden = 'hidden'
 }
 

packages/global/core/module/template/system/classifyQuestion.ts

@@ -17,8 +17,11 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
   flowType: FlowNodeTypeEnum.classifyQuestion,
   avatar: '/imgs/module/cq.png',
   name: '问题分类',
-  intro:
-    '根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：\n类型1: 打招呼\n类型2: 关于 laf 通用问题\n类型3: 关于 laf 代码问题\n类型4: 其他问题',
+  intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：
+类型1: 打招呼
+类型2: 关于商品“使用”问题
+类型3: 关于商品“购买”问题
+类型4: 其他问题`,
   showStatus: true,
   inputs: [
     Input_Template_TFSwitch,

packages/global/core/module/template/system/datasetSearch.ts

@@ -12,6 +12,7 @@ import {
 } from '../../constants';
 import { Input_Template_TFSwitch, Input_Template_UserChatInput } from '../input';
 import { Output_Template_Finish } from '../output';
+import { DatasetSearchModeEnum } from '../../../dataset/constant';
 
 export const DatasetSearchModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.datasetSearchNode,
@@ -36,7 +37,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.datasetSimilarity,
-      type: FlowNodeInputTypeEnum.slider,
+      type: FlowNodeInputTypeEnum.hidden,
       label: '最低相关性',
       value: 0.4,
       valueType: ModuleDataTypeEnum.number,
@@ -52,7 +53,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.datasetLimit,
-      type: FlowNodeInputTypeEnum.slider,
+      type: FlowNodeInputTypeEnum.hidden,
       label: '单次搜索上限',
       description: '最多取 n 条记录作为本次问题引用',
       value: 5,
@@ -68,13 +69,20 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       showTargetInPlugin: false
     },
     {
-      key: ModuleInputKeyEnum.datasetStartReRank,
-      type: FlowNodeInputTypeEnum.switch,
-      label: '结果重排',
-      description: '将召回的结果进行进一步重排，可增加召回率',
-      plusField: true,
-      value: false,
-      valueType: ModuleDataTypeEnum.boolean,
+      key: ModuleInputKeyEnum.datasetSearchMode,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: 'core.dataset.search.Mode',
+      valueType: ModuleDataTypeEnum.string,
+      showTargetInApp: false,
+      showTargetInPlugin: false,
+      value: DatasetSearchModeEnum.embedding
+    },
+    {
+      key: ModuleInputKeyEnum.datasetParamsModal,
+      type: FlowNodeInputTypeEnum.selectDatasetParamsModal,
+      label: '',
+      connected: false,
+      valueType: ModuleDataTypeEnum.any,
       showTargetInApp: false,
       showTargetInPlugin: false
     },

packages/global/core/module/template/system/history.ts

@@ -18,10 +18,12 @@ export const HistoryModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.historyMaxAmount,
       type: FlowNodeInputTypeEnum.numberInput,
       label: '最长记录数',
+      description:
+        '该记录数不代表模型可接收这么多的历史记录，具体可接收多少历史记录，取决于模型的能力，通常建议不要超过20条。',
       value: 6,
       valueType: ModuleDataTypeEnum.number,
       min: 0,
-      max: 50,
+      max: 100,
       showTargetInApp: false,
       showTargetInPlugin: false
     },

packages/service/core/dataset/data/schema.ts

@@ -45,7 +45,7 @@ const DatasetDataSchema = new Schema({
   },
   fullTextToken: {
     type: String,
-    required: true
+    default: ''
   },
   indexes: {
     type: [

packages/service/support/permission/controller.ts

@@ -105,14 +105,13 @@ export async function parseHeaderCert({
     };
   }
   // root user
-  async function parseRootKey(rootKey?: string, userId = '') {
+  async function parseRootKey(rootKey?: string) {
     if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
       return Promise.reject(ERROR_ENUM.unAuthorization);
     }
-    return userId;
   }
 
-  const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers ||
+  const { cookie, token, apikey, rootkey, authorization } = (req.headers ||
     {}) as ReqHeaderAuthType;
 
   const { uid, teamId, tmbId, appId, openApiKey, authType } = await (async () => {
@@ -129,9 +128,10 @@ export async function parseHeaderCert({
       };
     }
     if (authRoot && rootkey) {
+      await parseRootKey(rootkey);
       // root user
       return {
-        uid: await parseRootKey(rootkey, userid),
+        uid: '',
         teamId: '',
         tmbId: '',
         appId: '',