4.7-production (#1053)

* 4.7-alpha3 (#62)

* doc

* Optimize possible null Pointers and parts of Ux

* fix: mulity index training error

* feat: doc and rename question guide

* fix ios speech input (#59)

* fix: prompt editor variables nowrap (#61)

* change openapi import in http module with curl import (#60)

* chore(ui): dataset import modal ui (#58)

* chore(ui): dataset import modal ui

* use component

* fix height

* 4.7 (#63)

* fix: claude3 image type verification failed (#1038) (#1040)

* perf: curl import modal

* doc img

* perf: adapt cohere rerank

* perf: code

* perf: input style

* doc

---------

Co-authored-by: xiaotian <dimsky@163.com>

* fix: ts

* docker deploy

* perf: prompt call

* doc

* ts

* finish ui

* perf: outlink detail ux

* perf: user schema

* fix: plugin update

* feat: get current time plugin

* fix: ts

* perf: fetch anamation

* perf: mark ux

* doc

* perf: select app ux

* fix: split text custom string conflict

* peref: inform readed

* doc

* memo flow component

* perf: version

* faq

* feat: flow max runtimes

* feat: similarity tip

* feat: auto detect file encoding

* Supports asymmetric vector model

* fix: ts

* perf: max w

* move code

* perf: hide whisper

* fix: ts

* feat: system msg modal

* perf: catch error

* perf: inform tip

* fix: inform

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
Co-authored-by: xiaotian <dimsky@163.com>

packages/global/common/file/tools.ts

@@ -1,3 +1,5 @@
+import { detect } from 'jschardet';
+
 export const formatFileSize = (bytes: number): string => {
   if (bytes === 0) return '0 B';
 
@@ -7,3 +9,7 @@ export const formatFileSize = (bytes: number): string => {
 
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 };
+
+export const detectFileEncoding = (buffers: string | Buffer) => {
+  return detect(buffers)?.encoding || 'utf-8';
+};

packages/global/common/string/textSplitter.ts

@@ -30,7 +30,10 @@ export const splitText2Chunks = (props: {
 
   // The larger maxLen is, the next sentence is less likely to trigger splitting
   const stepReges: { reg: RegExp; maxLen: number }[] = [
-    ...customReg.map((text) => ({ reg: new RegExp(`(${text})`, 'g'), maxLen: chunkLen * 1.4 })),
+    ...customReg.map((text) => ({
+      reg: new RegExp(`(${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'g'),
+      maxLen: chunkLen * 1.4
+    })),
     { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
     { reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
     { reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },

packages/global/common/system/config/constants.ts

@@ -1,6 +1,7 @@
 export enum SystemConfigsTypeEnum {
   fastgpt = 'fastgpt',
-  fastgptPro = 'fastgptPro'
+  fastgptPro = 'fastgptPro',
+  systemMsgModal = 'systemMsgModal'
 }
 
 export const SystemConfigsTypeMap = {
@@ -9,5 +10,8 @@ export const SystemConfigsTypeMap = {
   },
   [SystemConfigsTypeEnum.fastgptPro]: {
     label: 'fastgptPro'
+  },
+  [SystemConfigsTypeEnum.systemMsgModal]: {
+    label: 'systemMsgModal'
   }
 };

packages/global/core/ai/api.d.ts

@@ -1,5 +0,0 @@
-export type PostReRankProps = {
-  query: string;
-  inputs: { id: string; text: string }[];
-};
-export type PostReRankResponse = { id: string; score?: number }[];

packages/global/core/ai/constants.ts

@@ -25,3 +25,8 @@ export const llmModelTypeFilterMap = {
   [LLMModelTypeEnum.toolCall]: 'usedInToolCall',
   [LLMModelTypeEnum.queryExtension]: 'usedInQueryExtension'
 };
+
+export enum EmbeddingTypeEnm {
+  query = 'query',
+  db = 'db'
+}

packages/global/core/ai/model.d.ts

@@ -30,23 +30,25 @@ export type LLMModelItemType = {
 };
 
 export type VectorModelItemType = {
-  model: string;
-  name: string;
+  model: string; // model name
+  name: string; // show name
   avatar?: string;
-  defaultToken: number;
-  charsPointsPrice: number;
-  maxToken: number;
-  weight: number;
-  hidden?: boolean;
-  defaultConfig?: Record<string, any>;
+  defaultToken: number; // split text default token
+  charsPointsPrice: number; // 1k tokens=n points
+  maxToken: number; // model max token
+  weight: number; // training weight
+  hidden?: boolean; // Disallow creation
+  defaultConfig?: Record<string, any>; // post request config
+  dbConfig?: Record<string, any>; // Custom parameters for storage
+  queryConfig?: Record<string, any>; // Custom parameters for query
 };
 
 export type ReRankModelItemType = {
   model: string;
   name: string;
   charsPointsPrice: number;
-  requestUrl?: string;
-  requestAuth?: string;
+  requestUrl: string;
+  requestAuth: string;
 };
 
 export type AudioSpeechModelType = {

packages/global/core/module/node/constant.ts

@@ -1,4 +1,5 @@
 export enum FlowNodeInputTypeEnum {
+  triggerAndFinish = 'triggerAndFinish',
   systemInput = 'systemInput', // history, userChatInput, variableInput
 
   input = 'input', // one line input

packages/global/core/module/template/input.ts

@@ -6,8 +6,8 @@ import { chatNodeSystemPromptTip } from './tip';
 
 export const Input_Template_Switch: FlowNodeInputItemType = {
   key: ModuleInputKeyEnum.switch,
-  type: FlowNodeInputTypeEnum.target,
-  label: 'core.module.input.label.switch',
+  type: FlowNodeInputTypeEnum.triggerAndFinish,
+  label: '',
   description: 'core.module.input.description.Trigger',
   valueType: ModuleIOValueTypeEnum.any,
   showTargetInApp: true,

packages/global/core/module/template/output.ts

@@ -13,10 +13,10 @@ export const Output_Template_UserChatInput: FlowNodeOutputItemType = {
 
 export const Output_Template_Finish: FlowNodeOutputItemType = {
   key: ModuleOutputKeyEnum.finish,
-  label: 'core.module.output.label.running done',
-  description: 'core.module.output.description.running done',
+  label: '',
+  description: '',
   valueType: ModuleIOValueTypeEnum.boolean,
-  type: FlowNodeOutputTypeEnum.source,
+  type: FlowNodeOutputTypeEnum.hidden,
   targets: []
 };
 

packages/global/core/module/template/system/datasetConcat.ts

@@ -48,7 +48,6 @@ export const DatasetConcatModule: FlowNodeTemplateType = {
       type: FlowNodeOutputTypeEnum.source,
       valueType: ModuleIOValueTypeEnum.datasetQuote,
       targets: []
-    },
-    Output_Template_Finish
+    }
   ]
 };

packages/global/core/module/template/system/datasetSearch.ts

@@ -14,13 +14,16 @@ import { Input_Template_Switch, Input_Template_UserChatInput } from '../input';
 import { Output_Template_Finish, Output_Template_UserChatInput } from '../output';
 import { DatasetSearchModeEnum } from '../../../dataset/constants';
 
+export const Dataset_SEARCH_DESC =
+  '调用“语义检索”和“全文检索”能力，从“知识库”中查找可能与问题相关的参考内容';
+
 export const DatasetSearchModule: FlowNodeTemplateType = {
   id: FlowNodeTypeEnum.datasetSearchNode,
   templateType: FlowNodeTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.datasetSearchNode,
   avatar: '/imgs/module/db.png',
   name: '知识库搜索',
-  intro: '调用知识库搜索能力，查找“有可能”与问题相关的内容',
+  intro: Dataset_SEARCH_DESC,
   showStatus: true,
   isTool: true,
   inputs: [
@@ -125,7 +128,6 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
       type: FlowNodeOutputTypeEnum.source,
       valueType: ModuleIOValueTypeEnum.datasetQuote,
       targets: []
-    },
-    Output_Template_Finish
+    }
   ]
 };

packages/global/core/module/template/system/http468.ts

@@ -99,7 +99,6 @@ export const HttpModule468: FlowNodeTemplateType = {
     }
   ],
   outputs: [
-    Output_Template_Finish,
     {
       key: ModuleOutputKeyEnum.httpRawResponse,
       label: '原始响应',

packages/global/core/module/type.d.ts

@@ -114,6 +114,7 @@ export type ChatDispatchProps = {
   inputFiles?: UserChatItemValueItemType['file'][];
   stream: boolean;
   detail: boolean; // response detail
+  maxRunTimes: number;
 };
 
 export type ModuleDispatchProps<T> = ChatDispatchProps & {

packages/global/package.json

@@ -11,7 +11,8 @@
     "nanoid": "^4.0.1",
     "js-yaml": "^4.1.0",
     "timezones-list": "^3.0.2",
-    "next": "13.5.2"
+    "next": "13.5.2",
+    "jschardet": "3.1.1"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",

packages/global/support/user/inform/constants.ts

@@ -1,13 +1,17 @@
-export enum InformTypeEnum {
-  system = 'system',
-  admin = 'admin'
+export enum InformLevelEnum {
+  'common' = 'common',
+  'important' = 'important',
+  'emergency' = 'emergency'
 }
 
-export const InformTypeMap = {
-  [InformTypeEnum.system]: {
-    label: '系统通知'
+export const InformLevelMap = {
+  [InformLevelEnum.common]: {
+    label: '普通'
   },
-  [InformTypeEnum.admin]: {
-    label: '管理员'
+  [InformLevelEnum.important]: {
+    label: '重要'
+  },
+  [InformLevelEnum.emergency]: {
+    label: '紧急'
   }
 };

packages/global/support/user/inform/type.d.ts

@@ -1,17 +1,19 @@
-import { InformTypeEnum } from './constants';
+import { InformLevelEnum } from './constants';
 
 export type SendInformProps = {
-  tmbId?: string;
-  type: `${InformTypeEnum}`;
   title: string;
   content: string;
+  level: `${InformLevelEnum}`;
+};
+export type SendInform2UserProps = SendInformProps & {
+  tmbId: string;
 };
 
 export type UserInformSchema = {
   _id: string;
   userId: string;
   time: Date;
-  type: `${InformTypeEnum}`;
+  level: `${InformLevelEnum}`;
   title: string;
   content: string;
   read: boolean;

packages/global/support/user/type.d.ts

@@ -1,12 +1,14 @@
-import { InformTypeEnum, UserStatusEnum } from './constant';
+import { UserStatusEnum } from './constant';
 import { TeamItemType } from './team/type';
 
 export type UserModelSchema = {
   _id: string;
   username: string;
+  email?: string;
+  phonePrefix?: number;
+  phone?: string;
   password: string;
   avatar: string;
-  balance: number;
   promotionRate: number;
   inviterId?: string;
   openaiKey: string;
@@ -24,7 +26,6 @@ export type UserType = {
   _id: string;
   username: string;
   avatar: string;
-  balance: number;
   timezone: string;
   promotionRate: UserModelSchema['promotionRate'];
   openaiAccount: UserModelSchema['openaiAccount'];