4.7-production (#1053)

* 4.7-alpha3 (#62) * doc * Optimize possible null Pointers and parts of Ux * fix: mulity index training error * feat: doc and rename question guide * fix ios speech input (#59) * fix: prompt editor variables nowrap (#61) * change openapi import in http module with curl import (#60) * chore(ui): dataset import modal ui (#58) * chore(ui): dataset import modal ui * use component * fix height * 4.7 (#63) * fix: claude3 image type verification failed (#1038) (#1040) * perf: curl import modal * doc img * perf: adapt cohere rerank * perf: code * perf: input style * doc --------- Co-authored-by: xiaotian <dimsky@163.com> * fix: ts * docker deploy * perf: prompt call * doc * ts * finish ui * perf: outlink detail ux * perf: user schema * fix: plugin update * feat: get current time plugin * fix: ts * perf: fetch anamation * perf: mark ux * doc * perf: select app ux * fix: split text custom string conflict * peref: inform readed * doc * memo flow component * perf: version * faq * feat: flow max runtimes * feat: similarity tip * feat: auto detect file encoding * Supports asymmetric vector model * fix: ts * perf: max w * move code * perf: hide whisper * fix: ts * feat: system msg modal * perf: catch error * perf: inform tip * fix: inform --------- Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com> Co-authored-by: xiaotian <dimsky@163.com>
2025-07-25 06:14:06 +00:00 · 2024-03-26 12:09:31 +08:00
parent ef15ca894e
commit 911512b36d
180 changed files with 2179 additions and 1361 deletions
--- a/packages/global/common/file/tools.ts
+++ b/packages/global/common/file/tools.ts
@@ -1,3 +1,5 @@
+import { detect } from 'jschardet';
+
 export const formatFileSize = (bytes: number): string => {
  if (bytes === 0) return '0 B';

@@ -7,3 +9,7 @@ export const formatFileSize = (bytes: number): string => {

  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 };
+
+export const detectFileEncoding = (buffers: string | Buffer) => {
+  return detect(buffers)?.encoding || 'utf-8';
+};
--- a/packages/global/common/string/textSplitter.ts
+++ b/packages/global/common/string/textSplitter.ts
@@ -30,7 +30,10 @@ export const splitText2Chunks = (props: {

  // The larger maxLen is, the next sentence is less likely to trigger splitting
  const stepReges: { reg: RegExp; maxLen: number }[] = [
-    ...customReg.map((text) => ({ reg: new RegExp(`(${text})`, 'g'), maxLen: chunkLen * 1.4 })),
+    ...customReg.map((text) => ({
+      reg: new RegExp(`(${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'g'),
+      maxLen: chunkLen * 1.4
+    })),
    { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
    { reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
    { reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },
--- a/packages/global/common/system/config/constants.ts
+++ b/packages/global/common/system/config/constants.ts
@@ -1,6 +1,7 @@
 export enum SystemConfigsTypeEnum {
  fastgpt = 'fastgpt',
-  fastgptPro = 'fastgptPro'
+  fastgptPro = 'fastgptPro',
+  systemMsgModal = 'systemMsgModal'
 }

 export const SystemConfigsTypeMap = {
@@ -9,5 +10,8 @@ export const SystemConfigsTypeMap = {
  },
  [SystemConfigsTypeEnum.fastgptPro]: {
    label: 'fastgptPro'
+  },
+  [SystemConfigsTypeEnum.systemMsgModal]: {
+    label: 'systemMsgModal'
  }
 };
--- a/packages/global/core/ai/api.d.ts
+++ b/packages/global/core/ai/api.d.ts
@@ -1,5 +0,0 @@
-export type PostReRankProps = {
-  query: string;
-  inputs: { id: string; text: string }[];
-};
-export type PostReRankResponse = { id: string; score?: number }[];
--- a/packages/global/core/ai/constants.ts
+++ b/packages/global/core/ai/constants.ts
@@ -25,3 +25,8 @@ export const llmModelTypeFilterMap = {
  [LLMModelTypeEnum.toolCall]: 'usedInToolCall',
  [LLMModelTypeEnum.queryExtension]: 'usedInQueryExtension'
 };
+
+export enum EmbeddingTypeEnm {
+  query = 'query',
+  db = 'db'
+}
--- a/packages/global/core/ai/model.d.ts
+++ b/packages/global/core/ai/model.d.ts
@@ -30,23 +30,25 @@ export type LLMModelItemType = {
 };

 export type VectorModelItemType = {
-  model: string;
-  name: string;
+  model: string; // model name
+  name: string; // show name
  avatar?: string;
-  defaultToken: number;
-  charsPointsPrice: number;
-  maxToken: number;
-  weight: number;
-  hidden?: boolean;
-  defaultConfig?: Record<string, any>;
+  defaultToken: number; // split text default token
+  charsPointsPrice: number; // 1k tokens=n points
+  maxToken: number; // model max token
+  weight: number; // training weight
+  hidden?: boolean; // Disallow creation
+  defaultConfig?: Record<string, any>; // post request config
+  dbConfig?: Record<string, any>; // Custom parameters for storage
+  queryConfig?: Record<string, any>; // Custom parameters for query
 };

 export type ReRankModelItemType = {
  model: string;
  name: string;
  charsPointsPrice: number;
-  requestUrl?: string;
-  requestAuth?: string;
+  requestUrl: string;
+  requestAuth: string;
 };

 export type AudioSpeechModelType = {
--- a/packages/global/core/module/node/constant.ts
+++ b/packages/global/core/module/node/constant.ts
@@ -1,4 +1,5 @@
 export enum FlowNodeInputTypeEnum {
+  triggerAndFinish = 'triggerAndFinish',
  systemInput = 'systemInput', // history, userChatInput, variableInput

  input = 'input', // one line input
--- a/packages/global/core/module/template/input.ts
+++ b/packages/global/core/module/template/input.ts
@@ -6,8 +6,8 @@ import { chatNodeSystemPromptTip } from './tip';

 export const Input_Template_Switch: FlowNodeInputItemType = {
  key: ModuleInputKeyEnum.switch,
-  type: FlowNodeInputTypeEnum.target,
-  label: 'core.module.input.label.switch',
+  type: FlowNodeInputTypeEnum.triggerAndFinish,
+  label: '',
  description: 'core.module.input.description.Trigger',
  valueType: ModuleIOValueTypeEnum.any,
  showTargetInApp: true,
--- a/packages/global/core/module/template/output.ts
+++ b/packages/global/core/module/template/output.ts
@@ -13,10 +13,10 @@ export const Output_Template_UserChatInput: FlowNodeOutputItemType = {

 export const Output_Template_Finish: FlowNodeOutputItemType = {
  key: ModuleOutputKeyEnum.finish,
-  label: 'core.module.output.label.running done',
-  description: 'core.module.output.description.running done',
+  label: '',
+  description: '',
  valueType: ModuleIOValueTypeEnum.boolean,
-  type: FlowNodeOutputTypeEnum.source,
+  type: FlowNodeOutputTypeEnum.hidden,
  targets: []
 };

--- a/packages/global/core/module/template/system/datasetConcat.ts
+++ b/packages/global/core/module/template/system/datasetConcat.ts
@@ -48,7 +48,6 @@ export const DatasetConcatModule: FlowNodeTemplateType = {
      type: FlowNodeOutputTypeEnum.source,
      valueType: ModuleIOValueTypeEnum.datasetQuote,
      targets: []
-    },
-    Output_Template_Finish
+    }
  ]
 };
--- a/packages/global/core/module/template/system/datasetSearch.ts
+++ b/packages/global/core/module/template/system/datasetSearch.ts
@@ -14,13 +14,16 @@ import { Input_Template_Switch, Input_Template_UserChatInput } from '../input';
 import { Output_Template_Finish, Output_Template_UserChatInput } from '../output';
 import { DatasetSearchModeEnum } from '../../../dataset/constants';

+export const Dataset_SEARCH_DESC =
+  '调用“语义检索”和“全文检索”能力，从“知识库”中查找可能与问题相关的参考内容';
+
 export const DatasetSearchModule: FlowNodeTemplateType = {
  id: FlowNodeTypeEnum.datasetSearchNode,
  templateType: FlowNodeTemplateTypeEnum.functionCall,
  flowType: FlowNodeTypeEnum.datasetSearchNode,
  avatar: '/imgs/module/db.png',
  name: '知识库搜索',
-  intro: '调用知识库搜索能力，查找“有可能”与问题相关的内容',
+  intro: Dataset_SEARCH_DESC,
  showStatus: true,
  isTool: true,
  inputs: [
@@ -125,7 +128,6 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
      type: FlowNodeOutputTypeEnum.source,
      valueType: ModuleIOValueTypeEnum.datasetQuote,
      targets: []
-    },
-    Output_Template_Finish
+    }
  ]
 };
--- a/packages/global/core/module/template/system/http468.ts
+++ b/packages/global/core/module/template/system/http468.ts
@@ -99,7 +99,6 @@ export const HttpModule468: FlowNodeTemplateType = {
    }
  ],
  outputs: [
-    Output_Template_Finish,
    {
      key: ModuleOutputKeyEnum.httpRawResponse,
      label: '原始响应',
--- a/packages/global/core/module/type.d.ts
+++ b/packages/global/core/module/type.d.ts
@@ -114,6 +114,7 @@ export type ChatDispatchProps = {
  inputFiles?: UserChatItemValueItemType['file'][];
  stream: boolean;
  detail: boolean; // response detail
+  maxRunTimes: number;
 };

 export type ModuleDispatchProps<T> = ChatDispatchProps & {
--- a/packages/global/package.json
+++ b/packages/global/package.json
@@ -11,7 +11,8 @@
    "nanoid": "^4.0.1",
    "js-yaml": "^4.1.0",
    "timezones-list": "^3.0.2",
-    "next": "13.5.2"
+    "next": "13.5.2",
+    "jschardet": "3.1.1"
  },
  "devDependencies": {
    "@types/js-yaml": "^4.0.9",
--- a/packages/global/support/user/inform/constants.ts
+++ b/packages/global/support/user/inform/constants.ts
@@ -1,13 +1,17 @@
-export enum InformTypeEnum {
-  system = 'system',
-  admin = 'admin'
+export enum InformLevelEnum {
+  'common' = 'common',
+  'important' = 'important',
+  'emergency' = 'emergency'
 }

-export const InformTypeMap = {
-  [InformTypeEnum.system]: {
-    label: '系统通知'
+export const InformLevelMap = {
+  [InformLevelEnum.common]: {
+    label: '普通'
  },
-  [InformTypeEnum.admin]: {
-    label: '管理员'
+  [InformLevelEnum.important]: {
+    label: '重要'
+  },
+  [InformLevelEnum.emergency]: {
+    label: '紧急'
  }
 };
--- a/packages/global/support/user/inform/type.d.ts
+++ b/packages/global/support/user/inform/type.d.ts
@@ -1,17 +1,19 @@
-import { InformTypeEnum } from './constants';
+import { InformLevelEnum } from './constants';

 export type SendInformProps = {
-  tmbId?: string;
-  type: `${InformTypeEnum}`;
  title: string;
  content: string;
+  level: `${InformLevelEnum}`;
+};
+export type SendInform2UserProps = SendInformProps & {
+  tmbId: string;
 };

 export type UserInformSchema = {
  _id: string;
  userId: string;
  time: Date;
-  type: `${InformTypeEnum}`;
+  level: `${InformLevelEnum}`;
  title: string;
  content: string;
  read: boolean;
--- a/packages/global/support/user/type.d.ts
+++ b/packages/global/support/user/type.d.ts
@@ -1,12 +1,14 @@
-import { InformTypeEnum, UserStatusEnum } from './constant';
+import { UserStatusEnum } from './constant';
 import { TeamItemType } from './team/type';

 export type UserModelSchema = {
  _id: string;
  username: string;
+  email?: string;
+  phonePrefix?: number;
+  phone?: string;
  password: string;
  avatar: string;
-  balance: number;
  promotionRate: number;
  inviterId?: string;
  openaiKey: string;
@@ -24,7 +26,6 @@ export type UserType = {
  _id: string;
  username: string;
  avatar: string;
-  balance: number;
  timezone: string;
  promotionRate: UserModelSchema['promotionRate'];
  openaiAccount: UserModelSchema['openaiAccount'];
--- a/packages/service/common/api/serverRequest.ts
+++ b/packages/service/common/api/serverRequest.ts
@@ -35,7 +35,7 @@ function checkRes(data: ResponseDataType) {
  } else if (data?.code && (data.code < 200 || data.code >= 400)) {
    return Promise.reject(data);
  }
-  return data.data;
+  return data;
 }

 /**
--- a/packages/service/common/file/utils.ts
+++ b/packages/service/common/file/utils.ts
@@ -10,15 +10,15 @@ export const removeFilesByPaths = (paths: string[]) => {
  });
 };

-const imageTypeMap: Record<string, string> = {
-  '/': 'image/jpeg',
-  i: 'image/png',
-  R: 'image/gif',
-  U: 'image/webp',
-  Q: 'image/bmp'
-};
+export const guessBase64ImageType = (str: string) => {
+  const imageTypeMap: Record<string, string> = {
+    '/': 'image/jpeg',
+    i: 'image/png',
+    R: 'image/gif',
+    U: 'image/webp',
+    Q: 'image/bmp'
+  };

-export const guessImageTypeFromBase64 = (str: string) => {
  const defaultType = 'image/jpeg';
  if (typeof str !== 'string' || str.length === 0) {
    return defaultType;
--- a/packages/service/common/vectorStore/controller.ts
+++ b/packages/service/common/vectorStore/controller.ts
@@ -24,7 +24,8 @@ export const insertDatasetDataVector = async ({
 }) => {
  const { vectors, tokens } = await getVectorsByText({
    model,
-    input: query
+    input: query,
+    type: 'db'
  });
  const { insertId } = await getVectorObj().insert({
    ...props,
--- a/packages/service/core/ai/embedding/index.ts
+++ b/packages/service/core/ai/embedding/index.ts
@@ -1,14 +1,16 @@
 import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../config';
 import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
+import { EmbeddingTypeEnm } from '@fastgpt/global/core/ai/constants';

 type GetVectorProps = {
  model: VectorModelItemType;
  input: string;
+  type?: `${EmbeddingTypeEnm}`;
 };

 // text to vector
-export async function getVectorsByText({ model, input }: GetVectorProps) {
+export async function getVectorsByText({ model, input, type }: GetVectorProps) {
  if (!input) {
    return Promise.reject({
      code: 500,
@@ -23,6 +25,8 @@ export async function getVectorsByText({ model, input }: GetVectorProps) {
    const result = await ai.embeddings
      .create({
        ...model.defaultConfig,
+        ...(type === EmbeddingTypeEnm.db && model.dbConfig),
+        ...(type === EmbeddingTypeEnm.query && model.queryConfig),
        model: model.model,
        input: [input]
      })
--- a/packages/service/core/ai/functions/queryExtension.ts
+++ b/packages/service/core/ai/functions/queryExtension.ts
@@ -160,7 +160,7 @@ A: ${chatBg}

    return {
      rawQuery: query,
-      extensionQueries: queries,
+      extensionQueries: Array.isArray(queries) ? queries : [],
      model,
      tokens: countGptMessagesTokens(messages)
    };
--- a/packages/service/core/ai/rerank/index.ts
+++ b/packages/service/core/ai/rerank/index.ts
@@ -1,7 +1,21 @@
-import { PostReRankProps, PostReRankResponse } from '@fastgpt/global/core/ai/api.d';
 import { POST } from '../../../common/api/serverRequest';

-export function reRankRecall({ query, inputs }: PostReRankProps) {
+type PostReRankResponse = {
+  id: string;
+  results: {
+    index: number;
+    relevance_score: number;
+  }[];
+};
+type ReRankCallResult = { id: string; score?: number }[];
+
+export function reRankRecall({
+  query,
+  documents
+}: {
+  query: string;
+  documents: { id: string; text: string }[];
+}): Promise<ReRankCallResult> {
  const model = global.reRankModels[0];

  if (!model || !model?.requestUrl) {
@@ -12,19 +26,24 @@ export function reRankRecall({ query, inputs }: PostReRankProps) {
  return POST<PostReRankResponse>(
    model.requestUrl,
    {
+      model: model.model,
      query,
-      inputs
+      documents: documents.map((doc) => doc.text)
    },
    {
      headers: {
        Authorization: `Bearer ${model.requestAuth}`
      },
-      timeout: 120000
+      timeout: 30000
    }
  )
    .then((data) => {
      console.log('rerank time:', Date.now() - start);
-      return data;
+
+      return data?.results?.map((item) => ({
+        id: documents[item.index].id,
+        score: item.relevance_score
+      }));
    })
    .catch((err) => {
      console.log('rerank error:', err);
--- a/packages/service/core/dataset/search/controller.ts
+++ b/packages/service/core/dataset/search/controller.ts
@@ -77,7 +77,8 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
  const embeddingRecall = async ({ query, limit }: { query: string; limit: number }) => {
    const { vectors, tokens } = await getVectorsByText({
      model: getVectorModel(model),
-      input: query
+      input: query,
+      type: 'query'
    });

    const { results } = await recallFromVectorStore({
@@ -225,7 +226,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
    try {
      const results = await reRankRecall({
        query,
-        inputs: data.map((item) => ({
+        documents: data.map((item) => ({
          id: item.id,
          text: `${item.q}\n${item.a}`
        }))
--- a/packages/service/core/workflow/dispatch/agent/runTool/constants.ts
+++ b/packages/service/core/workflow/dispatch/agent/runTool/constants.ts
@@ -1,12 +1,9 @@
 export const Prompt_Tool_Call = `<Instruction>
 你是一个智能机器人，除了可以回答用户问题外，你还掌握工具的使用能力。有时候，你可以依赖工具的运行结果，来更准确的回答用户。
-下面是你可以使用的工具，使用 JSON Schema 的格式声明，其中 toolId 是工具的 description 是工具的描述，parameters 是工具的参数，包括参数的类型和描述，required 是必填参数的列表。

-"""
-{{toolsPrompt}}
-"""
+工具使用了 JSON Schema 的格式声明，其中 toolId 是工具的 description 是工具的描述，parameters 是工具的参数，包括参数的类型和描述，required 是必填参数的列表。

-接下来，请你根据工具描述，决定回答问题或是使用工具。在完成任务过程中，USER代表用户的输入，TOOL_RESPONSE代表工具运行结果。ASSISTANT 代表你的输出。
+请你根据工具描述，决定回答问题或是使用工具。在完成任务过程中，USER代表用户的输入，TOOL_RESPONSE代表工具运行结果。ASSISTANT 代表你的输出。
 你的每次输出都必须以0,1开头，代表是否需要调用工具：
 0: 不使用工具，直接回答内容。
 1: 使用工具，返回工具调用的参数。
@@ -29,7 +26,13 @@ TOOL_RESPONSE: """
 ANSWER: 0: 今天杭州是晴天，适合去西湖、灵隐寺、千岛湖等地玩。
 </Instruction>

-现在，我们开始吧！
+现在，我们开始吧！下面是你本次可以使用的工具：
+
+"""
+{{toolsPrompt}}
+"""
+
+下面是正式的对话内容：

 USER: {{question}}
 ANSWER: 
--- a/packages/service/core/workflow/dispatch/index.ts
+++ b/packages/service/core/workflow/dispatch/index.ts
@@ -125,6 +125,7 @@ export async function dispatchWorkFlow({
    }
    if (nodeDispatchUsages) {
      chatNodeUsages = chatNodeUsages.concat(nodeDispatchUsages);
+      props.maxRunTimes -= nodeDispatchUsages.length;
    }
    if (toolResponses !== undefined) {
      if (Array.isArray(toolResponses) && toolResponses.length === 0) return;
@@ -217,7 +218,7 @@ export async function dispatchWorkFlow({
    );
  }
  async function moduleRun(module: RunningModuleItemType): Promise<any> {
-    if (res.closed) return Promise.resolve();
+    if (res.closed || props.maxRunTimes <= 0) return Promise.resolve();

    if (stream && detail && module.showStatus) {
      responseStatus({
--- a/packages/service/core/workflow/dispatch/plugin/run.ts
+++ b/packages/service/core/workflow/dispatch/plugin/run.ts
@@ -19,7 +19,6 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
    mode,
    teamId,
    tmbId,
-    module,
    params: { pluginId, ...data }
  } = props;

--- a/packages/service/core/workflow/dispatch/tools/http468.ts
+++ b/packages/service/core/workflow/dispatch/tools/http468.ts
@@ -157,7 +157,7 @@ async function fetchData({
  body: Record<string, any>;
  params: Record<string, any>;
 }): Promise<Record<string, any>> {
-  const { data: response } = await axios<Record<string, any>>({
+  const { data: response } = await axios({
    method,
    baseURL: `http://${SERVICE_LOCAL_HOST}`,
    url,
@@ -241,7 +241,8 @@ async function fetchData({
  };

  return {
-    formatResponse: parseJson(response),
+    formatResponse:
+      typeof response === 'object' && !Array.isArray(response) ? parseJson(response) : {},
    rawResponse: response
  };
 }
--- a/packages/service/support/user/controller.ts
+++ b/packages/service/support/user/controller.ts
@@ -39,7 +39,6 @@ export async function getUserDetail({
    _id: user._id,
    username: user.username,
    avatar: user.avatar,
-    balance: user.balance,
    timezone: user.timezone,
    promotionRate: user.promotionRate,
    openaiAccount: user.openaiAccount,
--- a/packages/service/support/user/inform/type.d.ts
+++ b/packages/service/support/user/inform/type.d.ts
@@ -0,0 +1,4 @@
+export type SystemMsgModalValueType = {
+  id: string;
+  content: string;
+};
--- a/packages/service/support/user/schema.ts
+++ b/packages/service/support/user/schema.ts
@@ -1,7 +1,6 @@
 import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { hashStr } from '@fastgpt/global/common/string/tools';
-import { PRICE_SCALE } from '@fastgpt/global/support/wallet/constants';
 import type { UserModelSchema } from '@fastgpt/global/support/user/type';
 import { UserStatusEnum, userStatusMap } from '@fastgpt/global/support/user/constant';

@@ -19,6 +18,15 @@ const UserSchema = new Schema({
    required: true,
    unique: true // 唯一
  },
+  email: {
+    type: String
+  },
+  phonePrefix: {
+    type: Number
+  },
+  phone: {
+    type: String
+  },
  password: {
    type: String,
    required: true,
@@ -34,10 +42,6 @@ const UserSchema = new Schema({
    type: String,
    default: '/icon/human.svg'
  },
-  balance: {
-    type: Number,
-    default: 2 * PRICE_SCALE
-  },
  inviterId: {
    // 谁邀请注册的
    type: Schema.Types.ObjectId,
--- a/packages/service/support/user/team/controller.ts
+++ b/packages/service/support/user/team/controller.ts
@@ -70,8 +70,6 @@ export async function createDefaultTeam({
  });

  if (!tmb) {
-    console.log('create default team', userId);
-
    // create
    const [{ _id: insertedId }] = await MongoTeam.create(
      [
@@ -99,6 +97,7 @@ export async function createDefaultTeam({
      ],
      { session }
    );
+    console.log('create default team', userId);
  } else {
    console.log('default team exist', userId);
    await MongoTeam.findByIdAndUpdate(tmb.teamId, {
--- a/packages/web/common/file/read/rawText.ts
+++ b/packages/web/common/file/read/rawText.ts
@@ -1,3 +1,5 @@
+import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
+
 /**
 * read file raw text
 */
@@ -6,15 +8,27 @@ export const readFileRawText = (file: File) => {
    try {
      const reader = new FileReader();
      reader.onload = () => {
-        resolve({
-          rawText: reader.result as string
-        });
+        //@ts-ignore
+        const encode = detectFileEncoding(reader.result);
+
+        // 再次读取文件，这次使用检测到的编码
+        const reader2 = new FileReader();
+        reader2.onload = () => {
+          resolve({
+            rawText: reader2.result as string
+          });
+        };
+        reader2.onerror = (err) => {
+          console.log('Error reading file with detected encoding:', err);
+          reject('Read file error with detected encoding');
+        };
+        reader2.readAsText(file, encode);
      };
      reader.onerror = (err) => {
        console.log('error txt read:', err);
        reject('Read file error');
      };
-      reader.readAsText(file);
+      reader.readAsBinaryString(file);
    } catch (error) {
      reject(error);
    }
--- a/packages/web/components/common/Icon/constants.ts
+++ b/packages/web/components/common/Icon/constants.ts
@@ -16,6 +16,7 @@ export const iconPaths = {
  'common/confirm/rightTip': () => import('./icons/common/confirm/rightTip.svg'),
  'common/courseLight': () => import('./icons/common/courseLight.svg'),
  'common/customTitleLight': () => import('./icons/common/customTitleLight.svg'),
+  'common/data': () => import('./icons/common/data.svg'),
  'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'),
  'common/errorFill': () => import('./icons/common/errorFill.svg'),
  'common/file/move': () => import('./icons/common/file/move.svg'),
--- a/packages/web/components/common/Icon/icons/common/data.svg
+++ b/packages/web/components/common/Icon/icons/common/data.svg
@@ -0,0 +1,9 @@
+<svg t="1711347951866" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="6234"
+    width="128" height="128">
+    <path
+        d="M979.2 158.848a32 32 0 0 0-38.4-23.936l-166.848 38.656a32 32 0 0 0-15.808 53.408L794.56 264.64l-194.56 173.568-101.024-95.68a32 32 0 0 0-45.152 1.152l-216.736 227.264A64 64 0 1 0 288 633.6c0-6.944-1.376-13.472-3.424-19.712l193.536-202.944 99.232 93.952a32 32 0 0 0 43.296 0.64L839.04 310.72l41.504 42.976a32 32 0 0 0 53.92-13.92l44.448-165.408a31.84 31.84 0 0 0 0.288-15.52z"
+        p-id="6235"></path>
+    <path
+        d="M928 450.464a32 32 0 0 0-32 32V736a32 32 0 0 1-32 32H160a32 32 0 0 1-32-32V160a32 32 0 0 1 32-32h530.656a32 32 0 0 0 0-64H160a96 96 0 0 0-96 96v576a96 96 0 0 0 96 96h704a96 96 0 0 0 96-96v-253.536a32 32 0 0 0-32-32zM912 896h-800a32 32 0 0 0 0 64h800a32 32 0 0 0 0-64z"
+        p-id="6236"></path>
+</svg>
--- a/packages/web/components/common/Icon/icons/core/chat/recordFill.svg
+++ b/packages/web/components/common/Icon/icons/core/chat/recordFill.svg
@@ -1,3 +1,4 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
-  <path fill-rule="evenodd" clip-rule="evenodd" d="M7.64302 0.976311C8.26814 0.351189 9.11599 0 10 0C10.8841 0 11.7319 0.351189 12.3571 0.976311C12.9822 1.60143 13.3334 2.44928 13.3334 3.33333V10C13.3334 10.8841 12.9822 11.7319 12.3571 12.357C11.7319 12.9821 10.8841 13.3333 10 13.3333C9.11599 13.3333 8.26814 12.9821 7.64302 12.357C7.0179 11.7319 6.66671 10.8841 6.66671 10V3.33333C6.66671 2.44928 7.0179 1.60143 7.64302 0.976311ZM10 1.66667C9.55801 1.66667 9.13409 1.84226 8.82153 2.15482C8.50897 2.46738 8.33337 2.89131 8.33337 3.33333V10C8.33337 10.442 8.50897 10.866 8.82153 11.1785C9.13409 11.4911 9.55801 11.6667 10 11.6667C10.4421 11.6667 10.866 11.4911 11.1786 11.1785C11.4911 10.866 11.6667 10.442 11.6667 10V3.33333C11.6667 2.89131 11.4911 2.46738 11.1786 2.15482C10.866 1.84226 10.4421 1.66667 10 1.66667ZM4.16671 7.5C4.62694 7.5 5.00004 7.8731 5.00004 8.33333V10C5.00004 11.3261 5.52682 12.5979 6.46451 13.5355C7.40219 14.4732 8.67396 15 10 15C11.3261 15 12.5979 14.4732 13.5356 13.5355C14.4733 12.5979 15 11.3261 15 10V8.33333C15 7.8731 15.3731 7.5 15.8334 7.5C16.2936 7.5 16.6667 7.8731 16.6667 8.33333V10C16.6667 11.7681 15.9643 13.4638 14.7141 14.714C13.6619 15.7662 12.2942 16.4304 10.8334 16.6144V18.3333H13.3334C13.7936 18.3333 14.1667 18.7064 14.1667 19.1667C14.1667 19.6269 13.7936 20 13.3334 20H6.66671C6.20647 20 5.83337 19.6269 5.83337 19.1667C5.83337 18.7064 6.20647 18.3333 6.66671 18.3333H9.16671V16.6144C7.70587 16.4304 6.33818 15.7662 5.286 14.714C4.03575 13.4638 3.33337 11.7681 3.33337 10V8.33333C3.33337 7.8731 3.70647 7.5 4.16671 7.5Z" fill="#485058"/>
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M7.64302 0.976311C8.26814 0.351189 9.11599 0 10 0C10.8841 0 11.7319 0.351189 12.3571 0.976311C12.9822 1.60143 13.3334 2.44928 13.3334 3.33333V10C13.3334 10.8841 12.9822 11.7319 12.3571 12.357C11.7319 12.9821 10.8841 13.3333 10 13.3333C9.11599 13.3333 8.26814 12.9821 7.64302 12.357C7.0179 11.7319 6.66671 10.8841 6.66671 10V3.33333C6.66671 2.44928 7.0179 1.60143 7.64302 0.976311ZM10 1.66667C9.55801 1.66667 9.13409 1.84226 8.82153 2.15482C8.50897 2.46738 8.33337 2.89131 8.33337 3.33333V10C8.33337 10.442 8.50897 10.866 8.82153 11.1785C9.13409 11.4911 9.55801 11.6667 10 11.6667C10.4421 11.6667 10.866 11.4911 11.1786 11.1785C11.4911 10.866 11.6667 10.442 11.6667 10V3.33333C11.6667 2.89131 11.4911 2.46738 11.1786 2.15482C10.866 1.84226 10.4421 1.66667 10 1.66667ZM4.16671 7.5C4.62694 7.5 5.00004 7.8731 5.00004 8.33333V10C5.00004 11.3261 5.52682 12.5979 6.46451 13.5355C7.40219 14.4732 8.67396 15 10 15C11.3261 15 12.5979 14.4732 13.5356 13.5355C14.4733 12.5979 15 11.3261 15 10V8.33333C15 7.8731 15.3731 7.5 15.8334 7.5C16.2936 7.5 16.6667 7.8731 16.6667 8.33333V10C16.6667 11.7681 15.9643 13.4638 14.7141 14.714C13.6619 15.7662 12.2942 16.4304 10.8334 16.6144V18.3333H13.3334C13.7936 18.3333 14.1667 18.7064 14.1667 19.1667C14.1667 19.6269 13.7936 20 13.3334 20H6.66671C6.20647 20 5.83337 19.6269 5.83337 19.1667C5.83337 18.7064 6.20647 18.3333 6.66671 18.3333H9.16671V16.6144C7.70587 16.4304 6.33818 15.7662 5.286 14.714C4.03575 13.4638 3.33337 11.7681 3.33337 10V8.33333C3.33337 7.8731 3.70647 7.5 4.16671 7.5Z" />
 </svg>
--- a/packages/web/components/common/Icon/icons/core/chat/stopSpeechFill.svg
+++ b/packages/web/components/common/Icon/icons/core/chat/stopSpeechFill.svg
@@ -1,10 +1,12 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none">
  <g clip-path="url(#clip0_74_2)">
-    <path fill-rule="evenodd" clip-rule="evenodd" d="M10 2.49999C5.85791 2.49999 2.50004 5.85786 2.50004 10C2.50004 14.1421 5.85791 17.5 10 17.5C14.1422 17.5 17.5 14.1421 17.5 10C17.5 5.85786 14.1422 2.49999 10 2.49999ZM0.833374 10C0.833374 4.93739 4.93743 0.833328 10 0.833328C15.0627 0.833328 19.1667 4.93739 19.1667 10C19.1667 15.0626 15.0627 19.1667 10 19.1667C4.93743 19.1667 0.833374 15.0626 0.833374 10ZM6.66671 7.5C6.66671 7.03976 7.0398 6.66666 7.50004 6.66666H12.5C12.9603 6.66666 13.3334 7.03976 13.3334 7.5V12.5C13.3334 12.9602 12.9603 13.3333 12.5 13.3333H7.50004C7.0398 13.3333 6.66671 12.9602 6.66671 12.5V7.5ZM8.33337 8.33333V11.6667H11.6667V8.33333H8.33337Z" fill="#3370FF"/>
+    <path fill-rule="evenodd" clip-rule="evenodd"
+      d="M10 2.49999C5.85791 2.49999 2.50004 5.85786 2.50004 10C2.50004 14.1421 5.85791 17.5 10 17.5C14.1422 17.5 17.5 14.1421 17.5 10C17.5 5.85786 14.1422 2.49999 10 2.49999ZM0.833374 10C0.833374 4.93739 4.93743 0.833328 10 0.833328C15.0627 0.833328 19.1667 4.93739 19.1667 10C19.1667 15.0626 15.0627 19.1667 10 19.1667C4.93743 19.1667 0.833374 15.0626 0.833374 10ZM6.66671 7.5C6.66671 7.03976 7.0398 6.66666 7.50004 6.66666H12.5C12.9603 6.66666 13.3334 7.03976 13.3334 7.5V12.5C13.3334 12.9602 12.9603 13.3333 12.5 13.3333H7.50004C7.0398 13.3333 6.66671 12.9602 6.66671 12.5V7.5ZM8.33337 8.33333V11.6667H11.6667V8.33333H8.33337Z"
+      fill="#3370FF" />
  </g>
  <defs>
    <clipPath id="clip0_74_2">
-      <rect width="20" height="20" fill="white"/>
+      <rect width="20" height="20" fill="white" />
    </clipPath>
  </defs>
 </svg>
--- a/packages/web/components/common/Input/HttpInput/Editor.tsx
+++ b/packages/web/components/common/Input/HttpInput/Editor.tsx
@@ -5,7 +5,7 @@ import { ContentEditable } from '@lexical/react/LexicalContentEditable';
 import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin';
 import { OnChangePlugin } from '@lexical/react/LexicalOnChangePlugin';
 import LexicalErrorBoundary from '@lexical/react/LexicalErrorBoundary';
-import { Box } from '@chakra-ui/react';
+import { Box, Flex } from '@chakra-ui/react';
 import styles from './index.module.scss';
 import { EditorState, LexicalEditor } from 'lexical';
 import { getNanoid } from '@fastgpt/global/common/string/tools';
@@ -78,7 +78,15 @@ export default function Editor({
  );

  return (
-    <Box position={'relative'} width={'full'} h={`${h}px`} cursor={'text'} overflowY={'visible'}>
+    <Flex
+      position={'relative'}
+      width={'full'}
+      minH={`${h}px`}
+      h={'full'}
+      flexDirection={'column'}
+      cursor={'text'}
+      overflowY={'visible'}
+    >
      <LexicalComposer initialConfig={initialConfig} key={key}>
        <PlainTextPlugin
          contentEditable={<ContentEditable className={styles.contentEditable} />}
@@ -125,6 +133,6 @@ export default function Editor({
      {focus && hasDropDownPlugin && (
        <DropDownMenu variables={dropdownVariables} setDropdownValue={setDropdownValue} />
      )}
-    </Box>
+    </Flex>
  );
 }
--- a/packages/web/components/common/Input/HttpInput/index.module.scss
+++ b/packages/web/components/common/Input/HttpInput/index.module.scss
@@ -3,7 +3,9 @@
  display: flex;
  flex-direction: column;
  justify-content: center;
+  flex-grow: 1;
  overflow: auto;
  padding-left: 8px;
+  padding-right: 8px;
  position: relative;
 }
--- a/packages/web/components/common/Textarea/PromptEditor/modules/DropDownMenu/index.tsx
+++ b/packages/web/components/common/Textarea/PromptEditor/modules/DropDownMenu/index.tsx
@@ -42,6 +42,7 @@ export default function DropDownMenu({
      p={2}
      borderRadius={'md'}
      position={'absolute'}
+      top={'100%'}
      w={'auto'}
      zIndex={99999}
      maxH={'300px'}
--- a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx
+++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx
@@ -99,7 +99,7 @@ export default function VariablePickerPlugin({
                    }}
                  >
                    <MyIcon name={(item.icon as any) || 'core/modules/variable'} w={'14px'} />
-                    <Box ml={2} fontSize={'sm'}>
+                    <Box ml={2} fontSize={'sm'} whiteSpace={'nowrap'}>
                      {item.key}
                      {item.key !== item.label && `(${item.label})`}
                    </Box>
--- a/packages/web/hooks/useConfirm.tsx
+++ b/packages/web/hooks/useConfirm.tsx
@@ -0,0 +1,131 @@
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { useDisclosure, Button, ModalBody, ModalFooter } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+import MyModal from '../components/common/MyModal';
+
+export const useConfirm = (props?: {
+  title?: string;
+  iconSrc?: string | '';
+  content?: string;
+  showCancel?: boolean;
+  type?: 'common' | 'delete';
+  hideFooter?: boolean;
+}) => {
+  const { t } = useTranslation();
+
+  const map = useMemo(() => {
+    const map = {
+      common: {
+        title: t('common.confirm.Common Tip'),
+        bg: undefined,
+        iconSrc: 'common/confirm/commonTip'
+      },
+      delete: {
+        title: t('common.Delete Warning'),
+        bg: 'red.600',
+        iconSrc: 'common/confirm/deleteTip'
+      }
+    };
+    if (props?.type && map[props.type]) return map[props.type];
+    return map.common;
+  }, [props?.type, t]);
+
+  const {
+    title = map?.title || t('Warning'),
+    iconSrc = map?.iconSrc,
+    content,
+    showCancel = true,
+    hideFooter = false
+  } = props || {};
+  const [customContent, setCustomContent] = useState<string | React.ReactNode>(content);
+
+  const { isOpen, onOpen, onClose } = useDisclosure();
+
+  const confirmCb = useRef<any>();
+  const cancelCb = useRef<any>();
+
+  return {
+    openConfirm: useCallback(
+      (confirm?: any, cancel?: any, customContent?: string | React.ReactNode) => {
+        confirmCb.current = confirm;
+        cancelCb.current = cancel;
+
+        customContent && setCustomContent(customContent);
+
+        return onOpen;
+      },
+      [onOpen]
+    ),
+    onClose,
+    ConfirmModal: useCallback(
+      ({
+        closeText = t('common.Close'),
+        confirmText = t('common.Confirm'),
+        isLoading,
+        bg,
+        countDown = 0
+      }: {
+        closeText?: string;
+        confirmText?: string;
+        isLoading?: boolean;
+        bg?: string;
+        countDown?: number;
+      }) => {
+        const timer = useRef<any>();
+        const [countDownAmount, setCountDownAmount] = useState(countDown);
+
+        useEffect(() => {
+          timer.current = setInterval(() => {
+            setCountDownAmount((val) => {
+              if (val <= 0) {
+                clearInterval(timer.current);
+              }
+              return val - 1;
+            });
+          }, 1000);
+        }, []);
+
+        return (
+          <MyModal
+            isOpen={isOpen}
+            onClose={onClose}
+            iconSrc={iconSrc}
+            title={title}
+            maxW={['90vw', '500px']}
+          >
+            <ModalBody pt={5}>{customContent}</ModalBody>
+            {!hideFooter && (
+              <ModalFooter>
+                {showCancel && (
+                  <Button
+                    variant={'whiteBase'}
+                    onClick={() => {
+                      onClose();
+                      typeof cancelCb.current === 'function' && cancelCb.current();
+                    }}
+                  >
+                    {closeText}
+                  </Button>
+                )}
+
+                <Button
+                  bg={bg ? bg : map.bg}
+                  isDisabled={countDownAmount > 0}
+                  ml={4}
+                  isLoading={isLoading}
+                  onClick={() => {
+                    onClose();
+                    typeof confirmCb.current === 'function' && confirmCb.current();
+                  }}
+                >
+                  {countDownAmount > 0 ? `${countDownAmount}s` : confirmText}
+                </Button>
+              </ModalFooter>
+            )}
+          </MyModal>
+        );
+      },
+      [customContent, hideFooter, iconSrc, isOpen, map.bg, onClose, showCancel, t, title]
+    )
+  };
+};
--- a/packages/web/hooks/useRequest.tsx
+++ b/packages/web/hooks/useRequest.tsx
@@ -0,0 +1,41 @@
+import { useToast } from './useToast';
+import { useMutation } from '@tanstack/react-query';
+import type { UseMutationOptions } from '@tanstack/react-query';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { useTranslation } from 'next-i18next';
+
+interface Props extends UseMutationOptions<any, any, any, any> {
+  successToast?: string | null;
+  errorToast?: string | null;
+}
+
+export const useRequest = ({ successToast, errorToast, onSuccess, onError, ...props }: Props) => {
+  const { toast } = useToast();
+  const { t } = useTranslation();
+  const mutation = useMutation<unknown, unknown, any, unknown>({
+    ...props,
+    onSuccess(res, variables: void, context: unknown) {
+      onSuccess?.(res, variables, context);
+      successToast &&
+        toast({
+          title: successToast,
+          status: 'success'
+        });
+    },
+    onError(err: any, variables: void, context: unknown) {
+      onError?.(err, variables, context);
+
+      if (errorToast !== undefined) {
+        const errText = t(getErrText(err, errorToast || ''));
+        if (errText) {
+          toast({
+            title: errText,
+            status: 'error'
+          });
+        }
+      }
+    }
+  });
+
+  return mutation;
+};