4.6.2-production (#518)

2025-07-23 05:12:39 +00:00 · 2023-11-26 16:13:45 +08:00
parent 3acbf1ab17
commit f818260711
30 changed files with 477 additions and 283 deletions
--- a/projects/app/package.json
+++ b/projects/app/package.json
@@ -1,6 +1,6 @@
 {
  "name": "app",
-  "version": "4.6.1",
+  "version": "4.6.2",
  "private": false,
  "scripts": {
    "dev": "next dev",
@@ -38,7 +38,7 @@
    "jsdom": "^22.1.0",
    "jsonwebtoken": "^9.0.2",
    "lodash": "^4.17.21",
-    "mammoth": "^1.5.1",
+    "mammoth": "^1.6.0",
    "mermaid": "^10.2.3",
    "multer": "1.4.5-lts.1",
    "nanoid": "^4.0.1",
--- a/projects/app/public/docs/versionIntro.md
+++ b/projects/app/public/docs/versionIntro.md
@@ -1,12 +1,13 @@
-### Fast GPT V4.6
+### Fast GPT V4.6.2

 1. 新增 - 团队空间
 2. 新增 - 多路向量（多个向量映射一组数据）
 3. 新增 - tts语音
-4. 线上环境新增 - ReRank向量召回，提高召回精度
-5. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈
-6. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
-7. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
-8. [使用文档](https://doc.fastgpt.in/docs/intro/)
-9. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
-10. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)
+4. 新增 - 语音输入
+5. 新增 - 增强召回方式，提高召回精度
+6. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈
+7. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
+8. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
+9. [使用文档](https://doc.fastgpt.in/docs/intro/)
+10. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
+11. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)
--- a/projects/app/public/imgs/modal/key.svg
+++ b/projects/app/public/imgs/modal/key.svg
@@ -1 +1 @@
-<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700745751866" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="36975" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M220.150923 700.501366l167.543301-167.5433a73.062148 73.062148 0 0 1 103.327242 103.327242l-365.310741 365.310741a73.062148 73.062148 0 1 1-103.327242-103.286288l94.481152-94.481152-64.257012-64.257013a73.062148 73.062148 0 1 1 103.327242-103.286288l64.216058 64.216058z" fill="#CCDAFF" p-id="36976"></path><path d="M475.909397 49.003242a365.310741 365.310741 0 1 1 365.310741 632.782092 365.310741 365.310741 0 0 1-365.310741-632.741138z m118.685036 205.589677a127.981622 127.981622 0 1 0 127.981622 221.643692 127.981622 127.981622 0 0 0-127.981622-221.684646z" fill="#244DD5" p-id="36977"></path></svg>
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700983497588" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="6628" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M698.483573 594.905936A287.506808 287.506808 0 1 1 984.611923 306.020671v1.181535a287.309885 287.309885 0 0 1-286.12835 287.70373z" fill="#FFFFFF" p-id="6629"></path><path d="M698.483573 39.387645A267.814561 267.814561 0 1 1 433.229005 308.777585v-1.575379A267.420716 267.420716 0 0 1 698.483573 39.387645m0-39.384494A307.199055 307.199055 0 1 0 1004.30417 308.580663v-1.378457A306.411365 306.411365 0 0 0 698.680495 0.003151z" fill="#007FB7" p-id="6630"></path><path d="M787.689452 236.310116m-78.768988 0a78.768988 78.768988 0 1 0 157.537977 0 78.768988 78.768988 0 1 0-157.537977 0Z" fill="#D1EBF2" p-id="6631"></path><path d="M787.689452 177.233375a59.076741 59.076741 0 1 1-59.076741 59.076741 59.076741 59.076741 0 0 1 59.076741-59.076741m0-39.384495a98.461236 98.461236 0 1 0 98.461236 98.461236 98.461236 98.461236 0 0 0-98.461236-98.461236z" fill="#007FB7" p-id="6632"></path><path d="M39.384062 974.57246v-113.033499l390.300338-392.466484 162.067194 108.701204-116.381181 58.682896v124.455002l-135.876505 5.316906v131.150366l-127.014993 4.923062-65.772106 99.248925L39.384062 974.57246z" fill="#D1EBF2" p-id="6633"></path><path d="M433.229005 494.475475l120.713474 80.935136-75.421306 38.006037-21.661472 10.830736v118.153482l-98.461235 3.741527-38.793727 2.166148v131.347288l-98.461236 3.741527h-19.692247l-11.224581 16.73841L137.845298 979.101677l-78.768988-19.692247v-89.796647l374.152695-374.152695m-5.119985-50.805998L19.691815 853.268218v136.467272L155.56832 1024l67.938253-102.399685 135.876505-5.316907v-131.150365l135.876505-5.316907v-131.347288L630.151476 580.333673l-203.027068-136.664195z" fill="#007FB7" p-id="6634"></path></svg>
--- a/projects/app/public/locales/en/common.json
+++ b/projects/app/public/locales/en/common.json
@@ -193,6 +193,9 @@
      "unKnow": "There was an accident"
    },
    "export": "",
+    "file": {
+      "Select file amount limit 100": "You can select a maximum of 100 files at a time"
+    },
    "folder": {
      "Drag Tip": "Click and move",
      "Move Success": "Move Success",
@@ -260,10 +263,22 @@
      "Similarity": "Similarity",
      "data": {
        "Edit": "Edit Data",
+        "data is deleted": "Data is deleted",
        "id": "Data ID"
      },
+      "import": {
+        "Ideal chunk length": "Ideal chunk length",
+        "Ideal chunk length Tips": "Segment by end symbol. We recommend that your document should be properly punctuated to ensure that each complete sentence length does not exceed this value \n Chinese document recommended 400~1000\n English document recommended 600~1200"
+      },
      "test": {
-        "Test Result": "Results"
+        "Test": "Start",
+        "Test Result": "Results",
+        "Test Text": "Text",
+        "Test Text Placeholder": "Enter the text you want to test",
+        "delete test history": "Delete the test result",
+        "test history": "Test History",
+        "test result placeholder": "The test results will be presented here",
+        "test result tip": "The contents of the knowledge base are sorted according to their similarity to the test text, and you can adjust the corresponding text according to the test results. Note: The data in the test record may have been modified, clicking on a test data will show the latest data."
      }
    },
    "module": {
--- a/projects/app/public/locales/zh/common.json
+++ b/projects/app/public/locales/zh/common.json
@@ -193,6 +193,9 @@
      "unKnow": "出现了点意外~"
    },
    "export": "",
+    "file": {
+      "Select file amount limit 100": "每次最多选择100个文件"
+    },
    "folder": {
      "Drag Tip": "点我可拖动",
      "Move Success": "移动成功",
@@ -260,10 +263,22 @@
      "Similarity": "相似度",
      "data": {
        "Edit": "编辑数据",
+        "data is deleted": "该数据已被删除",
        "id": "数据ID"
      },
+      "import": {
+        "Ideal chunk length": "理想分块长度",
+        "Ideal chunk length Tips": "按结束符号进行分段。我们建议您的文档应合理的使用标点符号，以确保每个完整的句子长度不要超过该值\n中文文档建议400~1000\n英文文档建议600~1200"
+      },
      "test": {
-        "Test Result": "测试结果"
+        "Test": "测试",
+        "Test Result": "测试结果",
+        "Test Text": "测试文本",
+        "Test Text Placeholder": "输入需要测试的文本",
+        "delete test history": "删除该测试结果",
+        "test history": "测试历史",
+        "test result placeholder": "测试结果将在这里展示",
+        "test result tip": "根据知识库内容与测试文本的相似度进行排序，你可以根据测试结果调整对应的文本。\n注意：测试记录中的数据可能已经被修改过，点击某条测试数据后将展示最新的数据。"
      }
    },
    "module": {
--- a/projects/app/src/components/ChatBox/MessageInput.tsx
+++ b/projects/app/src/components/ChatBox/MessageInput.tsx
@@ -8,7 +8,7 @@ import MyIcon from '../Icon';
 import styles from './index.module.scss';
 import { useRouter } from 'next/router';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { useToast } from '@/web/common/hooks/useToast';
 import { customAlphabet } from 'nanoid';
 import { IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
@@ -72,7 +72,7 @@ const MessageInput = ({
  const uploadFile = async (file: FileItemType) => {
    if (file.type === FileTypeEnum.image) {
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file: file.rawFile,
          maxW: 1000,
          maxH: 1000,
--- a/projects/app/src/components/Markdown/img/Image.tsx
+++ b/projects/app/src/components/Markdown/img/Image.tsx
@@ -1,5 +1,6 @@
 import React, { useState } from 'react';
 import {
+  Box,
  Image,
  Modal,
  ModalCloseButton,
@@ -8,6 +9,7 @@ import {
  Skeleton,
  useDisclosure
 } from '@chakra-ui/react';
+import MyModal from '@/components/MyModal';

 const MdImage = ({ src }: { src?: string }) => {
  const [isLoading, setIsLoading] = useState(true);
@@ -43,17 +45,21 @@ const MdImage = ({ src }: { src?: string }) => {
          onOpen();
        }}
      />
-      <Modal isOpen={isOpen} onClose={onClose}>
+      <Modal isOpen={isOpen} onClose={onClose} isCentered>
        <ModalOverlay />
-        <ModalContent m={'auto'}>
-          <Image
-            src={src}
-            alt={''}
-            fallbackSrc={'/imgs/errImg.png'}
-            fallbackStrategy={'onError'}
-            loading="eager"
-            objectFit={'contain'}
-          />
+        <ModalContent maxW={'80vw'} maxH={'auto'}>
+          <Box>
+            <Image
+              borderRadius={'md'}
+              src={src}
+              alt={''}
+              w={'auto'}
+              h={'auto'}
+              fallbackSrc={'/imgs/errImg.png'}
+              fallbackStrategy={'onError'}
+              objectFit={'contain'}
+            />
+          </Box>
        </ModalContent>
        <ModalCloseButton bg={'myWhite.500'} zIndex={999999} />
      </Modal>
--- a/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx
+++ b/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx
@@ -2,7 +2,7 @@ import React, { useCallback, useState } from 'react';
 import { useForm } from 'react-hook-form';
 import { useTranslation } from 'next-i18next';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { useToast } from '@/web/common/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useRequest } from '@/web/common/hooks/useRequest';
@@ -49,7 +49,7 @@ function EditModal({
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/global/core/prompt/AIChat.ts
+++ b/projects/app/src/global/core/prompt/AIChat.ts
@@ -35,7 +35,7 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [
 1. 背景知识是最新的实时的信息，使用背景知识回答问题。
 2. 优先使用背景知识的内容回答我的问题，答案应与背景知识严格一致。
 3. 背景知识无法回答我的问题时，可以忽略背景知识，根据你的知识来自由回答。
-4. 使用对话的风格，自然的回答问题。
+4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
 我的问题是:"{{question}}"`
  },
  {
@@ -49,7 +49,7 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [
 1. 背景知识是最新的实时的信息，使用背景知识回答问题，其中 instruction 是相关介绍，output 是预期回答或补充。
 2. 优先使用背景知识的内容回答我的问题，答案应与背景知识严格一致。
 3. 背景知识无法回答我的问题时，可以忽略背景知识，根据你的知识来自由回答。
-4. 使用对话的风格，自然的回答问题。
+4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
 我的问题是:"{{question}}"`
  },
  {
@@ -63,7 +63,7 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [
 1. 背景知识是最新的实时的信息，是你的唯一信息来源，使用背景知识回答问题。
 2. 优先使用背景知识回答我的问题，答案与背景知识完全一致，无需做其他回答。
 3. 背景知识与问题无关，或背景知识无法回答本次问题时，则拒绝回答本次问题：“我不太清除xxx”。
-4. 使用对话的风格，自然的回答问题。
+4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
 我的问题是:"{{question}}"`
  },
  {
--- a/projects/app/src/global/core/prompt/agent.ts
+++ b/projects/app/src/global/core/prompt/agent.ts
@@ -1,18 +1,17 @@
 export const Prompt_AgentQA = {
-  prompt: `我会给你一段文本，{{theme}}，学习它们，并整理学习成果，要求为：
+  description: `我会给你一段文本，学习它们，并整理学习成果，要求为：
 1. 提出问题并给出每个问题的答案。
 2. 每个答案都要详细完整，给出相关原文描述，答案可以包含普通文字、链接、代码、表格、公示、媒体链接等 markdown 元素。
 3. 最多提出 30 个问题。
-4. 按格式返回多个问题和答案:
-
+`,
+  fixedText: `最后，你需要按下面的格式返回多个问题和答案:
 Q1: 问题。
 A1: 答案。
 Q2:
 A2:
 ……

-我的文本："""{{text}}"""`,
-  defaultTheme: '它们可能包含多个主题内容'
+我的文本："""{{text}}"""`
 };

 export const Prompt_ExtractJson = `你可以从 "对话记录" 中提取指定信息，并返回一个 JSON 对象，JSON 对象要求：
--- a/projects/app/src/pages/account/components/Info.tsx
+++ b/projects/app/src/pages/account/components/Info.tsx
@@ -17,7 +17,7 @@ import type { UserType } from '@fastgpt/global/support/user/type.d';
 import { useQuery } from '@tanstack/react-query';
 import dynamic from 'next/dynamic';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { feConfigs, systemVersion } from '@/web/common/system/staticData';
 import { useTranslation } from 'next-i18next';
 import { timezoneList } from '@fastgpt/global/common/time/timezone';
@@ -94,7 +94,7 @@ const UserInfo = () => {
      const file = e[0];
      if (!file || !userInfo) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/app/detail/components/InfoModal.tsx
+++ b/projects/app/src/pages/app/detail/components/InfoModal.tsx
@@ -14,7 +14,7 @@ import { useForm } from 'react-hook-form';
 import { AppSchema } from '@fastgpt/global/core/app/type.d';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import Avatar from '@/components/Avatar';
@@ -101,7 +101,7 @@ const InfoModal = ({
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/app/list/component/CreateModal.tsx
+++ b/projects/app/src/pages/app/list/component/CreateModal.tsx
@@ -13,7 +13,7 @@ import {
 } from '@chakra-ui/react';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useToast } from '@/web/common/hooks/useToast';
 import { postCreateApp } from '@/web/core/app/api';
@@ -58,7 +58,7 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
@@ -16,10 +16,12 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';

 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
+import { useTranslation } from 'next-i18next';

-const fileExtension = '.txt, .doc, .docx, .pdf, .md';
+const fileExtension = '.txt, .docx, .pdf, .md';

 const ChunkImport = () => {
+  const { t } = useTranslation();
  const { datasetDetail } = useDatasetStore();
  const vectorModel = datasetDetail.vectorModel;
  const unitPrice = vectorModel?.price || 0.2;
@@ -48,13 +50,8 @@ const ChunkImport = () => {
        {/* chunk size */}
        <Flex py={4} alignItems={'center'}>
          <Box>
-            段落长度
-            <MyTooltip
-              label={
-                '按结束标点符号进行分段。前后段落会有 20% 的内容重叠。\n中文文档建议不要超过1000，英文不要超过1500'
-              }
-              forceShow
-            >
+            {t('core.dataset.import.Ideal chunk length')}
+            <MyTooltip label={t('core.dataset.import.Ideal chunk length Tips')} forceShow>
              <QuestionOutlineIcon ml={1} />
            </MyTooltip>
          </Box>
--- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
@@ -48,6 +48,7 @@ export interface Props extends BoxProps {
  onPushFiles: (files: FileItemType[]) => void;
  tipText?: string;
  chunkLen?: number;
+  overlapRatio?: number;
  fileTemplate?: {
    type: string;
    filename: string;
@@ -63,6 +64,7 @@ const FileSelect = ({
  onPushFiles,
  tipText,
  chunkLen = 500,
+  overlapRatio,
  fileTemplate,
  showUrlFetch = true,
  showCreateFile = true,
@@ -97,6 +99,13 @@ const FileSelect = ({
  // select file
  const onSelectFile = useCallback(
    async (files: File[]) => {
+      if (files.length >= 100) {
+        return toast({
+          status: 'warning',
+          title: t('common.file.Select file amount limit 100')
+        });
+      }
+
      try {
        for await (let file of files) {
          const extension = file?.name?.split('.')?.pop()?.toLowerCase();
@@ -165,7 +174,6 @@ const FileSelect = ({
                return readTxtContent(file);
              case 'pdf':
                return readPdfContent(file);
-              case 'doc':
              case 'docx':
                return readDocContent(file);
            }
@@ -176,7 +184,8 @@ const FileSelect = ({
            text = simpleText(text);
            const splitRes = splitText2Chunks({
              text,
-              maxLen: chunkLen
+              chunkLen,
+              overlapRatio
            });

            const fileItem: FileItemType = {
@@ -206,7 +215,7 @@ const FileSelect = ({
      }
      setSelectingText(undefined);
    },
-    [chunkLen, datasetDetail._id, onPushFiles, t, toast]
+    [chunkLen, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
  );
  // link fetch
  const onUrlFetch = useCallback(
@@ -214,7 +223,8 @@ const FileSelect = ({
      const result: FileItemType[] = e.map(({ url, content }) => {
        const splitRes = splitText2Chunks({
          text: content,
-          maxLen: chunkLen
+          chunkLen,
+          overlapRatio
        });
        return {
          id: nanoid(),
@@ -234,7 +244,7 @@ const FileSelect = ({
      });
      onPushFiles(result);
    },
-    [chunkLen, onPushFiles]
+    [chunkLen, onPushFiles, overlapRatio]
  );
  // manual create file and copy data
  const onCreateFile = useCallback(
@@ -255,7 +265,8 @@ const FileSelect = ({

      const splitRes = splitText2Chunks({
        text: content,
-        maxLen: chunkLen
+        chunkLen,
+        overlapRatio
      });

      onPushFiles([
@@ -276,7 +287,7 @@ const FileSelect = ({
        }
      ]);
    },
-    [chunkLen, datasetDetail._id, onPushFiles]
+    [chunkLen, datasetDetail._id, onPushFiles, overlapRatio]
  );

  const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
--- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
@@ -41,16 +41,19 @@ const ImportData = ({
    const map = {
      [ImportTypeEnum.chunk]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
+        chunkOverlapRatio: 0.2,
        unitPrice: vectorModel?.price || 0.2,
        mode: TrainingModeEnum.chunk
      },
      [ImportTypeEnum.qa]: {
-        defaultChunkLen: agentModel?.maxContext * 0.6 || 9000,
+        defaultChunkLen: agentModel?.maxContext * 0.6 || 8000,
+        chunkOverlapRatio: 0,
        unitPrice: agentModel?.price || 3,
        mode: TrainingModeEnum.qa
      },
      [ImportTypeEnum.csv]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
+        chunkOverlapRatio: 0,
        unitPrice: vectorModel?.price || 0.2,
        mode: TrainingModeEnum.chunk
      }
--- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
@@ -44,6 +44,7 @@ type useImportStoreType = {
  price: number;
  uploading: boolean;
  chunkLen: number;
+  chunkOverlapRatio: number;
  setChunkLen: Dispatch<number>;
  showRePreview: boolean;
  setReShowRePreview: Dispatch<SetStateAction<boolean>>;
@@ -66,6 +67,7 @@ const StateContext = createContext<useImportStoreType>({
  },
  price: 0,
  chunkLen: 0,
+  chunkOverlapRatio: 0,
  setChunkLen: function (value: number): void {
    throw new Error('Function not implemented.');
  },
@@ -93,6 +95,7 @@ const Provider = ({
  vectorModel,
  agentModel,
  defaultChunkLen = 500,
+  chunkOverlapRatio = 0.2,
  importType,
  onUploadSuccess,
  children
@@ -104,6 +107,7 @@ const Provider = ({
  vectorModel: string;
  agentModel: string;
  defaultChunkLen: number;
+  chunkOverlapRatio: number;
  importType: `${ImportTypeEnum}`;
  onUploadSuccess: () => void;
  children: React.ReactNode;
@@ -180,7 +184,8 @@ const Provider = ({
        state.map((file) => {
          const splitRes = splitText2Chunks({
            text: file.text,
-            maxLen: chunkLen
+            chunkLen,
+            overlapRatio: chunkOverlapRatio
          });

          return {
@@ -228,6 +233,7 @@ const Provider = ({
    onclickUpload,
    uploading,
    chunkLen,
+    chunkOverlapRatio,
    setChunkLen,
    showRePreview,
    setReShowRePreview
@@ -413,7 +419,8 @@ export const SelectorContainer = ({
  tip?: string;
  children: React.ReactNode;
 }) => {
-  const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen } = useImportStore();
+  const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen, chunkOverlapRatio } =
+    useImportStore();
  return (
    <Box
      h={'100%'}
@@ -432,6 +439,7 @@ export const SelectorContainer = ({
          setFiles((state) => files.concat(state));
        }}
        chunkLen={chunkLen}
+        overlapRatio={chunkOverlapRatio}
        showUrlFetch={showUrlFetch}
        showCreateFile={showCreateFile}
        fileTemplate={fileTemplate}
--- a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
@@ -1,15 +1,14 @@
-import React, { useState, useMemo } from 'react';
-import { Box, Flex, Button, Input } from '@chakra-ui/react';
+import React, { useState } from 'react';
+import { Box, Flex, Button, Textarea } from '@chakra-ui/react';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import MyTooltip from '@/components/MyTooltip';
-import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons';
+import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { Prompt_AgentQA } from '@/global/core/prompt/agent';
-import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';

-const fileExtension = '.txt, .doc, .docx, .pdf, .md';
+const fileExtension = '.txt, .docx, .pdf, .md';

 const QAImport = () => {
  const { datasetDetail } = useDatasetStore();
@@ -31,36 +30,27 @@ const QAImport = () => {
    content: `该任务无法终止！导入后会自动调用大模型生成问答对，会有一些细节丢失，请确认！如果余额不足，未完成的任务会被暂停。`
  });

-  const [prompt, setPrompt] = useState('');
-
-  const previewQAPrompt = useMemo(() => {
-    return replaceVariable(Prompt_AgentQA.prompt, {
-      theme: prompt || Prompt_AgentQA.defaultTheme
-    });
-  }, [prompt]);
+  const [prompt, setPrompt] = useState(Prompt_AgentQA.description);

  return (
    <Box display={['block', 'flex']} h={['auto', '100%']}>
      <SelectorContainer fileExtension={fileExtension}>
        {/* prompt */}
-        <Box py={5}>
-          <Box mb={2}>
-            QA 拆分引导词{' '}
-            <MyTooltip label={previewQAPrompt} forceShow>
-              <InfoOutlineIcon ml={1} />
-            </MyTooltip>
+        <Box p={3} bg={'myWhite.600'} borderRadius={'md'}>
+          <Box mb={1} fontWeight={'bold'}>
+            QA 拆分引导词
          </Box>
-          <Flex alignItems={'center'} fontSize={'sm'}>
-            <Box mr={2}>文件主题</Box>
-            <Input
-              fontSize={'sm'}
-              flex={1}
-              placeholder={Prompt_AgentQA.defaultTheme}
-              bg={'myWhite.500'}
+          <Box whiteSpace={'pre-wrap'} fontSize={'sm'}>
+            <Textarea
              defaultValue={prompt}
-              onChange={(e) => setPrompt(e.target.value || '')}
+              rows={8}
+              fontSize={'sm'}
+              onChange={(e) => {
+                setPrompt(e.target.value);
+              }}
            />
-          </Flex>
+            <Box>{Prompt_AgentQA.fixedText}</Box>
+          </Box>
        </Box>
        {/* price */}
        <Flex py={5} alignItems={'center'}>
@@ -81,10 +71,7 @@ const QAImport = () => {
              重新生成预览
            </Button>
          )}
-          <Button
-            isDisabled={uploading}
-            onClick={openConfirm(() => onclickUpload({ prompt: previewQAPrompt }))}
-          >
+          <Button isDisabled={uploading} onClick={openConfirm(() => onclickUpload({ prompt }))}>
            {uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
          </Button>
        </Flex>
--- a/projects/app/src/pages/dataset/detail/components/Info.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Info.tsx
@@ -15,7 +15,7 @@ import { useToast } from '@/web/common/hooks/useToast';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { UseFormReturn } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
 import Avatar from '@/components/Avatar';
 import Tag from '@/components/Tag';
@@ -95,7 +95,7 @@ const Info = (
      }
      setBtnLoading(false);
    },
-    [updateDataset, datasetId, loadDatasetDetail, toast, loadDatasets]
+    [updateDataset, datasetId, toast, loadDatasets]
  );
  const saveSubmitError = useCallback(() => {
    // deep search message
@@ -119,7 +119,7 @@ const Info = (
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/dataset/detail/components/Test.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Test.tsx
@@ -81,43 +81,45 @@ const Test = ({ datasetId }: { datasetId: string }) => {
        borderRight={['none', theme.borders.base]}
      >
        <Box border={'2px solid'} borderColor={'myBlue.600'} p={3} mx={4} borderRadius={'md'}>
-          <Box fontSize={'sm'} fontWeight={'bold'}>
-            <MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'myBlue.700'} />
-            测试文本
-          </Box>
-          <Textarea
-            rows={6}
-            resize={'none'}
-            variant={'unstyled'}
-            maxLength={datasetDetail.vectorModel.maxToken}
-            placeholder="输入需要测试的文本"
-            value={inputText}
-            onChange={(e) => setInputText(e.target.value)}
-          />
-          <Flex alignItems={'center'} justifyContent={'flex-end'}>
+          <Flex alignItems={'center'}>
+            <Box fontSize={'sm'} fontWeight={'bold'} flex={1}>
+              <MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'myBlue.700'} />
+              {t('core.dataset.test.Test Text')}
+            </Box>
            {feConfigs?.isPlus && (
              <Flex alignItems={'center'}>
                {t('dataset.recall.rerank')}
                <Switch ml={1} isChecked={rerank} onChange={(e) => setRerank(e.target.checked)} />
              </Flex>
            )}
+          </Flex>
+          <Textarea
+            rows={6}
+            resize={'none'}
+            variant={'unstyled'}
+            maxLength={datasetDetail.vectorModel.maxToken}
+            placeholder={t('core.dataset.test.Test Text Placeholder')}
+            value={inputText}
+            onChange={(e) => setInputText(e.target.value)}
+          />
+          <Flex alignItems={'center'} justifyContent={'flex-end'}>
            <Box mx={3} color={'myGray.500'}>
              {inputText.length}
            </Box>
            <Button isDisabled={inputText === ''} isLoading={isLoading} onClick={mutate}>
-              测试
+              {t('core.dataset.test.Test')}
            </Button>
          </Flex>
        </Box>
        <Box mt={5} flex={'1 0 0'} px={4} overflow={'overlay'} display={['none', 'block']}>
          <Flex alignItems={'center'} color={'myGray.600'}>
            <MyIcon mr={2} name={'history'} w={'16px'} h={'16px'} />
-            <Box fontSize={'2xl'}>测试历史</Box>
+            <Box fontSize={'2xl'}>{t('core.dataset.test.test history')}</Box>
          </Flex>
          <Box mt={2}>
            <Flex py={2} fontWeight={'bold'} borderBottom={theme.borders.sm}>
-              <Box flex={1}>测试文本</Box>
-              <Box w={'80px'}>时间</Box>
+              <Box flex={1}>{t('core.dataset.test.Test Text')}</Box>
+              <Box w={'80px'}>{t('common.Time')}</Box>
              <Box w={'14px'}></Box>
            </Flex>
            {kbTestHistory.map((item) => (
@@ -139,7 +141,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                  {item.text}
                </Box>
                <Box w={'80px'}>{formatTimeToChatTime(item.time)}</Box>
-                <MyTooltip label={'删除该测试记录'}>
+                <MyTooltip label={t('core.dataset.test.delete test history')}>
                  <Box w={'14px'} h={'14px'}>
                    <MyIcon
                      className="delete"
@@ -171,7 +173,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
          >
            <MyIcon name={'empty'} color={'transparent'} w={'54px'} />
            <Box mt={3} color={'myGray.600'}>
-              测试结果将在这里展示
+              {t('core.dataset.test.test result placeholder')}
            </Box>
          </Flex>
        ) : (
@@ -180,12 +182,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
              <Box fontSize={'3xl'} color={'myGray.600'}>
                {t('core.dataset.test.Test Result')}
              </Box>
-              <MyTooltip
-                label={
-                  '根据知识库内容与测试文本的相似度进行排序，你可以根据测试结果调整对应的文本。\n注意：测试记录中的数据可能已经被修改过，点击某条测试数据后将展示最新的数据。'
-                }
-                forceShow
-              >
+              <MyTooltip label={t('core.dataset.test.test result tip')} forceShow>
                <QuestionOutlineIcon
                  mx={2}
                  color={'myGray.600'}
@@ -221,7 +218,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                      const data = await getDatasetDataItemById(item.id);

                      if (!data) {
-                        throw new Error('该数据已被删除');
+                        throw new Error(t('core.dataset.data.data is deleted'));
                      }

                      setEditInputData({
--- a/projects/app/src/pages/dataset/list/component/CreateModal.tsx
+++ b/projects/app/src/pages/dataset/list/component/CreateModal.tsx
@@ -11,7 +11,7 @@ import {
 } from '@chakra-ui/react';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useRouter } from 'next/router';
@@ -57,7 +57,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/plugin/list/component/EditModal.tsx
+++ b/projects/app/src/pages/plugin/list/component/EditModal.tsx
@@ -11,7 +11,7 @@ import {
 } from '@chakra-ui/react';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useRouter } from 'next/router';
@@ -136,7 +136,7 @@ const CreateModal = ({
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/service/events/generateQA.ts
+++ b/projects/app/src/service/events/generateQA.ts
@@ -118,17 +118,14 @@ export async function generateQA(): Promise<any> {
  try {
    const startTime = Date.now();
    const model = data.model ?? global.qaModels[0].model;
+    const prompt = `${data.prompt || Prompt_AgentQA.description}
+${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;

    // request LLM to get QA
    const messages: ChatMessageItemType[] = [
      {
        role: 'user',
-        content: data.prompt
-          ? replaceVariable(data.prompt, { text })
-          : replaceVariable(Prompt_AgentQA.prompt, {
-              theme: Prompt_AgentQA.defaultTheme,
-              text
-            })
+        content: prompt
      }
    ];

@@ -244,7 +241,7 @@ function formatSplitText(text: string, rawText: string) {

  // empty result. direct split chunk
  if (result.length === 0) {
-    const splitRes = splitText2Chunks({ text: rawText, maxLen: 500 });
+    const splitRes = splitText2Chunks({ text: rawText, chunkLen: 512 });
    splitRes.chunks.forEach((chunk) => {
      result.push({
        q: chunk,
--- a/projects/app/src/web/common/file/controller.ts
+++ b/projects/app/src/web/common/file/controller.ts
@@ -33,74 +33,98 @@ export const uploadFiles = ({
 * compress image. response base64
 * @param maxSize The max size of the compressed image
 */
-export const compressImgAndUpload = ({
-  file,
+export const compressBase64ImgAndUpload = ({
+  base64,
  maxW = 200,
  maxH = 200,
  maxSize = 1024 * 100, // 100kb
  expiredTime
+}: {
+  base64: string;
+  maxW?: number;
+  maxH?: number;
+  maxSize?: number;
+  expiredTime?: Date;
+}) => {
+  return new Promise<string>((resolve, reject) => {
+    const fileType = /^data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,/.exec(base64)?.[1] || 'image/jpeg';
+
+    const img = new Image();
+    img.src = base64;
+    img.onload = async () => {
+      let width = img.width;
+      let height = img.height;
+
+      if (width > height) {
+        if (width > maxW) {
+          height *= maxW / width;
+          width = maxW;
+        }
+      } else {
+        if (height > maxH) {
+          width *= maxH / height;
+          height = maxH;
+        }
+      }
+
+      const canvas = document.createElement('canvas');
+      canvas.width = width;
+      canvas.height = height;
+      const ctx = canvas.getContext('2d');
+
+      if (!ctx) {
+        return reject('压缩图片异常');
+      }
+
+      ctx.drawImage(img, 0, 0, width, height);
+      const compressedDataUrl = canvas.toDataURL(fileType, 0.8);
+      // 移除 canvas 元素
+      canvas.remove();
+
+      if (compressedDataUrl.length > maxSize) {
+        return reject('图片太大了');
+      }
+
+      try {
+        const src = await postUploadImg(compressedDataUrl, expiredTime);
+        resolve(src);
+      } catch (error) {
+        reject(error);
+      }
+    };
+  });
+};
+export const compressImgFileAndUpload = async ({
+  file,
+  maxW,
+  maxH,
+  maxSize,
+  expiredTime
 }: {
  file: File;
  maxW?: number;
  maxH?: number;
  maxSize?: number;
  expiredTime?: Date;
-}) =>
-  new Promise<string>((resolve, reject) => {
-    const reader = new FileReader();
-    reader.readAsDataURL(file);
+}) => {
+  const reader = new FileReader();
+  reader.readAsDataURL(file);
+
+  const base64 = await new Promise<string>((resolve, reject) => {
    reader.onload = async () => {
-      const img = new Image();
-      // @ts-ignore
-      img.src = reader.result;
-      img.onload = async () => {
-        let width = img.width;
-        let height = img.height;
-
-        if (width > height) {
-          if (width > maxW) {
-            height *= maxW / width;
-            width = maxW;
-          }
-        } else {
-          if (height > maxH) {
-            width *= maxH / height;
-            height = maxH;
-          }
-        }
-
-        const canvas = document.createElement('canvas');
-        canvas.width = width;
-        canvas.height = height;
-        const ctx = canvas.getContext('2d');
-
-        if (!ctx) {
-          return reject('压缩图片异常');
-        }
-
-        ctx.drawImage(img, 0, 0, width, height);
-        const compressedDataUrl = canvas.toDataURL(file.type, 0.8);
-        // 移除 canvas 元素
-        canvas.remove();
-
-        if (compressedDataUrl.length > maxSize) {
-          return reject('图片太大了');
-        }
-
-        const src = await (async () => {
-          try {
-            const src = await postUploadImg(compressedDataUrl, expiredTime);
-            return src;
-          } catch (error) {
-            return compressedDataUrl;
-          }
-        })();
-
-        resolve(src);
-      };
+      resolve(reader.result as string);
    };
    reader.onerror = (err) => {
      console.log(err);
      reject('压缩图片异常');
    };
  });
+
+  return compressBase64ImgAndUpload({
+    base64,
+    maxW,
+    maxH,
+    maxSize,
+    expiredTime
+  });
+};
--- a/projects/app/src/web/common/file/utils.ts
+++ b/projects/app/src/web/common/file/utils.ts
@@ -1,6 +1,6 @@
 import mammoth from 'mammoth';
 import Papa from 'papaparse';
-import { postUploadImg } from '@/web/common/file/api';
+import { compressBase64ImgAndUpload } from './controller';

 /**
 * 读取 txt 文件内容
@@ -51,16 +51,30 @@ export const readPdfContent = (file: File) =>
        const headerThreshold = pageHeight * 0.07; // 假设页头在页面顶部5%的区域内
        const footerThreshold = pageHeight * 0.93; // 假设页脚在页面底部5%的区域内

-        const pageText = tokenizedText.items
-          .filter((token: TokenType) => {
-            return (
-              !token.transform ||
-              (token.transform[5] > headerThreshold && token.transform[5] < footerThreshold)
-            );
+        const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => {
+          return (
+            !token.transform ||
+            (token.transform[5] > headerThreshold && token.transform[5] < footerThreshold)
+          );
+        });
+
+        // concat empty string 'hasEOL'
+        for (let i = 0; i < pageTexts.length; i++) {
+          const item = pageTexts[i];
+          if (item.str === '' && pageTexts[i - 1]) {
+            pageTexts[i - 1].hasEOL = item.hasEOL;
+            pageTexts.splice(i, 1);
+            i--;
+          }
+        }
+
+        return pageTexts
+          .map((token) => {
+            const paragraphEnd = token.hasEOL && /([。？！.?!\n\r]|(\r\n))$/.test(token.str);
+
+            return paragraphEnd ? `${token.str}\n` : token.str;
          })
-          .map((token: TokenType) => token.str)
          .join('');
-        return pageText;
      };

      let reader = new FileReader();
@@ -100,10 +114,41 @@ export const readDocContent = (file: File) =>
      reader.onload = async ({ target }) => {
        if (!target?.result) return reject('读取 doc 文件失败');
        try {
-          const res = await mammoth.extractRawText({
+          // @ts-ignore
+          const res = await mammoth.convertToMarkdown({
            arrayBuffer: target.result as ArrayBuffer
          });
-          resolve(res?.value);
+
+          let rawText: string = res?.value || '';
+
+          // match base64, upload and replace it
+          const base64Regex = /data:image\/[a-zA-Z]+;base64,([^\)]+)/g;
+          const base64Arr = rawText.match(base64Regex) || [];
+
+          // upload base64 and replace it
+          await Promise.all(
+            base64Arr.map(async (base64) => {
+              try {
+                const str = await compressBase64ImgAndUpload({
+                  base64,
+                  maxW: 800,
+                  maxH: 800,
+                  maxSize: 1024 * 1024 * 2
+                });
+                rawText = rawText.replace(base64, str);
+              } catch (error) {
+                rawText = rawText.replace(base64, '');
+                rawText = rawText.replaceAll('![]()', '');
+              }
+            })
+          );
+
+          const trimReg = /\s*(!\[.*\]\(.*\))\s*/g;
+          if (trimReg.test(rawText)) {
+            rawText = rawText.replace(/\s*(!\[.*\]\(.*\))\s*/g, '$1');
+          }
+
+          resolve(rawText);
        } catch (error) {
          window.umami?.track('wordReadError', {
            err: error?.toString()