feat: url fetch and create file (#199)

* docs * docs * feat: url fetch and create file
2025-07-22 04:06:18 +00:00 · 2023-08-19 12:54:24 +08:00
parent 4054eb9d18
commit 1fcdd7cb8d
20 changed files with 583 additions and 359 deletions
--- a/README.md
+++ b/README.md
@@ -119,3 +119,4 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
 1. 允许作为后台服务直接商用，但不允许直接使用 saas 服务商用。
 2. 需保留相关版权信息。
 3. 完整请查看 [FstGPT Open Source License](./LICENSE)
+4. 联系方式：yujinlong@sealos.io, [点击查看定价策略](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)
--- a/client/public/imgs/files/url.svg
+++ b/client/public/imgs/files/url.svg
@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1692418843591" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4084" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M511.5 82c-236.6 0-429 192.4-429 429 0 236.5 192.5 429 429 429 236.6 0 429-192.4 429-429 0-236.5-192.4-429-429-429z m377.6 403.8H734.3c-4-139.9-41.4-259.9-97.5-331.9C776.5 203 879 332 889.1 485.8z m-402.8-349v349h-147c5.5-175.5 68.6-322.6 147-349z m0 399.4v349c-78.4-26.4-141.4-173.5-147-349h147z m50.5 349v-349h147c-5.6 175.5-68.6 322.6-147 349z m0-399.4v-349c78.4 26.4 141.4 173.5 147 349h-147zM386.3 153.9c-56.1 72-93.5 192-97.5 331.9H133.9C144.1 332 246.5 203 386.3 153.9zM133.9 536.2h154.8c4 139.9 41.4 259.9 97.5 331.9C246.5 819 144.1 690 133.9 536.2z m502.8 331.9c56.1-72 93.5-192 97.5-331.9H889C879 690 776.5 819 636.7 868.1z" fill="#5F9BEB" p-id="4085"></path></svg>
--- a/client/public/locales/en/common.json
+++ b/client/public/locales/en/common.json
@@ -54,9 +54,14 @@
  },
  "file": {
    "Click to download CSV template": "Click to download CSV template",
-    "Drag and drop": "Drag and drop files here, or click",
+    "Create File": "Create File",
+    "Create file": "Create file",
+    "Drag and drop": "Drag and drop files here",
+    "Fetch Url": "Fetch Url",
    "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
    "Release the mouse to upload the file": "Release the mouse to upload the file",
+    "Select a maximum of 10 files": "Select a maximum of 10 files",
+    "max 10": "Max 10 files",
    "select a document": "select a document",
    "support": "support {{fileExtension}} file",
    "upload error description": "Only upload multiple files or one folder at a time"
--- a/client/public/locales/zh/common.json
+++ b/client/public/locales/zh/common.json
@@ -54,9 +54,14 @@
  },
  "file": {
    "Click to download CSV template": "点击下载 CSV 模板",
-    "Drag and drop": "拖拽文件至此，或点击",
+    "Create File": "创建新文件",
+    "Create file": "创建文件",
+    "Drag and drop": "拖拽文件至此",
+    "Fetch Url": "链接读取",
    "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
    "Release the mouse to upload the file": "松开鼠标上传文件",
+    "Select a maximum of 10 files": "最多选择10个文件",
+    "max 10": "最多选择 10 个文件",
    "select a document": "选择文件",
    "support": "支持 {{fileExtension}} 文件",
    "upload error description": "单次只支持上传多个文件或者一个文件夹"
--- a/client/src/api/plugins/common.ts
+++ b/client/src/api/plugins/common.ts
@@ -0,0 +1,6 @@
+import { GET, POST, PUT, DELETE } from '../request';
+
+import type { FetchResultItem } from '@/types/plugin';
+
+export const fetchUrls = (urlList: string[]) =>
+  POST<FetchResultItem[]>(`/plugins/urlFetch`, { urlList });
--- a/client/src/constants/flow/ModuleTemplate.ts
+++ b/client/src/constants/flow/ModuleTemplate.ts
@@ -182,7 +182,7 @@ export const ChatModule: FlowModuleTemplateType = {
    {
      key: TaskResponseKeyEnum.answerText,
      label: '模型回复',
-      description: '如果外接了内容，会在回复结束时自动添加\n\n',
+      description: '将在 stream 回复完毕后触发',
      valueType: FlowValueTypeEnum.string,
      type: FlowOutputItemTypeEnum.source,
      targets: []
--- a/client/src/hooks/useSelectFile.tsx
+++ b/client/src/hooks/useSelectFile.tsx
@@ -1,8 +1,12 @@
 import React, { useRef, useCallback } from 'react';
 import { Box } from '@chakra-ui/react';
+import { useToast } from './useToast';
+import { useTranslation } from 'react-i18next';

 export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
+  const { t } = useTranslation();
  const { fileType = '*', multiple = false } = props || {};
+  const { toast } = useToast();
  const SelectFileDom = useRef<HTMLInputElement>(null);

  const File = useCallback(
@@ -15,12 +19,18 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean })
          multiple={multiple}
          onChange={(e) => {
            if (!e.target.files || e.target.files?.length === 0) return;
+            if (e.target.files.length > 10) {
+              return toast({
+                status: 'warning',
+                title: t('file.Select a maximum of 10 files')
+              });
+            }
            onSelect(Array.from(e.target.files));
          }}
        />
      </Box>
    ),
-    [fileType, multiple]
+    [fileType, multiple, t, toast]
  );

  const onOpen = useCallback(() => {
--- a/client/src/pages/api/plugins/urlFetch.ts
+++ b/client/src/pages/api/plugins/urlFetch.ts
@@ -5,12 +5,9 @@ import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
+import type { FetchResultItem } from '@/types/plugin';
+import { simpleText } from '@/utils/file';

-type FetchResultItem = {
-  url: string;
-  title: string;
-  content: string;
-};
 export type UrlFetchResponse = FetchResultItem[];

 const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
@@ -38,10 +35,11 @@ const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
          const reader = new Readability(dom.window.document);
          const article = reader.parse();

+          const content = article?.textContent || '';
+
          return {
            url,
-            title: article?.title || '',
-            content: article?.textContent || ''
+            content: simpleText(`${article?.title}\n${content}`)
          };
        })
      )
--- a/client/src/pages/kb/detail/components/DataCard.tsx
+++ b/client/src/pages/kb/detail/components/DataCard.tsx
@@ -217,7 +217,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
              <Box color={'myGray.600'}>{item.a}</Box>
            </Box>
            <Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
-              <Box className={'textEllipsis'} flex={1}>
+              <Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
                {item.source?.trim()}
              </Box>
              <IconButton
--- a/client/src/pages/kb/detail/components/Import/Chunk.tsx
+++ b/client/src/pages/kb/detail/components/Import/Chunk.tsx
@@ -13,7 +13,7 @@ import {
 } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
 import { useConfirm } from '@/hooks/useConfirm';
-import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
+import { useRouter } from 'next/router';
 import { useMutation } from '@tanstack/react-query';
 import { postKbDataFromList } from '@/api/plugins/kb';
 import { splitText2Chunks } from '@/utils/file';
@@ -25,26 +25,13 @@ import CloseIcon from '@/components/Icon/close';
 import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { fileImgs } from '@/constants/common';
-import { customAlphabet } from 'nanoid';
 import { TrainingModeEnum } from '@/constants/plugin';
-import FileSelect from './FileSelect';
-import { useRouter } from 'next/router';
-const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+import FileSelect, { type FileItemType } from './FileSelect';

 const fileExtension = '.txt, .doc, .docx, .pdf, .md';

-type FileItemType = {
-  id: string;
-  filename: string;
-  text: string;
-  icon: string;
-  chunks: string[];
-  tokens: number;
-};
-
 const ChunkImport = ({ kbId }: { kbId: string }) => {
-  const model = vectorModelList[0]?.model;
+  const model = vectorModelList[0]?.model || 'text-embedding-ada-002';
  const unitPrice = vectorModelList[0]?.price || 0.2;
  const theme = useTheme();
  const router = useRouter();
@@ -52,7 +39,6 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {

  const [chunkLen, setChunkLen] = useState(500);
  const [showRePreview, setShowRePreview] = useState(false);
-  const [selecting, setSelecting] = useState(false);
  const [files, setFiles] = useState<FileItemType[]>([]);
  const [previewFile, setPreviewFile] = useState<FileItemType>();
  const [successChunks, setSuccessChunks] = useState(0);
@@ -72,73 +58,9 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
    content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。`
  });

-  const onSelectFile = useCallback(
-    async (files: File[]) => {
-      setSelecting(true);
-      try {
-        let promise = Promise.resolve();
-        files.forEach((file) => {
-          promise = promise.then(async () => {
-            const extension = file?.name?.split('.')?.pop()?.toLowerCase();
-            const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src;
-            const text = await (async () => {
-              switch (extension) {
-                case 'txt':
-                case 'md':
-                  return readTxtContent(file);
-                case 'pdf':
-                  return readPdfContent(file);
-                case 'doc':
-                case 'docx':
-                  return readDocContent(file);
-              }
-              return '';
-            })();
-
-            if (icon && text) {
-              const splitRes = splitText2Chunks({
-                text: text,
-                maxLen: chunkLen
-              });
-
-              setFiles((state) => [
-                {
-                  id: nanoid(),
-                  filename: file.name,
-                  text,
-                  icon,
-                  ...splitRes
-                },
-                ...state
-              ]);
-            }
-          });
-        });
-        await promise;
-      } catch (error: any) {
-        console.log(error);
-        toast({
-          title: typeof error === 'string' ? error : '解析文件失败',
-          status: 'error'
-        });
-      }
-      setSelecting(false);
-    },
-    [chunkLen, toast]
-  );
-
  const { mutate: onclickUpload, isLoading: uploading } = useMutation({
    mutationFn: async () => {
-      const chunks: { a: string; q: string; source: string }[] = [];
-      files.forEach((file) =>
-        file.chunks.forEach((chunk) => {
-          chunks.push({
-            q: chunk,
-            a: '',
-            source: file.filename
-          });
-        })
-      );
+      const chunks = files.map((file) => file.chunks).flat();

      // subsection import
      let success = 0;
@@ -177,18 +99,22 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {

  const onRePreview = useCallback(async () => {
    try {
-      const splitRes = files.map((item) =>
-        splitText2Chunks({
-          text: item.text,
-          maxLen: chunkLen
-        })
-      );
-
      setFiles((state) =>
-        state.map((file, index) => ({
-          ...file,
-          ...splitRes[index]
-        }))
+        state.map((file) => {
+          const splitRes = splitText2Chunks({
+            text: file.text,
+            maxLen: chunkLen
+          });
+          return {
+            ...file,
+            tokens: splitRes.tokens,
+            chunks: splitRes.chunks.map((chunk) => ({
+              q: chunk,
+              a: '',
+              source: file.filename
+            }))
+          };
+        })
      );
      setPreviewFile(undefined);
      setShowRePreview(false);
@@ -198,7 +124,12 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
        title: getErrText(error, '文本分段异常')
      });
    }
-  }, [chunkLen, files, toast]);
+  }, [chunkLen, toast]);
+
+  const filenameStyles = {
+    className: 'textEllipsis',
+    maxW: '400px'
+  };

  return (
    <Box display={['block', 'flex']} h={['auto', '100%']}>
@@ -212,8 +143,10 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
      >
        <FileSelect
          fileExtension={fileExtension}
-          onSelectFile={onSelectFile}
-          isLoading={selecting}
+          onPushFiles={(files) => {
+            setFiles((state) => files.concat(state));
+          }}
+          chunkLen={chunkLen}
          py={emptyFiles ? '100px' : 5}
        />

@@ -241,7 +174,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
                  onClick={() => setPreviewFile(item)}
                >
                  <Image src={item.icon} w={'16px'} alt={''} />
-                  <Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis">
+                  <Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
                    {item.filename}
                  </Box>
                  <MyIcon
@@ -333,7 +266,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
              pt={[4, 8]}
              bg={'myWhite.400'}
            >
-              <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
+              <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'} {...filenameStyles}>
                {previewFile.filename}
              </Box>
              <CloseIcon
@@ -373,12 +306,19 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
            </Box>
          ) : (
            <Box h={'100%'} pt={[4, 8]} overflow={'overlay'}>
-              <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
-                分段预览({totalChunk}组)
-              </Box>
+              <Flex px={[4, 8]} alignItems={'center'}>
+                <Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
+                  分段预览({totalChunk}组)
+                </Box>
+                {totalChunk > 100 && (
+                  <Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
+                    仅展示部分
+                  </Box>
+                )}
+              </Flex>
              <Box px={[4, 8]} overflow={'overlay'}>
                {files.map((file) =>
-                  file.chunks.map((item, i) => (
+                  file.chunks.slice(0, 50).map((chunk, i) => (
                    <Box
                      key={i}
                      py={4}
@@ -389,9 +329,18 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
                      _hover={{ ...hoverDeleteStyles }}
                    >
                      <Flex mb={1} px={4} userSelect={'none'}>
-                        <Box px={3} py={'1px'} border={theme.borders.base} borderRadius={'md'}>
+                        <Box
+                          flexShrink={0}
+                          px={3}
+                          py={'1px'}
+                          border={theme.borders.base}
+                          borderRadius={'md'}
+                        >
                          # {i + 1}
                        </Box>
+                        <Box ml={2} fontSize={'sm'} color={'myhGray.500'} {...filenameStyles}>
+                          {file.filename}
+                        </Box>
                        <Box flex={1} />
                        <DeleteIcon
                          onClick={() => {
@@ -417,11 +366,12 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
                        whiteSpace={'pre-wrap'}
                        wordBreak={'break-all'}
                        contentEditable
-                        dangerouslySetInnerHTML={{ __html: item }}
+                        dangerouslySetInnerHTML={{ __html: chunk.q }}
                        onBlur={(e) => {
                          // @ts-ignore
                          const val = e.target.innerText;

+                          /* delete file */
                          if (val === '') {
                            setFiles((state) =>
                              state.map((stateFile) =>
@@ -437,14 +387,16 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
                              )
                            );
                          } else {
-                            setFiles((state) =>
-                              state.map((stateFile) =>
-                                stateFile.id === file.id
+                            // update file
+                            setFiles((stateFiles) =>
+                              stateFiles.map((stateFile) =>
+                                file.id === stateFile.id
                                  ? {
-                                      ...file,
-                                      chunks: file.chunks.map((chunk, index) =>
-                                        i === index ? val : chunk
-                                      )
+                                      ...stateFile,
+                                      chunks: stateFile.chunks.map((chunk, index) => ({
+                                        ...chunk,
+                                        q: i === index ? val : chunk.q
+                                      }))
                                    }
                                  : stateFile
                              )
--- a/client/src/pages/kb/detail/components/Import/CreateFileModal.tsx
+++ b/client/src/pages/kb/detail/components/Import/CreateFileModal.tsx
@@ -0,0 +1,63 @@
+import React from 'react';
+import { useTranslation } from 'next-i18next';
+import MyModal from '@/components/MyModal';
+import { Box, Input, Textarea, ModalBody, ModalFooter, Button } from '@chakra-ui/react';
+import { useForm } from 'react-hook-form';
+
+const CreateFileModal = ({
+  onClose,
+  onSuccess
+}: {
+  onClose: () => void;
+  onSuccess: (e: { filename: string; content: string }) => void;
+}) => {
+  const { t } = useTranslation();
+  const { register, handleSubmit } = useForm({
+    defaultValues: {
+      filename: '',
+      content: ''
+    }
+  });
+
+  return (
+    <MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}>
+      <ModalBody>
+        <Box mb={1} fontSize={'sm'}>
+          文件名
+        </Box>
+        <Input
+          mb={5}
+          {...register('filename', {
+            required: '文件名不能为空'
+          })}
+        />
+        <Box mb={1} fontSize={'sm'}>
+          文件内容
+        </Box>
+        <Textarea
+          {...register('content', {
+            required: '文件内容不能为空'
+          })}
+          rows={12}
+          whiteSpace={'nowrap'}
+          resize={'both'}
+        />
+      </ModalBody>
+      <ModalFooter>
+        <Button variant={'base'} mr={4} onClick={onClose}>
+          取消
+        </Button>
+        <Button
+          onClick={() => {
+            handleSubmit(onSuccess)();
+            onClose();
+          }}
+        >
+          确认
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};
+
+export default CreateFileModal;
--- a/client/src/pages/kb/detail/components/Import/Csv.tsx
+++ b/client/src/pages/kb/detail/components/Import/Csv.tsx
@@ -8,28 +8,18 @@ import { getErrText } from '@/utils/tools';
 import { vectorModelList } from '@/store/static';
 import MyIcon from '@/components/Icon';
 import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
-import { customAlphabet } from 'nanoid';
 import { TrainingModeEnum } from '@/constants/plugin';
-import FileSelect from './FileSelect';
+import FileSelect, { type FileItemType } from './FileSelect';
 import { useRouter } from 'next/router';
-const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
-import { readCsvContent } from '@/utils/file';

 const fileExtension = '.csv';

-type FileItemType = {
-  id: string;
-  filename: string;
-  chunks: { q: string; a: string; source?: string }[];
-};
-
 const CsvImport = ({ kbId }: { kbId: string }) => {
  const model = vectorModelList[0]?.model;
  const theme = useTheme();
  const router = useRouter();
  const { toast } = useToast();

-  const [selecting, setSelecting] = useState(false);
  const [files, setFiles] = useState<FileItemType[]>([]);
  const [successChunks, setSuccessChunks] = useState(0);

@@ -43,58 +33,9 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
    content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。`
  });

-  const onSelectFile = useCallback(
-    async (files: File[]) => {
-      setSelecting(true);
-      try {
-        let promise = Promise.resolve();
-        files
-          .filter((file) => /csv$/.test(file.name))
-          .forEach((file) => {
-            promise = promise.then(async () => {
-              const { header, data } = await readCsvContent(file);
-              if (header[0] !== 'question' || header[1] !== 'answer') {
-                throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
-              }
-
-              setFiles((state) => [
-                {
-                  id: nanoid(),
-                  filename: file.name,
-                  chunks: data.map((item) => ({
-                    q: item[0],
-                    a: item[1],
-                    source: item[2]
-                  }))
-                },
-                ...state
-              ]);
-            });
-          });
-        await promise;
-      } catch (error: any) {
-        console.log(error);
-        toast({
-          title: getErrText(error, '解析文件失败'),
-          status: 'error'
-        });
-      }
-      setSelecting(false);
-    },
-    [toast]
-  );
-
  const { mutate: onclickUpload, isLoading: uploading } = useMutation({
    mutationFn: async () => {
-      const chunks: { a: string; q: string; source: string }[] = [];
-      files.forEach((file) =>
-        file.chunks.forEach((chunk) => {
-          chunks.push({
-            ...chunk,
-            source: chunk.source || file.filename
-          });
-        })
-      );
+      const chunks = files.map((file) => file.chunks).flat();

      // subsection import
      let success = 0;
@@ -131,6 +72,10 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
    }
  });

+  const filenameStyles = {
+    className: 'textEllipsis',
+    maxW: '400px'
+  };
  return (
    <Box display={['block', 'flex']} h={['auto', '100%']}>
      <Flex
@@ -143,11 +88,12 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
      >
        <FileSelect
          fileExtension={fileExtension}
-          onSelectFile={onSelectFile}
-          isLoading={selecting}
          tipText={
            'file.If the imported file is garbled, please convert CSV to UTF-8 encoding format'
          }
+          onPushFiles={(files) => setFiles((state) => files.concat(state))}
+          showUrlFetch={false}
+          showCreateFile={false}
          py={emptyFiles ? '100px' : 5}
          isCsv
        />
@@ -169,7 +115,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
                  _hover={{ ...hoverDeleteStyles }}
                >
                  <Image src={'/imgs/files/csv.svg'} w={'16px'} alt={''} />
-                  <Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis">
+                  <Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
                    {item.filename}
                  </Box>
                  <MyIcon
@@ -203,9 +149,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
      </Flex>
      {!emptyFiles && (
        <Box flex={'2 0 0'} w={['100%', 0]} h={'100%'} pt={[4, 8]} overflow={'overlay'}>
-          <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
-            数据预览({totalChunk}组)
-          </Box>
+          <Flex px={[4, 8]} alignItems={'center'}>
+            <Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
+              分段预览({totalChunk}组)
+            </Box>
+            {totalChunk > 100 && (
+              <Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
+                仅展示部分
+              </Box>
+            )}
+          </Flex>
          <Box px={[4, 8]} overflow={'overlay'}>
            {files.map((file) =>
              file.chunks.slice(0, 100).map((item, i) => (
--- a/client/src/pages/kb/detail/components/Import/FileSelect.tsx
+++ b/client/src/pages/kb/detail/components/Import/FileSelect.tsx
@@ -2,30 +2,53 @@ import MyIcon from '@/components/Icon';
 import { useLoading } from '@/hooks/useLoading';
 import { useSelectFile } from '@/hooks/useSelectFile';
 import { useToast } from '@/hooks/useToast';
-import { fileDownload } from '@/utils/file';
-import { Box, Flex, Text, type BoxProps } from '@chakra-ui/react';
+import { fileDownload, readCsvContent, simpleText, splitText2Chunks } from '@/utils/file';
+import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
+import { fileImgs } from '@/constants/common';
 import { DragEvent, useCallback, useState } from 'react';
 import { useTranslation } from 'next-i18next';
+import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
+import { customAlphabet } from 'nanoid';
+import dynamic from 'next/dynamic';
+import MyTooltip from '@/components/MyTooltip';
+import { FetchResultItem } from '@/types/plugin';

+const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
+const CreateFileModal = dynamic(() => import('./CreateFileModal'));
+
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;
+
+export type FileItemType = {
+  id: string;
+  filename: string;
+  chunks: { q: string; a: string; source?: string }[];
+  text: string;
+  icon: string;
+  tokens: number;
+};
 interface Props extends BoxProps {
  fileExtension: string;
+  onPushFiles: (files: FileItemType[]) => void;
  tipText?: string;
-  onSelectFile: (files: File[]) => Promise<void>;
-  isLoading?: boolean;
+  chunkLen?: number;
  isCsv?: boolean;
+  showUrlFetch?: boolean;
+  showCreateFile?: boolean;
 }

 const FileSelect = ({
  fileExtension,
-  onSelectFile,
-  isLoading,
+  onPushFiles,
  tipText,
+  chunkLen = 500,
  isCsv = false,
+  showUrlFetch = true,
+  showCreateFile = true,
  ...props
 }: Props) => {
  const { Loading: FileSelectLoading } = useLoading();
  const { t } = useTranslation();
-  const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;

  const { toast } = useToast();

@@ -35,6 +58,154 @@ const FileSelect = ({
  });

  const [isDragging, setIsDragging] = useState(false);
+  const [selecting, setSelecting] = useState(false);
+
+  const {
+    isOpen: isOpenUrlFetch,
+    onOpen: onOpenUrlFetch,
+    onClose: onCloseUrlFetch
+  } = useDisclosure();
+  const {
+    isOpen: isOpenCreateFile,
+    onOpen: onOpenCreateFile,
+    onClose: onCloseCreateFile
+  } = useDisclosure();
+
+  const onSelectFile = useCallback(
+    async (files: File[]) => {
+      setSelecting(true);
+      try {
+        // Parse file by file
+        let promise = Promise.resolve<FileItemType[]>([]);
+        files.forEach((file) => {
+          promise = promise.then(async (result) => {
+            const extension = file?.name?.split('.')?.pop()?.toLowerCase();
+
+            /* text file */
+            const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src;
+            let text = await (async () => {
+              switch (extension) {
+                case 'txt':
+                case 'md':
+                  return readTxtContent(file);
+                case 'pdf':
+                  return readPdfContent(file);
+                case 'doc':
+                case 'docx':
+                  return readDocContent(file);
+              }
+              return '';
+            })();
+
+            if (!icon) return result;
+
+            if (text) {
+              text = simpleText(text);
+              const splitRes = splitText2Chunks({
+                text,
+                maxLen: chunkLen
+              });
+              const fileItem: FileItemType = {
+                id: nanoid(),
+                filename: file.name,
+                icon,
+                text,
+                tokens: splitRes.tokens,
+                chunks: splitRes.chunks.map((chunk) => ({
+                  q: chunk,
+                  a: '',
+                  source: file.name
+                }))
+              };
+              return [fileItem].concat(result);
+            }
+
+            /* csv file */
+            if (extension === 'csv') {
+              const { header, data } = await readCsvContent(file);
+              if (header[0] !== 'question' || header[1] !== 'answer') {
+                throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
+              }
+              const fileItem: FileItemType = {
+                id: nanoid(),
+                filename: file.name,
+                icon,
+                tokens: 0,
+                text: '',
+                chunks: data.map((item) => ({
+                  q: item[0],
+                  a: item[1],
+                  source: item[2] || file.name
+                }))
+              };
+              return [fileItem].concat(result);
+            }
+            return result;
+          });
+        });
+
+        const chunkFiles = await promise;
+
+        onPushFiles(chunkFiles);
+      } catch (error: any) {
+        console.log(error);
+        toast({
+          title: typeof error === 'string' ? error : '解析文件失败',
+          status: 'error'
+        });
+      }
+      setSelecting(false);
+    },
+    [chunkLen, onPushFiles, toast]
+  );
+  const onUrlFetch = useCallback(
+    (e: FetchResultItem[]) => {
+      const result = e.map(({ url, content }) => {
+        const splitRes = splitText2Chunks({
+          text: content,
+          maxLen: chunkLen
+        });
+        return {
+          id: nanoid(),
+          filename: url,
+          icon: '/imgs/files/url.svg',
+          text: content,
+          tokens: splitRes.tokens,
+          chunks: splitRes.chunks.map((chunk) => ({
+            q: chunk,
+            a: '',
+            source: url
+          }))
+        };
+      });
+      onPushFiles(result);
+    },
+    [chunkLen, onPushFiles]
+  );
+  const onCreateFile = useCallback(
+    ({ filename, content }: { filename: string; content: string }) => {
+      content = simpleText(content);
+      const splitRes = splitText2Chunks({
+        text: content,
+        maxLen: chunkLen
+      });
+      onPushFiles([
+        {
+          id: nanoid(),
+          filename,
+          icon: '/imgs/files/txt.svg',
+          text: content,
+          tokens: splitRes.tokens,
+          chunks: splitRes.chunks.map((chunk) => ({
+            q: chunk,
+            a: '',
+            source: filename
+          }))
+        }
+      ]);
+    },
+    [chunkLen, onPushFiles]
+  );

  const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
    e.preventDefault();
@@ -46,56 +217,69 @@ const FileSelect = ({
    setIsDragging(false);
  };

-  const handleDrop = useCallback(async (e: DragEvent<HTMLDivElement>) => {
-    e.preventDefault();
-    setIsDragging(false);
+  const handleDrop = useCallback(
+    async (e: DragEvent<HTMLDivElement>) => {
+      e.preventDefault();
+      setIsDragging(false);

-    const items = e.dataTransfer.items;
-    const fileList: File[] = [];
+      const items = e.dataTransfer.items;
+      const fileList: File[] = [];

-    if (e.dataTransfer.items.length <= 1) {
-      const traverseFileTree = async (item: any) => {
-        return new Promise<void>((resolve, reject) => {
-          if (item.isFile) {
-            item.file((file: File) => {
-              fileList.push(file);
-              resolve();
-            });
-          } else if (item.isDirectory) {
-            const dirReader = item.createReader();
-            dirReader.readEntries(async (entries: any[]) => {
-              for (let i = 0; i < entries.length; i++) {
-                await traverseFileTree(entries[i]);
-              }
-              resolve();
-            });
+      if (e.dataTransfer.items.length <= 1) {
+        const traverseFileTree = async (item: any) => {
+          return new Promise<void>((resolve, reject) => {
+            if (item.isFile) {
+              item.file((file: File) => {
+                fileList.push(file);
+                resolve();
+              });
+            } else if (item.isDirectory) {
+              const dirReader = item.createReader();
+              dirReader.readEntries(async (entries: any[]) => {
+                for (let i = 0; i < entries.length; i++) {
+                  await traverseFileTree(entries[i]);
+                }
+                resolve();
+              });
+            }
+          });
+        };
+
+        for (let i = 0; i < items.length; i++) {
+          const item = items[i].webkitGetAsEntry();
+          if (item) {
+            await traverseFileTree(item);
          }
-        });
-      };
+        }
+      } else {
+        const files = Array.from(e.dataTransfer.files);
+        let isErr = files.some((item) => item.type === '');
+        if (isErr) {
+          return toast({
+            title: t('file.upload error description'),
+            status: 'error'
+          });
+        }

-      for (let i = 0; i < items.length; i++) {
-        const item = items[i].webkitGetAsEntry();
-        if (item) {
-          await traverseFileTree(item);
+        for (let i = 0; i < files.length; i++) {
+          fileList.push(files[i]);
        }
      }
-    } else {
-      const files = Array.from(e.dataTransfer.files);
-      let isErr = files.some((item) => item.type === '');
-      if (isErr) {
-        return toast({
-          title: t('file.upload error description'),
-          status: 'error'
-        });
-      }

-      for (let i = 0; i < files.length; i++) {
-        fileList.push(files[i]);
-      }
+      onSelectFile(fileList);
+    },
+    [onSelectFile, t, toast]
+  );
+
+  const SelectTextStyles: BoxProps = {
+    ml: 1,
+    as: 'span',
+    cursor: 'pointer',
+    color: 'myBlue.700',
+    _hover: {
+      textDecoration: 'underline'
    }
-
-    onSelectFile(fileList);
-  }, []);
+  };

  return (
    <Box
@@ -120,10 +304,28 @@ const FileSelect = ({
          t('file.Release the mouse to upload the file')
        ) : (
          <Box>
-            {t('file.Drag and drop')}
-            <Text ml={1} as={'span'} cursor={'pointer'} color={'myBlue.700'} onClick={onOpen}>
-              {t('file.select a document')}
-            </Text>
+            {t('file.Drag and drop')},
+            <MyTooltip label={t('file.max 10')}>
+              <Box {...SelectTextStyles} onClick={onOpen}>
+                {t('file.select a document')}
+              </Box>
+            </MyTooltip>
+            {showUrlFetch && (
+              <>
+                ,
+                <Box {...SelectTextStyles} onClick={onOpenUrlFetch}>
+                  {t('file.Fetch Url')}
+                </Box>
+              </>
+            )}
+            {showCreateFile && (
+              <>
+                ,
+                <Box {...SelectTextStyles} onClick={onOpenCreateFile}>
+                  {t('file.Create file')}
+                </Box>
+              </>
+            )}
          </Box>
        )}
      </Flex>
@@ -151,8 +353,10 @@ const FileSelect = ({
          {t('file.Click to download CSV template')}
        </Box>
      )}
-      <FileSelectLoading loading={isLoading} fixed={false} />
+      <FileSelectLoading loading={selecting} fixed={false} />
      <File onSelect={onSelectFile} />
+      {isOpenUrlFetch && <UrlFetchModal onClose={onCloseUrlFetch} onSuccess={onUrlFetch} />}
+      {isOpenCreateFile && <CreateFileModal onClose={onCloseCreateFile} onSuccess={onCreateFile} />}
    </Box>
  );
 };
--- a/client/src/pages/kb/detail/components/Import/QA.tsx
+++ b/client/src/pages/kb/detail/components/Import/QA.tsx
@@ -2,7 +2,6 @@ import React, { useState, useCallback, useMemo } from 'react';
 import { Box, Flex, Button, useTheme, Image, Input } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
 import { useConfirm } from '@/hooks/useConfirm';
-import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { useMutation } from '@tanstack/react-query';
 import { postKbDataFromList } from '@/api/plugins/kb';
 import { splitText2Chunks } from '@/utils/file';
@@ -14,24 +13,12 @@ import CloseIcon from '@/components/Icon/close';
 import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { fileImgs } from '@/constants/common';
-import { customAlphabet } from 'nanoid';
 import { TrainingModeEnum } from '@/constants/plugin';
-import FileSelect from './FileSelect';
+import FileSelect, { type FileItemType } from './FileSelect';
 import { useRouter } from 'next/router';
-const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);

 const fileExtension = '.txt, .doc, .docx, .pdf, .md';

-type FileItemType = {
-  id: string;
-  filename: string;
-  text: string;
-  icon: string;
-  chunks: string[];
-  tokens: number;
-};
-
 const QAImport = ({ kbId }: { kbId: string }) => {
  const model = qaModelList[0]?.model;
  const unitPrice = qaModelList[0]?.price || 3;
@@ -40,7 +27,6 @@ const QAImport = ({ kbId }: { kbId: string }) => {
  const router = useRouter();
  const { toast } = useToast();

-  const [selecting, setSelecting] = useState(false);
  const [files, setFiles] = useState<FileItemType[]>([]);
  const [showRePreview, setShowRePreview] = useState(false);
  const [previewFile, setPreviewFile] = useState<FileItemType>();
@@ -62,77 +48,13 @@ const QAImport = ({ kbId }: { kbId: string }) => {
    content: `该任务无法终止！导入后会自动调用大模型生成问答对，会有一些细节丢失，请确认！如果余额不足，未完成的任务会被暂停。`
  });

-  const onSelectFile = useCallback(
-    async (files: File[]) => {
-      setSelecting(true);
-      try {
-        let promise = Promise.resolve();
-        files.forEach((file) => {
-          promise = promise.then(async () => {
-            const extension = file?.name?.split('.')?.pop()?.toLowerCase();
-            const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src;
-            const text = await (async () => {
-              switch (extension) {
-                case 'txt':
-                case 'md':
-                  return readTxtContent(file);
-                case 'pdf':
-                  return readPdfContent(file);
-                case 'doc':
-                case 'docx':
-                  return readDocContent(file);
-              }
-              return '';
-            })();
-
-            if (icon && text) {
-              const splitRes = splitText2Chunks({
-                text: text,
-                maxLen: chunkLen
-              });
-
-              setFiles((state) => [
-                {
-                  id: nanoid(),
-                  filename: file.name,
-                  text,
-                  icon,
-                  ...splitRes
-                },
-                ...state
-              ]);
-            }
-          });
-        });
-        await promise;
-      } catch (error: any) {
-        console.log(error);
-        toast({
-          title: typeof error === 'string' ? error : '解析文件失败',
-          status: 'error'
-        });
-      }
-      setSelecting(false);
-    },
-    [chunkLen, toast]
-  );
-
  const { mutate: onclickUpload, isLoading: uploading } = useMutation({
    mutationFn: async () => {
-      const chunks: { a: string; q: string; source: string }[] = [];
-      files.forEach((file) =>
-        file.chunks.forEach((chunk) => {
-          chunks.push({
-            q: chunk,
-            a: '',
-            source: file.filename
-          });
-        })
-      );
+      const chunks = files.map((file) => file.chunks).flat();

      // subsection import
      let success = 0;
-      const step = 500;
+      const step = 300;
      for (let i = 0; i < chunks.length; i += step) {
        const { insertLen } = await postKbDataFromList({
          kbId,
@@ -168,18 +90,22 @@ const QAImport = ({ kbId }: { kbId: string }) => {

  const onRePreview = useCallback(async () => {
    try {
-      const splitRes = files.map((item) =>
-        splitText2Chunks({
-          text: item.text,
-          maxLen: chunkLen
-        })
-      );
-
      setFiles((state) =>
-        state.map((file, index) => ({
-          ...file,
-          ...splitRes[index]
-        }))
+        state.map((file) => {
+          const splitRes = splitText2Chunks({
+            text: file.text,
+            maxLen: chunkLen
+          });
+          return {
+            ...file,
+            tokens: splitRes.tokens,
+            chunks: splitRes.chunks.map((chunk) => ({
+              q: chunk,
+              a: '',
+              source: file.filename
+            }))
+          };
+        })
      );
      setPreviewFile(undefined);
      setShowRePreview(false);
@@ -189,7 +115,12 @@ const QAImport = ({ kbId }: { kbId: string }) => {
        title: getErrText(error, '文本分段异常')
      });
    }
-  }, [chunkLen, files, toast]);
+  }, [chunkLen, toast]);
+
+  const filenameStyles = {
+    className: 'textEllipsis',
+    maxW: '400px'
+  };

  return (
    <Box display={['block', 'flex']} h={['auto', '100%']}>
@@ -203,8 +134,10 @@ const QAImport = ({ kbId }: { kbId: string }) => {
      >
        <FileSelect
          fileExtension={fileExtension}
-          onSelectFile={onSelectFile}
-          isLoading={selecting}
+          onPushFiles={(files) => {
+            setFiles((state) => files.concat(state));
+          }}
+          chunkLen={chunkLen}
          py={emptyFiles ? '100px' : 5}
        />

@@ -232,7 +165,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
                  onClick={() => setPreviewFile(item)}
                >
                  <Image src={item.icon} w={'16px'} alt={''} />
-                  <Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis">
+                  <Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
                    {item.filename}
                  </Box>
                  <MyIcon
@@ -314,7 +247,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
              pt={[4, 8]}
              bg={'myWhite.400'}
            >
-              <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
+              <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'} {...filenameStyles}>
                {previewFile.filename}
              </Box>
              <CloseIcon
@@ -353,14 +286,21 @@ const QAImport = ({ kbId }: { kbId: string }) => {
            </Box>
          ) : (
            <Box h={'100%'} pt={[4, 8]} overflow={'overlay'}>
-              <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}>
-                分段预览({totalChunk}组)
-              </Box>
+              <Flex px={[4, 8]} alignItems={'center'}>
+                <Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
+                  分段预览({totalChunk}组)
+                </Box>
+                {totalChunk > 100 && (
+                  <Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
+                    仅展示部分
+                  </Box>
+                )}
+              </Flex>
              <Box px={[4, 8]} overflow={'overlay'}>
                {files.map((file) =>
-                  file.chunks.map((item, i) => (
+                  file.chunks.slice(0, 30).map((chunk, i) => (
                    <Box
-                      key={item}
+                      key={i}
                      py={4}
                      bg={'myWhite.500'}
                      my={2}
@@ -372,6 +312,9 @@ const QAImport = ({ kbId }: { kbId: string }) => {
                        <Box px={3} py={'1px'} border={theme.borders.base} borderRadius={'md'}>
                          # {i + 1}
                        </Box>
+                        <Box ml={2} fontSize={'sm'} color={'myhGray.500'} {...filenameStyles}>
+                          {file.filename}
+                        </Box>
                        <Box flex={1} />
                        <DeleteIcon
                          onClick={() => {
@@ -397,11 +340,12 @@ const QAImport = ({ kbId }: { kbId: string }) => {
                        whiteSpace={'pre-wrap'}
                        wordBreak={'break-all'}
                        contentEditable
-                        dangerouslySetInnerHTML={{ __html: item }}
+                        dangerouslySetInnerHTML={{ __html: chunk.q }}
                        onBlur={(e) => {
                          // @ts-ignore
                          const val = e.target.innerText;

+                          /* delete file */
                          if (val === '') {
                            setFiles((state) =>
                              state.map((stateFile) =>
@@ -417,14 +361,16 @@ const QAImport = ({ kbId }: { kbId: string }) => {
                              )
                            );
                          } else {
-                            setFiles((state) =>
-                              state.map((stateFile) =>
-                                stateFile.id === file.id
+                            // update file
+                            setFiles((stateFiles) =>
+                              stateFiles.map((stateFile) =>
+                                file.id === stateFile.id
                                  ? {
-                                      ...file,
-                                      chunks: file.chunks.map((chunk, index) =>
-                                        i === index ? val : chunk
-                                      )
+                                      ...stateFile,
+                                      chunks: stateFile.chunks.map((chunk, index) => ({
+                                        ...chunk,
+                                        q: i === index ? val : chunk.q
+                                      }))
                                    }
                                  : stateFile
                              )
--- a/client/src/pages/kb/detail/components/Import/UrlFetchModal.tsx
+++ b/client/src/pages/kb/detail/components/Import/UrlFetchModal.tsx
@@ -0,0 +1,67 @@
+import React, { useRef } from 'react';
+import { useTranslation } from 'next-i18next';
+import MyModal from '@/components/MyModal';
+import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
+import type { FetchResultItem } from '@/types/plugin';
+import { useRequest } from '@/hooks/useRequest';
+import { fetchUrls } from '@/api/plugins/common';
+
+const UrlFetchModal = ({
+  onClose,
+  onSuccess
+}: {
+  onClose: () => void;
+  onSuccess: (e: FetchResultItem[]) => void;
+}) => {
+  const { t } = useTranslation();
+  const Dom = useRef<HTMLTextAreaElement>(null);
+
+  const { mutate, isLoading } = useRequest({
+    mutationFn: async () => {
+      const val = Dom.current?.value || '';
+      const urls = val.split('\n').filter((e) => e);
+      const res = await fetchUrls(urls);
+
+      onSuccess(res);
+      onClose();
+    },
+    errorToast: '获取链接失败'
+  });
+
+  return (
+    <MyModal
+      title={
+        <>
+          <Box>{t('file.Fetch Url')}</Box>
+          <Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
+            目前仅支持读取静态链接，请注意检查结果
+          </Box>
+        </>
+      }
+      top={'15vh'}
+      isOpen
+      onClose={onClose}
+      w={'600px'}
+    >
+      <ModalBody>
+        <Textarea
+          ref={Dom}
+          rows={12}
+          whiteSpace={'nowrap'}
+          resize={'both'}
+          placeholder={'最多10个链接，每行一个。'}
+        />
+      </ModalBody>
+      <ModalFooter>
+        <Button variant={'base'} mr={4} onClick={onClose}>
+          取消
+        </Button>
+        <Button isLoading={isLoading} onClick={mutate}>
+          确认
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};
+
+export default UrlFetchModal;
--- a/client/src/types/plugin.d.ts
+++ b/client/src/types/plugin.d.ts
@@ -28,3 +28,8 @@ export type KbTestItemType = {
  time: Date;
  results: (KbDataItemType & { score: number })[];
 };
+
+export type FetchResultItem = {
+  url: string;
+  content: string;
+};
--- a/client/src/utils/file.ts
+++ b/client/src/utils/file.ts
@@ -151,7 +151,7 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
  const overlapLen = Math.floor(maxLen * 0.3); // Overlap length

  try {
-    const splitTexts = text.split(/(?<=[。！？.!?])/g);
+    const splitTexts = text.split(/(?<=[。！？；.!?;])/g);
    const chunks: string[] = [];

    let preChunk = '';
@@ -268,3 +268,11 @@ export const compressImg = ({
      reject('压缩图片异常');
    };
  });
+
+/* simple text, remove chinese space and extra \n */
+export const simpleText = (text: string) => {
+  text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
+  text = text.replace(/\n{2,}/g, '\n');
+  text = text.replace(/\s{2,}/g, ' ');
+  return text;
+};
--- a/docSite/docs/develop/deploy/docker.md
+++ b/docSite/docs/develop/deploy/docker.md
@@ -69,8 +69,8 @@ services:
      - ./mongo/data:/data/db
  fastgpt:
    container_name: fastgpt
-    # image: c121914yu/fast-gpt:latest # docker hub
-    image: ghcr.io/labring/fastgpt:latest # 阿里云
+    # image: ghcr.io/labring/fastgpt:latest # git
+    image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:latest # 阿里云
    ports:
      - 3000:3000
    networks:
--- a/docSite/docs/develop/update/41init.md
+++ b/docSite/docs/develop/update/41init.md
@@ -15,6 +15,6 @@

 ## 执行初始化 API

-部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）
+部署新版项目，并发起 1 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）

 https://xxxxx/api/admin/initChatItem
--- a/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/41init.md
+++ b/docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/41init.md
@@ -15,6 +15,6 @@

 ## 执行初始化 API

-部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）
+部署新版项目，并发起 1 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）

 https://xxxxx/api/admin/initChatItem
				`@@ -0,0 +1 @@`
				<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1692418843591" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4084" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M511.5 82c-236.6 0-429 192.4-429 429 0 236.5 192.5 429 429 429 236.6 0 429-192.4 429-429 0-236.5-192.4-429-429-429z m377.6 403.8H734.3c-4-139.9-41.4-259.9-97.5-331.9C776.5 203 879 332 889.1 485.8z m-402.8-349v349h-147c5.5-175.5 68.6-322.6 147-349z m0 399.4v349c-78.4-26.4-141.4-173.5-147-349h147z m50.5 349v-349h147c-5.6 175.5-68.6 322.6-147 349z m0-399.4v-349c78.4 26.4 141.4 173.5 147 349h-147zM386.3 153.9c-56.1 72-93.5 192-97.5 331.9H133.9C144.1 332 246.5 203 386.3 153.9zM133.9 536.2h154.8c4 139.9 41.4 259.9 97.5 331.9C246.5 819 144.1 690 133.9 536.2z m502.8 331.9c56.1-72 93.5-192 97.5-331.9H889C879 690 776.5 819 636.7 868.1z" fill="#5F9BEB" p-id="4085"></path></svg>