V4.6.6-1 (#656)

2025-08-07 16:30:40 +00:00 · 2023-12-27 11:07:39 +08:00
parent 86286efb54
commit 759a2330e6
182 changed files with 3099 additions and 81685 deletions
--- a/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
@@ -7,7 +7,8 @@ import {
  NumberInputField,
  NumberInputStepper,
  NumberIncrementStepper,
-  NumberDecrementStepper
+  NumberDecrementStepper,
+  Input
 } from '@chakra-ui/react';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
@@ -18,7 +19,7 @@ import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
 import { useTranslation } from 'next-i18next';

-const fileExtension = '.txt, .docx, .pdf, .md';
+const fileExtension = '.txt, .docx, .pdf, .md, .html';

 const ChunkImport = () => {
  const { t } = useTranslation();
@@ -29,6 +30,7 @@ const ChunkImport = () => {
  const {
    chunkLen,
    setChunkLen,
+    setCustomSplitChar,
    successChunks,
    totalChunks,
    isUnselectedFile,
@@ -48,15 +50,15 @@ const ChunkImport = () => {
    <Box display={['block', 'flex']} h={['auto', '100%']}>
      <SelectorContainer fileExtension={fileExtension}>
        {/* chunk size */}
-        <Flex py={4} alignItems={'center'}>
+        <Box mt={4} alignItems={'center'}>
          <Box>
            {t('core.dataset.import.Ideal chunk length')}
            <MyTooltip label={t('core.dataset.import.Ideal chunk length Tips')} forceShow>
-              <QuestionOutlineIcon ml={1} />
+              <QuestionOutlineIcon />
            </MyTooltip>
          </Box>
          <Box
-            flex={1}
+            mt={1}
            css={{
              '& > span': {
                display: 'block'
@@ -69,7 +71,6 @@ const ChunkImport = () => {
              })}
            >
              <NumberInput
-                ml={4}
                defaultValue={chunkLen}
                min={100}
                max={datasetDetail.vectorModel.maxToken}
@@ -87,9 +88,28 @@ const ChunkImport = () => {
              </NumberInput>
            </MyTooltip>
          </Box>
-        </Flex>
+        </Box>
+        {/* custom split char */}
+        <Box mt={4} alignItems={'center'}>
+          <Box>
+            {t('core.dataset.import.Custom split char')}
+            <MyTooltip label={t('core.dataset.import.Custom split char Tips')} forceShow>
+              <QuestionOutlineIcon />
+            </MyTooltip>
+          </Box>
+          <Box mt={1}>
+            <Input
+              defaultValue={''}
+              placeholder="\n;======;==SPLIT=="
+              onChange={(e) => {
+                setCustomSplitChar(e.target.value);
+                setReShowRePreview(true);
+              }}
+            />
+          </Box>
+        </Box>
        {/* price */}
-        <Flex py={4} alignItems={'center'}>
+        <Flex mt={4} alignItems={'center'}>
          <Box>
            {t('core.dataset.import.Estimated Price')}
            <MyTooltip
@@ -105,12 +125,23 @@ const ChunkImport = () => {
        </Flex>
        <Flex mt={3}>
          {showRePreview && (
-            <Button variant={'base'} mr={4} onClick={onReSplitChunks}>
+            <Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
              {t('core.dataset.import.Re Preview')}
            </Button>
          )}
-          <Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
-            {uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
+          <Button
+            isDisabled={uploading}
+            onClick={() => {
+              onReSplitChunks();
+
+              openConfirm(onclickUpload)();
+            }}
+          >
+            {uploading ? (
+              <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
+            ) : (
+              t('common.Confirm Import')
+            )}
          </Button>
        </Flex>
      </SelectorContainer>
--- a/projects/app/src/pages/dataset/detail/components/Import/CreateFileModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/CreateFileModal.tsx
@@ -58,7 +58,7 @@ const CreateFileModal = ({
        />
      </ModalBody>
      <ModalFooter>
-        <Button variant={'base'} mr={4} onClick={onClose}>
+        <Button variant={'whiteBase'} mr={4} onClick={onClose}>
          {t('common.Close')}
        </Button>
        <Button isLoading={isLoading} onClick={mutate}>
--- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
@@ -7,11 +7,11 @@ import { simpleText } from '@fastgpt/global/common/string/tools';
 import {
  fileDownload,
  readCsvContent,
-  readTxtContent,
  readPdfContent,
  readDocContent
 } from '@/web/common/file/utils';
-import { uploadFiles } from '@/web/common/file/controller';
+import { readFileRawText, readMdFile, readHtmlFile } from '@fastgpt/web/common/file/read';
+import { getUploadMdImgController, uploadFiles } from '@/web/common/file/controller';
 import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
 import React, { DragEvent, useCallback, useState } from 'react';
 import { useTranslation } from 'next-i18next';
@@ -48,6 +48,7 @@ export interface Props extends BoxProps {
  onPushFiles: (files: FileItemType[]) => void;
  tipText?: string;
  chunkLen?: number;
+  customSplitChar?: string;
  overlapRatio?: number;
  fileTemplate?: {
    type: string;
@@ -64,6 +65,7 @@ const FileSelect = ({
  onPushFiles,
  tipText,
  chunkLen = 500,
+  customSplitChar,
  overlapRatio,
  fileTemplate,
  showUrlFetch = true,
@@ -133,7 +135,7 @@ const FileSelect = ({
          });
          const fileId = filesId[0];

-          /* csv file */
+          /* QA csv file */
          if (extension === 'csv') {
            const { header, data } = await readCsvContent(file);
            if (header[0] !== 'index' || header[1] !== 'content') {
@@ -168,8 +170,19 @@ const FileSelect = ({
          let text = await (async () => {
            switch (extension) {
              case 'txt':
+                return readFileRawText(file);
              case 'md':
-                return readTxtContent(file);
+                return readMdFile({
+                  file,
+                  uploadImgController: (base64Img) =>
+                    getUploadMdImgController({ base64Img, metadata: { fileId } })
+                });
+              case 'html':
+                return readHtmlFile({
+                  file,
+                  uploadImgController: (base64Img) =>
+                    getUploadMdImgController({ base64Img, metadata: { fileId } })
+                });
              case 'pdf':
                return readPdfContent(file);
              case 'docx':
@@ -185,7 +198,8 @@ const FileSelect = ({
            const { chunks, tokens } = splitText2Chunks({
              text,
              chunkLen,
-              overlapRatio
+              overlapRatio,
+              customReg: customSplitChar ? [customSplitChar] : []
            });

            const fileItem: FileItemType = {
@@ -213,7 +227,7 @@ const FileSelect = ({
      }
      setSelectingText(undefined);
    },
-    [chunkLen, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
+    [chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
  );
  // link fetch
  const onUrlFetch = useCallback(
@@ -222,7 +236,8 @@ const FileSelect = ({
        const { chunks, tokens } = splitText2Chunks({
          text: content,
          chunkLen,
-          overlapRatio
+          overlapRatio,
+          customReg: customSplitChar ? [customSplitChar] : []
        });
        return {
          id: nanoid(),
@@ -240,7 +255,7 @@ const FileSelect = ({
      });
      onPushFiles(result);
    },
-    [chunkLen, onPushFiles, overlapRatio]
+    [chunkLen, customSplitChar, onPushFiles, overlapRatio]
  );
  // manual create file and copy data
  const onCreateFile = useCallback(
@@ -262,7 +277,8 @@ const FileSelect = ({
      const { chunks, tokens } = splitText2Chunks({
        text: content,
        chunkLen,
-        overlapRatio
+        overlapRatio,
+        customReg: customSplitChar ? [customSplitChar] : []
      });

      onPushFiles([
@@ -281,7 +297,7 @@ const FileSelect = ({
        }
      ]);
    },
-    [chunkLen, datasetDetail._id, onPushFiles, overlapRatio]
+    [chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio]
  );

  const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
@@ -352,7 +368,7 @@ const FileSelect = ({
    ml: 1,
    as: 'span',
    cursor: 'pointer',
-    color: 'blue.600',
+    color: 'primary.600',
    _hover: {
      textDecoration: 'underline'
    }
@@ -417,7 +433,7 @@ const FileSelect = ({
          mt={1}
          cursor={'pointer'}
          textDecoration={'underline'}
-          color={'blue.500'}
+          color={'primary.500'}
          fontSize={'12px'}
          onClick={() =>
            fileDownload({
--- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
@@ -43,18 +43,20 @@ type useImportStoreType = {
  setSuccessChunks: Dispatch<SetStateAction<number>>;
  isUnselectedFile: boolean;
  totalChunks: number;
-  onclickUpload: (e: { prompt?: string }) => void;
+  onclickUpload: (e?: { prompt?: string }) => void;
  onReSplitChunks: () => void;
  price: number;
  uploading: boolean;
  chunkLen: number;
  chunkOverlapRatio: number;
  setChunkLen: Dispatch<number>;
+  customSplitChar?: string;
+  setCustomSplitChar: Dispatch<string>;
  showRePreview: boolean;
  setReShowRePreview: Dispatch<SetStateAction<boolean>>;
 };
 const StateContext = createContext<useImportStoreType>({
-  onclickUpload: function (e: { prompt?: string }): void {
+  onclickUpload: function (e?: { prompt?: string }): void {
    throw new Error('Function not implemented.');
  },
  uploading: false,
@@ -72,6 +74,10 @@ const StateContext = createContext<useImportStoreType>({
  price: 0,
  chunkLen: 0,
  chunkOverlapRatio: 0,
+  customSplitChar: undefined,
+  setCustomSplitChar: function (value: string): void {
+    throw new Error('Function not implemented.');
+  },
  setChunkLen: function (value: number): void {
    throw new Error('Function not implemented.');
  },
@@ -123,6 +129,7 @@ const Provider = ({
  const [files, setFiles] = useState<FileItemType[]>([]);
  const [successChunks, setSuccessChunks] = useState(0);
  const [chunkLen, setChunkLen] = useState(defaultChunkLen);
+  const [customSplitChar, setCustomSplitChar] = useState<string>();
  const [previewFile, setPreviewFile] = useState<FileItemType>();
  const [showRePreview, setReShowRePreview] = useState(false);

@@ -198,7 +205,8 @@ const Provider = ({
          const { chunks, tokens } = splitText2Chunks({
            text: file.rawText,
            chunkLen,
-            overlapRatio: chunkOverlapRatio
+            overlapRatio: chunkOverlapRatio,
+            customReg: customSplitChar ? [customSplitChar] : []
          });

          return {
@@ -218,7 +226,7 @@ const Provider = ({
        title: getErrText(error, t('core.dataset.import.Set Chunk Error'))
      });
    }
-  }, [chunkLen, chunkOverlapRatio, t, toast]);
+  }, [chunkLen, chunkOverlapRatio, customSplitChar, t, toast]);

  const reset = useCallback(() => {
    setFiles([]);
@@ -246,6 +254,8 @@ const Provider = ({
    onclickUpload,
    uploading,
    chunkLen,
+    customSplitChar,
+    setCustomSplitChar,
    chunkOverlapRatio,
    setChunkLen,
    showRePreview,
@@ -405,7 +415,7 @@ export const SelectorContainer = ({
      {...(isUnselectedFile
        ? {}
        : {
-            maxW: ['auto', '500px']
+            maxW: ['auto', '450px']
          })}
      p={[4, 8]}
    >
@@ -437,7 +447,7 @@ export const SelectorContainer = ({
              position={'relative'}
              alignItems={'center'}
              _hover={{
-                bg: 'blue.50',
+                bg: 'primary.50',
                '& .delete': {
                  display: 'block'
                }
--- a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
@@ -9,7 +9,7 @@ import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provide
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { useTranslation } from 'next-i18next';

-const fileExtension = '.txt, .docx, .pdf, .md';
+const fileExtension = '.txt, .docx, .pdf, .md .html';

 const QAImport = () => {
  const { t } = useTranslation();
@@ -71,11 +71,18 @@ const QAImport = () => {
        </Flex>
        <Flex mt={3}>
          {showRePreview && (
-            <Button variant={'base'} mr={4} onClick={onReSplitChunks}>
+            <Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
              {t('core.dataset.import.Re Preview')}
            </Button>
          )}
-          <Button isDisabled={uploading} onClick={openConfirm(() => onclickUpload({ prompt }))}>
+          <Button
+            isDisabled={uploading}
+            onClick={() => {
+              onReSplitChunks();
+
+              openConfirm(() => onclickUpload({ prompt }))();
+            }}
+          >
            {uploading ? (
              <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
            ) : (
--- a/projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx
@@ -80,7 +80,7 @@ const UrlFetchModal = ({
        </Box>
      </ModalBody>
      <ModalFooter>
-        <Button variant={'base'} mr={4} onClick={onClose}>
+        <Button variant={'whiteBase'} mr={4} onClick={onClose}>
          {t('common.Close')}
        </Button>
        <Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>
--- a/projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx
@@ -79,7 +79,7 @@ const WebsiteConfigModal = ({
        </Box>
      </ModalBody>
      <ModalFooter>
-        <Button variant={'base'} onClick={onClose}>
+        <Button variant={'whiteBase'} onClick={onClose}>
          {t('common.Close')}
        </Button>
        <Button