4.6.2-production (#518)

2025-07-30 02:12:38 +00:00 · 2023-11-26 16:13:45 +08:00
parent 3acbf1ab17
commit f818260711
30 changed files with 477 additions and 283 deletions
--- a/projects/app/src/pages/account/components/Info.tsx
+++ b/projects/app/src/pages/account/components/Info.tsx
@@ -17,7 +17,7 @@ import type { UserType } from '@fastgpt/global/support/user/type.d';
 import { useQuery } from '@tanstack/react-query';
 import dynamic from 'next/dynamic';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { feConfigs, systemVersion } from '@/web/common/system/staticData';
 import { useTranslation } from 'next-i18next';
 import { timezoneList } from '@fastgpt/global/common/time/timezone';
@@ -94,7 +94,7 @@ const UserInfo = () => {
      const file = e[0];
      if (!file || !userInfo) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/app/detail/components/InfoModal.tsx
+++ b/projects/app/src/pages/app/detail/components/InfoModal.tsx
@@ -14,7 +14,7 @@ import { useForm } from 'react-hook-form';
 import { AppSchema } from '@fastgpt/global/core/app/type.d';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import Avatar from '@/components/Avatar';
@@ -101,7 +101,7 @@ const InfoModal = ({
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/app/list/component/CreateModal.tsx
+++ b/projects/app/src/pages/app/list/component/CreateModal.tsx
@@ -13,7 +13,7 @@ import {
 } from '@chakra-ui/react';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useToast } from '@/web/common/hooks/useToast';
 import { postCreateApp } from '@/web/core/app/api';
@@ -58,7 +58,7 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
@@ -16,10 +16,12 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';

 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
+import { useTranslation } from 'next-i18next';

-const fileExtension = '.txt, .doc, .docx, .pdf, .md';
+const fileExtension = '.txt, .docx, .pdf, .md';

 const ChunkImport = () => {
+  const { t } = useTranslation();
  const { datasetDetail } = useDatasetStore();
  const vectorModel = datasetDetail.vectorModel;
  const unitPrice = vectorModel?.price || 0.2;
@@ -48,13 +50,8 @@ const ChunkImport = () => {
        {/* chunk size */}
        <Flex py={4} alignItems={'center'}>
          <Box>
-            段落长度
-            <MyTooltip
-              label={
-                '按结束标点符号进行分段。前后段落会有 20% 的内容重叠。\n中文文档建议不要超过1000，英文不要超过1500'
-              }
-              forceShow
-            >
+            {t('core.dataset.import.Ideal chunk length')}
+            <MyTooltip label={t('core.dataset.import.Ideal chunk length Tips')} forceShow>
              <QuestionOutlineIcon ml={1} />
            </MyTooltip>
          </Box>
--- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
@@ -48,6 +48,7 @@ export interface Props extends BoxProps {
  onPushFiles: (files: FileItemType[]) => void;
  tipText?: string;
  chunkLen?: number;
+  overlapRatio?: number;
  fileTemplate?: {
    type: string;
    filename: string;
@@ -63,6 +64,7 @@ const FileSelect = ({
  onPushFiles,
  tipText,
  chunkLen = 500,
+  overlapRatio,
  fileTemplate,
  showUrlFetch = true,
  showCreateFile = true,
@@ -97,6 +99,13 @@ const FileSelect = ({
  // select file
  const onSelectFile = useCallback(
    async (files: File[]) => {
+      if (files.length >= 100) {
+        return toast({
+          status: 'warning',
+          title: t('common.file.Select file amount limit 100')
+        });
+      }
+
      try {
        for await (let file of files) {
          const extension = file?.name?.split('.')?.pop()?.toLowerCase();
@@ -165,7 +174,6 @@ const FileSelect = ({
                return readTxtContent(file);
              case 'pdf':
                return readPdfContent(file);
-              case 'doc':
              case 'docx':
                return readDocContent(file);
            }
@@ -176,7 +184,8 @@ const FileSelect = ({
            text = simpleText(text);
            const splitRes = splitText2Chunks({
              text,
-              maxLen: chunkLen
+              chunkLen,
+              overlapRatio
            });

            const fileItem: FileItemType = {
@@ -206,7 +215,7 @@ const FileSelect = ({
      }
      setSelectingText(undefined);
    },
-    [chunkLen, datasetDetail._id, onPushFiles, t, toast]
+    [chunkLen, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
  );
  // link fetch
  const onUrlFetch = useCallback(
@@ -214,7 +223,8 @@ const FileSelect = ({
      const result: FileItemType[] = e.map(({ url, content }) => {
        const splitRes = splitText2Chunks({
          text: content,
-          maxLen: chunkLen
+          chunkLen,
+          overlapRatio
        });
        return {
          id: nanoid(),
@@ -234,7 +244,7 @@ const FileSelect = ({
      });
      onPushFiles(result);
    },
-    [chunkLen, onPushFiles]
+    [chunkLen, onPushFiles, overlapRatio]
  );
  // manual create file and copy data
  const onCreateFile = useCallback(
@@ -255,7 +265,8 @@ const FileSelect = ({

      const splitRes = splitText2Chunks({
        text: content,
-        maxLen: chunkLen
+        chunkLen,
+        overlapRatio
      });

      onPushFiles([
@@ -276,7 +287,7 @@ const FileSelect = ({
        }
      ]);
    },
-    [chunkLen, datasetDetail._id, onPushFiles]
+    [chunkLen, datasetDetail._id, onPushFiles, overlapRatio]
  );

  const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
--- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
@@ -41,16 +41,19 @@ const ImportData = ({
    const map = {
      [ImportTypeEnum.chunk]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
+        chunkOverlapRatio: 0.2,
        unitPrice: vectorModel?.price || 0.2,
        mode: TrainingModeEnum.chunk
      },
      [ImportTypeEnum.qa]: {
-        defaultChunkLen: agentModel?.maxContext * 0.6 || 9000,
+        defaultChunkLen: agentModel?.maxContext * 0.6 || 8000,
+        chunkOverlapRatio: 0,
        unitPrice: agentModel?.price || 3,
        mode: TrainingModeEnum.qa
      },
      [ImportTypeEnum.csv]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
+        chunkOverlapRatio: 0,
        unitPrice: vectorModel?.price || 0.2,
        mode: TrainingModeEnum.chunk
      }
--- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
@@ -44,6 +44,7 @@ type useImportStoreType = {
  price: number;
  uploading: boolean;
  chunkLen: number;
+  chunkOverlapRatio: number;
  setChunkLen: Dispatch<number>;
  showRePreview: boolean;
  setReShowRePreview: Dispatch<SetStateAction<boolean>>;
@@ -66,6 +67,7 @@ const StateContext = createContext<useImportStoreType>({
  },
  price: 0,
  chunkLen: 0,
+  chunkOverlapRatio: 0,
  setChunkLen: function (value: number): void {
    throw new Error('Function not implemented.');
  },
@@ -93,6 +95,7 @@ const Provider = ({
  vectorModel,
  agentModel,
  defaultChunkLen = 500,
+  chunkOverlapRatio = 0.2,
  importType,
  onUploadSuccess,
  children
@@ -104,6 +107,7 @@ const Provider = ({
  vectorModel: string;
  agentModel: string;
  defaultChunkLen: number;
+  chunkOverlapRatio: number;
  importType: `${ImportTypeEnum}`;
  onUploadSuccess: () => void;
  children: React.ReactNode;
@@ -180,7 +184,8 @@ const Provider = ({
        state.map((file) => {
          const splitRes = splitText2Chunks({
            text: file.text,
-            maxLen: chunkLen
+            chunkLen,
+            overlapRatio: chunkOverlapRatio
          });

          return {
@@ -228,6 +233,7 @@ const Provider = ({
    onclickUpload,
    uploading,
    chunkLen,
+    chunkOverlapRatio,
    setChunkLen,
    showRePreview,
    setReShowRePreview
@@ -413,7 +419,8 @@ export const SelectorContainer = ({
  tip?: string;
  children: React.ReactNode;
 }) => {
-  const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen } = useImportStore();
+  const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen, chunkOverlapRatio } =
+    useImportStore();
  return (
    <Box
      h={'100%'}
@@ -432,6 +439,7 @@ export const SelectorContainer = ({
          setFiles((state) => files.concat(state));
        }}
        chunkLen={chunkLen}
+        overlapRatio={chunkOverlapRatio}
        showUrlFetch={showUrlFetch}
        showCreateFile={showCreateFile}
        fileTemplate={fileTemplate}
--- a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
@@ -1,15 +1,14 @@
-import React, { useState, useMemo } from 'react';
-import { Box, Flex, Button, Input } from '@chakra-ui/react';
+import React, { useState } from 'react';
+import { Box, Flex, Button, Textarea } from '@chakra-ui/react';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import MyTooltip from '@/components/MyTooltip';
-import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons';
+import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { Prompt_AgentQA } from '@/global/core/prompt/agent';
-import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';

-const fileExtension = '.txt, .doc, .docx, .pdf, .md';
+const fileExtension = '.txt, .docx, .pdf, .md';

 const QAImport = () => {
  const { datasetDetail } = useDatasetStore();
@@ -31,36 +30,27 @@ const QAImport = () => {
    content: `该任务无法终止！导入后会自动调用大模型生成问答对，会有一些细节丢失，请确认！如果余额不足，未完成的任务会被暂停。`
  });

-  const [prompt, setPrompt] = useState('');
-
-  const previewQAPrompt = useMemo(() => {
-    return replaceVariable(Prompt_AgentQA.prompt, {
-      theme: prompt || Prompt_AgentQA.defaultTheme
-    });
-  }, [prompt]);
+  const [prompt, setPrompt] = useState(Prompt_AgentQA.description);

  return (
    <Box display={['block', 'flex']} h={['auto', '100%']}>
      <SelectorContainer fileExtension={fileExtension}>
        {/* prompt */}
-        <Box py={5}>
-          <Box mb={2}>
-            QA 拆分引导词{' '}
-            <MyTooltip label={previewQAPrompt} forceShow>
-              <InfoOutlineIcon ml={1} />
-            </MyTooltip>
+        <Box p={3} bg={'myWhite.600'} borderRadius={'md'}>
+          <Box mb={1} fontWeight={'bold'}>
+            QA 拆分引导词
          </Box>
-          <Flex alignItems={'center'} fontSize={'sm'}>
-            <Box mr={2}>文件主题</Box>
-            <Input
-              fontSize={'sm'}
-              flex={1}
-              placeholder={Prompt_AgentQA.defaultTheme}
-              bg={'myWhite.500'}
+          <Box whiteSpace={'pre-wrap'} fontSize={'sm'}>
+            <Textarea
              defaultValue={prompt}
-              onChange={(e) => setPrompt(e.target.value || '')}
+              rows={8}
+              fontSize={'sm'}
+              onChange={(e) => {
+                setPrompt(e.target.value);
+              }}
            />
-          </Flex>
+            <Box>{Prompt_AgentQA.fixedText}</Box>
+          </Box>
        </Box>
        {/* price */}
        <Flex py={5} alignItems={'center'}>
@@ -81,10 +71,7 @@ const QAImport = () => {
              重新生成预览
            </Button>
          )}
-          <Button
-            isDisabled={uploading}
-            onClick={openConfirm(() => onclickUpload({ prompt: previewQAPrompt }))}
-          >
+          <Button isDisabled={uploading} onClick={openConfirm(() => onclickUpload({ prompt }))}>
            {uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
          </Button>
        </Flex>
--- a/projects/app/src/pages/dataset/detail/components/Info.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Info.tsx
@@ -15,7 +15,7 @@ import { useToast } from '@/web/common/hooks/useToast';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { UseFormReturn } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
 import Avatar from '@/components/Avatar';
 import Tag from '@/components/Tag';
@@ -95,7 +95,7 @@ const Info = (
      }
      setBtnLoading(false);
    },
-    [updateDataset, datasetId, loadDatasetDetail, toast, loadDatasets]
+    [updateDataset, datasetId, toast, loadDatasets]
  );
  const saveSubmitError = useCallback(() => {
    // deep search message
@@ -119,7 +119,7 @@ const Info = (
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/dataset/detail/components/Test.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Test.tsx
@@ -81,43 +81,45 @@ const Test = ({ datasetId }: { datasetId: string }) => {
        borderRight={['none', theme.borders.base]}
      >
        <Box border={'2px solid'} borderColor={'myBlue.600'} p={3} mx={4} borderRadius={'md'}>
-          <Box fontSize={'sm'} fontWeight={'bold'}>
-            <MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'myBlue.700'} />
-            测试文本
-          </Box>
-          <Textarea
-            rows={6}
-            resize={'none'}
-            variant={'unstyled'}
-            maxLength={datasetDetail.vectorModel.maxToken}
-            placeholder="输入需要测试的文本"
-            value={inputText}
-            onChange={(e) => setInputText(e.target.value)}
-          />
-          <Flex alignItems={'center'} justifyContent={'flex-end'}>
+          <Flex alignItems={'center'}>
+            <Box fontSize={'sm'} fontWeight={'bold'} flex={1}>
+              <MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'myBlue.700'} />
+              {t('core.dataset.test.Test Text')}
+            </Box>
            {feConfigs?.isPlus && (
              <Flex alignItems={'center'}>
                {t('dataset.recall.rerank')}
                <Switch ml={1} isChecked={rerank} onChange={(e) => setRerank(e.target.checked)} />
              </Flex>
            )}
+          </Flex>
+          <Textarea
+            rows={6}
+            resize={'none'}
+            variant={'unstyled'}
+            maxLength={datasetDetail.vectorModel.maxToken}
+            placeholder={t('core.dataset.test.Test Text Placeholder')}
+            value={inputText}
+            onChange={(e) => setInputText(e.target.value)}
+          />
+          <Flex alignItems={'center'} justifyContent={'flex-end'}>
            <Box mx={3} color={'myGray.500'}>
              {inputText.length}
            </Box>
            <Button isDisabled={inputText === ''} isLoading={isLoading} onClick={mutate}>
-              测试
+              {t('core.dataset.test.Test')}
            </Button>
          </Flex>
        </Box>
        <Box mt={5} flex={'1 0 0'} px={4} overflow={'overlay'} display={['none', 'block']}>
          <Flex alignItems={'center'} color={'myGray.600'}>
            <MyIcon mr={2} name={'history'} w={'16px'} h={'16px'} />
-            <Box fontSize={'2xl'}>测试历史</Box>
+            <Box fontSize={'2xl'}>{t('core.dataset.test.test history')}</Box>
          </Flex>
          <Box mt={2}>
            <Flex py={2} fontWeight={'bold'} borderBottom={theme.borders.sm}>
-              <Box flex={1}>测试文本</Box>
-              <Box w={'80px'}>时间</Box>
+              <Box flex={1}>{t('core.dataset.test.Test Text')}</Box>
+              <Box w={'80px'}>{t('common.Time')}</Box>
              <Box w={'14px'}></Box>
            </Flex>
            {kbTestHistory.map((item) => (
@@ -139,7 +141,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                  {item.text}
                </Box>
                <Box w={'80px'}>{formatTimeToChatTime(item.time)}</Box>
-                <MyTooltip label={'删除该测试记录'}>
+                <MyTooltip label={t('core.dataset.test.delete test history')}>
                  <Box w={'14px'} h={'14px'}>
                    <MyIcon
                      className="delete"
@@ -171,7 +173,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
          >
            <MyIcon name={'empty'} color={'transparent'} w={'54px'} />
            <Box mt={3} color={'myGray.600'}>
-              测试结果将在这里展示
+              {t('core.dataset.test.test result placeholder')}
            </Box>
          </Flex>
        ) : (
@@ -180,12 +182,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
              <Box fontSize={'3xl'} color={'myGray.600'}>
                {t('core.dataset.test.Test Result')}
              </Box>
-              <MyTooltip
-                label={
-                  '根据知识库内容与测试文本的相似度进行排序，你可以根据测试结果调整对应的文本。\n注意：测试记录中的数据可能已经被修改过，点击某条测试数据后将展示最新的数据。'
-                }
-                forceShow
-              >
+              <MyTooltip label={t('core.dataset.test.test result tip')} forceShow>
                <QuestionOutlineIcon
                  mx={2}
                  color={'myGray.600'}
@@ -221,7 +218,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                      const data = await getDatasetDataItemById(item.id);

                      if (!data) {
-                        throw new Error('该数据已被删除');
+                        throw new Error(t('core.dataset.data.data is deleted'));
                      }

                      setEditInputData({
--- a/projects/app/src/pages/dataset/list/component/CreateModal.tsx
+++ b/projects/app/src/pages/dataset/list/component/CreateModal.tsx
@@ -11,7 +11,7 @@ import {
 } from '@chakra-ui/react';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useRouter } from 'next/router';
@@ -57,7 +57,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100
--- a/projects/app/src/pages/plugin/list/component/EditModal.tsx
+++ b/projects/app/src/pages/plugin/list/component/EditModal.tsx
@@ -11,7 +11,7 @@ import {
 } from '@chakra-ui/react';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImgAndUpload } from '@/web/common/file/controller';
+import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useRouter } from 'next/router';
@@ -136,7 +136,7 @@ const CreateModal = ({
      const file = e[0];
      if (!file) return;
      try {
-        const src = await compressImgAndUpload({
+        const src = await compressImgFileAndUpload({
          file,
          maxW: 100,
          maxH: 100