4.6.7-alpha commit (#743)

Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/pages/dataset/detail/components/CollectionCard.tsx

@@ -41,14 +41,13 @@ import { useEditTitle } from '@/web/common/hooks/useEditTitle';
 import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
 import EmptyTip from '@/components/EmptyTip';
 import {
-  FolderAvatarSrc,
   DatasetCollectionTypeEnum,
   TrainingModeEnum,
   DatasetTypeEnum,
   DatasetTypeMap,
   DatasetStatusEnum,
   DatasetCollectionSyncResultMap
-} from '@fastgpt/global/core/dataset/constant';
+} from '@fastgpt/global/core/dataset/constants';
 import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
 import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
 import { TabEnum } from '..';
@@ -62,11 +61,12 @@ import { useUserStore } from '@/web/support/user/useUserStore';
 import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
-import { DatasetCollectionSyncResultEnum } from '../../../../../../../packages/global/core/dataset/constant';
+import { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants';
 import MyBox from '@/components/common/MyBox';
+import { ImportDataSourceEnum } from './Import';
 
-const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
 const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {});
+const FileSourceSelector = dynamic(() => import('./Import/sourceSelector/FileSourceSelector'), {});
 
 const CollectionCard = () => {
   const BoxRef = useRef<HTMLDivElement>(null);
@@ -90,9 +90,9 @@ const CollectionCard = () => {
   });
 
   const {
-    isOpen: isOpenFileImportModal,
-    onOpen: onOpenFileImportModal,
-    onClose: onCloseFileImportModal
+    isOpen: isOpenFileSourceSelector,
+    onOpen: onOpenFileSourceSelector,
+    onClose: onCloseFileSourceSelector
   } = useDisclosure();
   const {
     isOpen: isOpenWebsiteModal,
@@ -159,12 +159,16 @@ const CollectionCard = () => {
               statusText: t('dataset.collections.Collection Embedding', {
                 total: collection.trainingAmount
               }),
-              color: 'myGray.500'
+              color: 'myGray.600',
+              bg: 'myGray.50',
+              borderColor: 'borderColor.low'
             };
           }
           return {
             statusText: t('core.dataset.collection.status.active'),
-            color: 'green.500'
+            color: 'green.600',
+            bg: 'green.50',
+            borderColor: 'green.300'
           };
         })();
 
@@ -299,7 +303,8 @@ const CollectionCard = () => {
   return (
     <MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
       <Flex ref={BoxRef} flexDirection={'column'} py={[1, 3]} h={'100%'}>
-        <Flex px={[2, 6]} alignItems={['flex-start', 'center']} h={'35px'}>
+        {/* header */}
+        <Flex px={[2, 6]} alignItems={'flex-start'} h={'35px'}>
           <Box flex={1}>
             <ParentPath
               paths={paths.map((path, i) => ({
@@ -343,7 +348,7 @@ const CollectionCard = () => {
               <MyInput
                 bg={'myGray.50'}
                 w={['100%', '250px']}
-                size={['sm', 'md']}
+                size={'sm'}
                 h={'36px'}
                 placeholder={t('common.Search') || ''}
                 value={searchText}
@@ -376,7 +381,7 @@ const CollectionCard = () => {
             <>
               {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
                 <MyMenu
-                  offset={[-40, 10]}
+                  offset={[-0, 10]}
                   width={120}
                   Button={
                     <MenuButton
@@ -405,7 +410,7 @@ const CollectionCard = () => {
                     {
                       child: (
                         <Flex>
-                          <Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
+                          <MyIcon name={'common/folderFill'} w={'20px'} mr={2} />
                           {t('Folder')}
                         </Flex>
                       ),
@@ -414,7 +419,7 @@ const CollectionCard = () => {
                     {
                       child: (
                         <Flex>
-                          <Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
+                          <MyIcon name={'core/dataset/manualCollection'} mr={2} w={'20px'} />
                           {t('core.dataset.Manual collection')}
                         </Flex>
                       ),
@@ -430,11 +435,27 @@ const CollectionCard = () => {
                     {
                       child: (
                         <Flex>
-                          <Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
-                          {t('core.dataset.File collection')}
+                          <MyIcon name={'core/dataset/fileCollection'} mr={2} w={'20px'} />
+                          {t('core.dataset.Text collection')}
                         </Flex>
                       ),
-                      onClick: onOpenFileImportModal
+                      onClick: onOpenFileSourceSelector
+                    },
+                    {
+                      child: (
+                        <Flex>
+                          <MyIcon name={'core/dataset/tableCollection'} mr={2} w={'20px'} />
+                          {t('core.dataset.Table collection')}
+                        </Flex>
+                      ),
+                      onClick: () =>
+                        router.replace({
+                          query: {
+                            ...router.query,
+                            currentTab: TabEnum.import,
+                            source: ImportDataSourceEnum.tableLocal
+                          }
+                        })
                     }
                   ]}
                 />
@@ -478,6 +499,7 @@ const CollectionCard = () => {
           )}
         </Flex>
 
+        {/* collection table */}
         <TableContainer
           px={[2, 6]}
           mt={[0, 3]}
@@ -545,11 +567,6 @@ const CollectionCard = () => {
                     } catch (error) {}
                     setDragTargetId(undefined);
                   }}
-                  title={
-                    collection.type === DatasetCollectionTypeEnum.folder
-                      ? t('dataset.collections.Click to view folder')
-                      : t('dataset.collections.Click to view file')
-                  }
                   onClick={() => {
                     if (collection.type === DatasetCollectionTypeEnum.folder) {
                       router.replace({
@@ -572,7 +589,7 @@ const CollectionCard = () => {
                   <Td w={'50px'}>{index + 1}</Td>
                   <Td minW={'150px'} maxW={['200px', '300px']} draggable>
                     <Flex alignItems={'center'}>
-                      <Image src={collection.icon} w={'16px'} mr={2} alt={''} />
+                      <MyIcon name={collection.icon as any} w={'16px'} mr={2} />
                       <MyTooltip label={t('common.folder.Drag Tip')} shouldWrapChildren={false}>
                         <Box fontWeight={'bold'} className="textEllipsis">
                           {collection.name}
@@ -583,19 +600,28 @@ const CollectionCard = () => {
                   <Td fontSize={'md'}>{collection.dataAmount || '-'}</Td>
                   <Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
                   <Td>
-                    <Flex
+                    <Box
+                      display={'inline-flex'}
                       alignItems={'center'}
+                      w={'auto'}
+                      color={collection.color}
+                      bg={collection.bg}
+                      borderWidth={'1px'}
+                      borderColor={collection.borderColor}
+                      px={3}
+                      py={1}
+                      borderRadius={'md'}
                       _before={{
                         content: '""',
-                        w: '10px',
-                        h: '10px',
+                        w: '6px',
+                        h: '6px',
                         mr: 2,
                         borderRadius: 'lg',
                         bg: collection.color
                       }}
                     >
                       {t(collection.statusText)}
-                    </Flex>
+                    </Box>
                   </Td>
                   <Td onClick={(e) => e.stopPropagation()}>
                     {collection.canWrite && userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
@@ -744,8 +770,8 @@ const CollectionCard = () => {
         <ConfirmDeleteModal />
         <ConfirmSyncModal />
         <EditTitleModal />
-        <EditCreateVirtualFileModal />
-        {isOpenFileImportModal && (
+        <EditCreateVirtualFileModal iconSrc={'modal/manualDataset'} closeBtnText={''} />
+        {/* {isOpenFileImportModal && (
           <FileImportModal
             datasetId={datasetId}
             parentId={parentId}
@@ -755,7 +781,8 @@ const CollectionCard = () => {
             }}
             onClose={onCloseFileImportModal}
           />
-        )}
+        )} */}
+        {isOpenFileSourceSelector && <FileSourceSelector onClose={onCloseFileSourceSelector} />}
         {!!editFolderData && (
           <EditFolderModal
             onClose={() => setEditFolderData(undefined)}

projects/app/src/pages/dataset/detail/components/DataCard.tsx

@@ -43,7 +43,7 @@ import {
   DatasetCollectionTypeMap,
   TrainingModeEnum,
   TrainingTypeMap
-} from '@fastgpt/global/core/dataset/constant';
+} from '@fastgpt/global/core/dataset/constants';
 import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
 import { formatFileSize } from '@fastgpt/global/common/file/tools';
 import { getFileAndOpen } from '@/web/core/dataset/utils';
@@ -65,7 +65,8 @@ const DataCard = () => {
   const [searchText, setSearchText] = useState('');
   const { toast } = useToast();
   const { openConfirm, ConfirmModal } = useConfirm({
-    content: t('dataset.Confirm to delete the data')
+    content: t('dataset.Confirm to delete the data'),
+    type: 'delete'
   });
   const { isOpen, onOpen, onClose } = useDisclosure();
 
@@ -173,202 +174,224 @@ const DataCard = () => {
   }, [collection, t]);
 
   return (
-    <Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
-      <Flex alignItems={'center'}>
-        <IconButton
-          mr={3}
-          icon={<MyIcon name={'common/backFill'} w={['14px', '18px']} color={'primary.500'} />}
-          variant={'whitePrimary'}
-          size={'smSquare'}
-          borderRadius={'50%'}
-          aria-label={''}
-          onClick={() =>
-            router.replace({
-              query: {
-                datasetId: router.query.datasetId,
-                parentId: router.query.parentId,
-                currentTab: TabEnum.collectionCard
-              }
-            })
-          }
-        />
-        <Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
-          <Box lineHeight={1.2}>
-            <RawSourceBox
-              sourceName={collection?.name}
-              sourceId={collection?.fileId || collection?.rawLink}
-              fontSize={['md', 'lg']}
-              color={'black'}
-              textDecoration={'none'}
-            />
-            <Box fontSize={'sm'} color={'myGray.500'}>
-              {t('core.dataset.collection.id')}:{' '}
-              <Box as={'span'} userSelect={'all'}>
-                {collection?._id}
+    <Box position={'relative'} py={[1, 5]} h={'100%'}>
+      <Flex ref={BoxRef} flexDirection={'column'} h={'100%'}>
+        <Flex alignItems={'center'} px={5}>
+          <IconButton
+            mr={3}
+            icon={<MyIcon name={'common/backFill'} w={['14px', '18px']} color={'primary.500'} />}
+            variant={'whitePrimary'}
+            size={'smSquare'}
+            borderRadius={'50%'}
+            aria-label={''}
+            onClick={() =>
+              router.replace({
+                query: {
+                  datasetId: router.query.datasetId,
+                  parentId: router.query.parentId,
+                  currentTab: TabEnum.collectionCard
+                }
+              })
+            }
+          />
+          <Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
+            <Box lineHeight={1.2}>
+              <RawSourceBox
+                sourceName={collection?.name}
+                sourceId={collection?.fileId || collection?.rawLink}
+                fontSize={['md', 'lg']}
+                color={'black'}
+                textDecoration={'none'}
+              />
+              <Box fontSize={'sm'} color={'myGray.500'}>
+                {t('core.dataset.collection.id')}:{' '}
+                <Box as={'span'} userSelect={'all'}>
+                  {collection?._id}
+                </Box>
               </Box>
             </Box>
-          </Box>
+          </Flex>
+          {canWrite && (
+            <Box>
+              <Button
+                mx={2}
+                variant={'whitePrimary'}
+                size={['sm', 'md']}
+                onClick={() => {
+                  if (!collection) return;
+                  setEditDataId('');
+                }}
+              >
+                {t('dataset.Insert Data')}
+              </Button>
+            </Box>
+          )}
+          {isPc && (
+            <MyTooltip label={t('core.dataset.collection.metadata.Read Metadata')}>
+              <IconButton
+                variant={'whiteBase'}
+                size={['sm', 'md']}
+                icon={<MyIcon name={'menu'} w={'18px'} />}
+                aria-label={''}
+                onClick={onOpen}
+              />
+            </MyTooltip>
+          )}
         </Flex>
-        {canWrite && (
+        <Flex my={3} alignItems={'center'} px={5}>
           <Box>
-            <Button
-              mx={2}
-              variant={'whitePrimary'}
-              size={['sm', 'md']}
-              onClick={() => {
-                if (!collection) return;
-                setEditDataId('');
-              }}
-            >
-              {t('dataset.Insert Data')}
-            </Button>
+            <Box as={'span'} fontSize={['md', 'lg']}>
+              {t('core.dataset.data.Total Amount', { total })}
+            </Box>
           </Box>
-        )}
-        {isPc && (
-          <MyTooltip label={t('core.dataset.collection.metadata.Read Metadata')}>
-            <IconButton
-              variant={'whiteBase'}
-              size={['sm', 'md']}
-              icon={<MyIcon name={'menu'} w={'18px'} />}
-              aria-label={''}
-              onClick={onOpen}
-            />
-          </MyTooltip>
-        )}
-      </Flex>
-      <Flex my={3} alignItems={'center'}>
-        <Box>
-          <Box as={'span'} fontSize={['md', 'lg']}>
-            {t('core.dataset.data.Total Amount', { total })}
-          </Box>
-        </Box>
-        <Box flex={1} mr={1} />
-        <MyInput
-          leftIcon={
-            <MyIcon
-              name="common/searchLight"
-              position={'absolute'}
-              w={'14px'}
-              color={'myGray.500'}
-            />
-          }
-          w={['200px', '300px']}
-          placeholder={t('core.dataset.data.Search data placeholder')}
-          value={searchText}
-          onChange={(e) => {
-            setSearchText(e.target.value);
-            getFirstData();
-          }}
-          onBlur={() => {
-            if (searchText === lastSearch.current) return;
-            getFirstData();
-          }}
-          onKeyDown={(e) => {
-            if (searchText === lastSearch.current) return;
-            if (e.key === 'Enter') {
-              getFirstData();
+          <Box flex={1} mr={1} />
+          <MyInput
+            leftIcon={
+              <MyIcon
+                name="common/searchLight"
+                position={'absolute'}
+                w={'14px'}
+                color={'myGray.500'}
+              />
             }
-          }}
-        />
-      </Flex>
-      <Grid
-        minH={'100px'}
-        gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
-        gridGap={4}
-      >
-        {datasetDataList.map((item, index) => (
-          <Card
-            key={item._id}
-            cursor={'pointer'}
-            p={3}
-            userSelect={'none'}
-            boxShadow={'none'}
-            bg={'myWhite.500'}
-            border={theme.borders.sm}
-            position={'relative'}
-            overflow={'hidden'}
-            _hover={{
-              borderColor: 'myGray.200',
-              boxShadow: 'lg',
-              bg: 'white',
-              '& .footer': { h: 'auto', p: 3 }
+            w={['200px', '300px']}
+            placeholder={t('core.dataset.data.Search data placeholder')}
+            value={searchText}
+            onChange={(e) => {
+              setSearchText(e.target.value);
+              getFirstData();
             }}
-            onClick={() => {
-              if (!collection) return;
-              setEditDataId(item._id);
+            onBlur={() => {
+              if (searchText === lastSearch.current) return;
+              getFirstData();
             }}
+            onKeyDown={(e) => {
+              if (searchText === lastSearch.current) return;
+              if (e.key === 'Enter') {
+                getFirstData();
+              }
+            }}
+          />
+        </Flex>
+        <Box flex={'1 0 0'} overflow={'auto'} px={5}>
+          <Grid
+            gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
+            gridGap={4}
           >
-            <Flex zIndex={1} alignItems={'center'} justifyContent={'space-between'}>
-              <Box border={theme.borders.base} px={2} fontSize={'sm'} mr={1} borderRadius={'md'}>
-                # {item.chunkIndex ?? '-'}
-              </Box>
-              <Box className={'textEllipsis'} color={'myGray.500'} fontSize={'xs'}>
-                ID:{item._id}
+            {datasetDataList.map((item, index) => (
+              <Card
+                key={item._id}
+                cursor={'pointer'}
+                p={3}
+                userSelect={'none'}
+                boxShadow={'none'}
+                bg={'myWhite.500'}
+                border={theme.borders.sm}
+                position={'relative'}
+                overflow={'hidden'}
+                _hover={{
+                  borderColor: 'myGray.200',
+                  boxShadow: 'lg',
+                  bg: 'white',
+                  '& .footer': { h: 'auto', p: 3 }
+                }}
+                onClick={() => {
+                  if (!collection) return;
+                  setEditDataId(item._id);
+                }}
+              >
+                <Flex zIndex={1} alignItems={'center'} justifyContent={'space-between'}>
+                  <Box
+                    border={theme.borders.base}
+                    px={2}
+                    fontSize={'sm'}
+                    mr={1}
+                    borderRadius={'md'}
+                  >
+                    # {item.chunkIndex ?? '-'}
+                  </Box>
+                  <Box className={'textEllipsis'} color={'myGray.500'} fontSize={'xs'}>
+                    ID:{item._id}
+                  </Box>
+                </Flex>
+                <Box
+                  maxH={'135px'}
+                  minH={'90px'}
+                  overflow={'hidden'}
+                  wordBreak={'break-all'}
+                  pt={1}
+                  pb={3}
+                  fontSize={'13px'}
+                >
+                  <Box color={'black'} mb={1}>
+                    {item.q}
+                  </Box>
+                  <Box color={'myGray.700'}>{item.a}</Box>
+
+                  <Flex
+                    className="footer"
+                    position={'absolute'}
+                    top={0}
+                    bottom={0}
+                    left={0}
+                    right={0}
+                    h={'0'}
+                    overflow={'hidden'}
+                    p={0}
+                    bg={'linear-gradient(to top, white,white 20%, rgba(255,255,255,0) 60%)'}
+                    alignItems={'flex-end'}
+                    fontSize={'sm'}
+                  >
+                    <Flex alignItems={'center'}>
+                      <MyIcon name="common/text/t" w={'14px'} mr={1} color={'myGray.500'} />
+                      {item.q.length + (item.a?.length || 0)}
+                    </Flex>
+                    <Box flex={1} />
+                    {canWrite && (
+                      <IconButton
+                        display={'flex'}
+                        icon={<DeleteIcon />}
+                        variant={'whiteDanger'}
+                        size={'xsSquare'}
+                        aria-label={'delete'}
+                        onClick={(e) => {
+                          e.stopPropagation();
+                          openConfirm(async () => {
+                            try {
+                              setIsLoading(true);
+                              await delOneDatasetDataById(item._id);
+                              getData(pageNum);
+                            } catch (error) {
+                              toast({
+                                title: getErrText(error),
+                                status: 'error'
+                              });
+                            }
+                            setIsLoading(false);
+                          })();
+                        }}
+                      />
+                    )}
+                  </Flex>
+                </Box>
+              </Card>
+            ))}
+          </Grid>
+          {total > pageSize && (
+            <Flex mt={2} justifyContent={'center'}>
+              <Pagination />
+            </Flex>
+          )}
+          {total === 0 && (
+            <Flex flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
+              <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
+              <Box mt={2} color={'myGray.500'}>
+                {t('core.dataset.data.Empty Tip')}
               </Box>
             </Flex>
-            <Box
-              maxH={'135px'}
-              minH={'90px'}
-              overflow={'hidden'}
-              wordBreak={'break-all'}
-              pt={1}
-              pb={3}
-              fontSize={'13px'}
-            >
-              <Box color={'black'} mb={1}>
-                {item.q}
-              </Box>
-              <Box color={'myGray.700'}>{item.a}</Box>
-
-              <Flex
-                className="footer"
-                position={'absolute'}
-                top={0}
-                bottom={0}
-                left={0}
-                right={0}
-                h={'0'}
-                overflow={'hidden'}
-                p={0}
-                bg={'linear-gradient(to top, white,white 20%, rgba(255,255,255,0) 60%)'}
-                alignItems={'flex-end'}
-                fontSize={'sm'}
-              >
-                <Flex alignItems={'center'}>
-                  <MyIcon name="common/text/t" w={'14px'} mr={1} color={'myGray.500'} />
-                  {item.q.length + (item.a?.length || 0)}
-                </Flex>
-                <Box flex={1} />
-                {canWrite && (
-                  <IconButton
-                    display={'flex'}
-                    icon={<DeleteIcon />}
-                    variant={'whiteDanger'}
-                    size={'xsSquare'}
-                    aria-label={'delete'}
-                    onClick={(e) => {
-                      e.stopPropagation();
-                      openConfirm(async () => {
-                        try {
-                          setIsLoading(true);
-                          await delOneDatasetDataById(item._id);
-                          getData(pageNum);
-                        } catch (error) {
-                          toast({
-                            title: getErrText(error),
-                            status: 'error'
-                          });
-                        }
-                        setIsLoading(false);
-                      })();
-                    }}
-                  />
-                )}
-              </Flex>
-            </Box>
-          </Card>
-        ))}
-      </Grid>
+          )}
+        </Box>
+      </Flex>
 
       {/* metadata drawer */}
       <Drawer isOpen={isOpen} placement="right" size={'md'} onClose={onClose}>
@@ -378,8 +401,8 @@ const DataCard = () => {
 
           <DrawerBody>
             {metadataList.map((item) => (
-              <Flex key={item.label} alignItems={'center'} mb={5}>
-                <Box color={'myGray.500'} w={'100px'}>
+              <Flex key={item.label} alignItems={'center'} mb={5} wordBreak={'break-all'}>
+                <Box color={'myGray.500'} flex={'0 0 100px'}>
                   {item.label}
                 </Box>
                 <Box>{item.value}</Box>
@@ -403,20 +426,6 @@ const DataCard = () => {
         </DrawerContent>
       </Drawer>
 
-      {total > pageSize && (
-        <Flex mt={2} justifyContent={'center'}>
-          <Pagination />
-        </Flex>
-      )}
-      {total === 0 && (
-        <Flex flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
-          <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
-          <Box mt={2} color={'myGray.500'}>
-            {t('core.dataset.data.Empty Tip')}
-          </Box>
-        </Flex>
-      )}
-
       {editDataId !== undefined && collection && (
         <InputDataModal
           collectionId={collection._id}

projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx

@@ -1,166 +0,0 @@
-import React from 'react';
-import {
-  Box,
-  Flex,
-  Button,
-  NumberInput,
-  NumberInputField,
-  NumberInputStepper,
-  NumberIncrementStepper,
-  NumberDecrementStepper,
-  Input,
-  Grid
-} from '@chakra-ui/react';
-import { useConfirm } from '@/web/common/hooks/useConfirm';
-import MyTooltip from '@/components/MyTooltip';
-import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-
-import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
-import { useTranslation } from 'next-i18next';
-
-const fileExtension = '.txt, .docx, .pdf, .md, .html';
-
-const ChunkImport = () => {
-  const { t } = useTranslation();
-  const { datasetDetail } = useDatasetStore();
-  const vectorModel = datasetDetail.vectorModel;
-  const unitPrice = vectorModel?.inputPrice || 0.002;
-
-  const {
-    chunkLen,
-    setChunkLen,
-    setCustomSplitChar,
-    successChunks,
-    totalChunks,
-    totalTokens,
-    isUnselectedFile,
-    price,
-    onclickUpload,
-    onReSplitChunks,
-    uploading,
-    showRePreview,
-    setReShowRePreview
-  } = useImportStore();
-
-  const { openConfirm, ConfirmModal } = useConfirm({
-    content: t('core.dataset.import.Import Tip')
-  });
-
-  return (
-    <Box display={['block', 'flex']} h={['auto', '100%']}>
-      <SelectorContainer fileExtension={fileExtension}>
-        {/* chunk size */}
-        <Box mt={4} alignItems={'center'}>
-          <Box>
-            {t('core.dataset.import.Ideal chunk length')}
-            <MyTooltip label={t('core.dataset.import.Ideal chunk length Tips')} forceShow>
-              <QuestionOutlineIcon />
-            </MyTooltip>
-          </Box>
-          <Box
-            mt={1}
-            css={{
-              '& > span': {
-                display: 'block'
-              }
-            }}
-          >
-            <MyTooltip
-              label={t('core.dataset.import.Chunk Range', {
-                max: datasetDetail.vectorModel.maxToken
-              })}
-            >
-              <NumberInput
-                defaultValue={chunkLen}
-                min={100}
-                max={datasetDetail.vectorModel.maxToken}
-                step={10}
-                onChange={(e) => {
-                  setChunkLen(+e);
-                  setReShowRePreview(true);
-                }}
-              >
-                <NumberInputField />
-                <NumberInputStepper>
-                  <NumberIncrementStepper />
-                  <NumberDecrementStepper />
-                </NumberInputStepper>
-              </NumberInput>
-            </MyTooltip>
-          </Box>
-        </Box>
-        {/* custom split char */}
-        <Box mt={4} alignItems={'center'}>
-          <Box>
-            {t('core.dataset.import.Custom split char')}
-            <MyTooltip label={t('core.dataset.import.Custom split char Tips')} forceShow>
-              <QuestionOutlineIcon />
-            </MyTooltip>
-          </Box>
-          <Box mt={1}>
-            <Input
-              defaultValue={''}
-              placeholder="\n;======;==SPLIT=="
-              onChange={(e) => {
-                setCustomSplitChar(e.target.value);
-                setReShowRePreview(true);
-              }}
-            />
-          </Box>
-        </Box>
-        <Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
-          <Flex alignItems={'center'}>
-            <Box>{t('core.dataset.import.Total tokens')}：</Box>
-            <Box>{totalTokens}</Box>
-          </Flex>
-          {/* price */}
-          <Flex alignItems={'center'}>
-            <Box>
-              {t('core.dataset.import.Estimated Price')}
-              <MyTooltip
-                label={t('core.dataset.import.Embedding Estimated Price Tips', {
-                  price: unitPrice
-                })}
-                forceShow
-              >
-                <QuestionOutlineIcon ml={1} />
-              </MyTooltip>
-            </Box>
-            <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
-          </Flex>
-        </Grid>
-        <Flex mt={3}>
-          {showRePreview && (
-            <Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
-              {t('core.dataset.import.Re Preview')}
-            </Button>
-          )}
-          <Button
-            isDisabled={uploading}
-            onClick={() => {
-              onReSplitChunks();
-
-              openConfirm(onclickUpload)();
-            }}
-          >
-            {uploading ? (
-              <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
-            ) : (
-              t('common.Confirm Import')
-            )}
-          </Button>
-        </Flex>
-      </SelectorContainer>
-
-      {!isUnselectedFile && (
-        <Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
-          <PreviewFileOrChunk />
-        </Box>
-      )}
-      <ConfirmModal />
-    </Box>
-  );
-};
-
-export default ChunkImport;

projects/app/src/pages/dataset/detail/components/Import/CreateFileModal.tsx

@@ -1,72 +0,0 @@
-import React from 'react';
-import { useTranslation } from 'next-i18next';
-import MyModal from '@/components/MyModal';
-import { Box, Input, Textarea, ModalBody, ModalFooter, Button } from '@chakra-ui/react';
-import { useForm } from 'react-hook-form';
-import { useRequest } from '@/web/common/hooks/useRequest';
-
-const CreateFileModal = ({
-  onClose,
-  onSuccess
-}: {
-  onClose: () => void;
-  onSuccess: (e: { filename: string; content: string }) => Promise<void>;
-}) => {
-  const { t } = useTranslation();
-  const { register, handleSubmit } = useForm({
-    defaultValues: {
-      filename: '',
-      content: ''
-    }
-  });
-
-  const { mutate, isLoading } = useRequest({
-    mutationFn: () => handleSubmit(onSuccess)(),
-    onSuccess: () => {
-      onClose();
-    }
-  });
-
-  return (
-    <MyModal
-      title={t('file.Create File')}
-      iconSrc="/imgs/modal/txt.svg"
-      isOpen
-      w={'600px'}
-      top={'15vh'}
-    >
-      <ModalBody>
-        <Box mb={1} fontSize={'sm'}>
-          {t('common.file.File Name')}
-        </Box>
-        <Input
-          mb={5}
-          {...register('filename', {
-            required: t('common.file.Filename Can not Be Empty')
-          })}
-        />
-        <Box mb={1} fontSize={'sm'}>
-          {t('common.file.File Content')}
-        </Box>
-        <Textarea
-          {...register('content', {
-            required: t('common.file.File content can not be empty')
-          })}
-          rows={12}
-          whiteSpace={'nowrap'}
-          resize={'both'}
-        />
-      </ModalBody>
-      <ModalFooter>
-        <Button variant={'whiteBase'} mr={4} onClick={onClose}>
-          {t('common.Close')}
-        </Button>
-        <Button isLoading={isLoading} onClick={mutate}>
-          {t('common.Confirm Create')}
-        </Button>
-      </ModalFooter>
-    </MyModal>
-  );
-};
-
-export default CreateFileModal;

projects/app/src/pages/dataset/detail/components/Import/Csv.tsx

@@ -1,90 +0,0 @@
-import React from 'react';
-import { Box, Flex, Button, Grid } from '@chakra-ui/react';
-import { useConfirm } from '@/web/common/hooks/useConfirm';
-import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
-import { useTranslation } from 'next-i18next';
-import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-import MyTooltip from '@/components/MyTooltip';
-import { QuestionOutlineIcon } from '@chakra-ui/icons';
-
-const fileExtension = '.csv';
-const csvTemplate = `index,content
-"必填内容","可选内容。CSV 中请注意内容不能包含双引号，双引号是列分割符号"
-"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
-"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
-
-const CsvImport = () => {
-  const { t } = useTranslation();
-  const {
-    successChunks,
-    totalChunks,
-    isUnselectedFile,
-    onclickUpload,
-    uploading,
-    totalTokens,
-    price
-  } = useImportStore();
-  const { datasetDetail } = useDatasetStore();
-
-  const vectorModel = datasetDetail.vectorModel;
-  const unitPrice = vectorModel?.inputPrice || 0.002;
-
-  const { openConfirm, ConfirmModal } = useConfirm({
-    content: t('core.dataset.import.Import Tip')
-  });
-
-  return (
-    <Box display={['block', 'flex']} h={['auto', '100%']}>
-      <SelectorContainer
-        fileExtension={fileExtension}
-        showUrlFetch={false}
-        fileTemplate={{
-          filename: 'csv templates.csv',
-          value: csvTemplate,
-          type: 'text/csv'
-        }}
-        tip={t('dataset.import csv tip')}
-      >
-        <Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
-          <Flex alignItems={'center'}>
-            <Box>{t('core.dataset.import.Total tokens')}：</Box>
-            <Box>{totalTokens}</Box>
-          </Flex>
-          {/* price */}
-          <Flex alignItems={'center'}>
-            <Box>
-              {t('core.dataset.import.Estimated Price')}
-              <MyTooltip
-                label={t('core.dataset.import.Embedding Estimated Price Tips', {
-                  price: unitPrice
-                })}
-                forceShow
-              >
-                <QuestionOutlineIcon ml={1} />
-              </MyTooltip>
-            </Box>
-            <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
-          </Flex>
-        </Grid>
-        <Flex mt={3}>
-          <Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
-            {uploading ? (
-              <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
-            ) : (
-              t('common.Confirm Import')
-            )}
-          </Button>
-        </Flex>
-      </SelectorContainer>
-
-      {!isUnselectedFile && (
-        <Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
-          <PreviewFileOrChunk />
-        </Box>
-      )}
-      <ConfirmModal />
-    </Box>
-  );
-};
-
-export default CsvImport;

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -1,446 +0,0 @@
-import MyIcon from '@fastgpt/web/components/common/Icon';
-import { useLoading } from '@/web/common/hooks/useLoading';
-import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
-import { useToast } from '@/web/common/hooks/useToast';
-import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
-import { simpleText } from '@fastgpt/global/common/string/tools';
-import { fileDownload, readCsvContent } from '@/web/common/file/utils';
-import { getUploadBase64ImgController, uploadFiles } from '@/web/common/file/controller';
-import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
-import React, { DragEvent, useCallback, useState } from 'react';
-import { useTranslation } from 'next-i18next';
-import { customAlphabet } from 'nanoid';
-import dynamic from 'next/dynamic';
-import MyTooltip from '@/components/MyTooltip';
-import { getErrText } from '@fastgpt/global/common/error/utils';
-import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-import { getFileIcon } from '@fastgpt/global/common/file/icon';
-import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
-import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
-import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
-import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
-import { readFileRawContent } from '@fastgpt/web/common/file/read/index';
-import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
-
-const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
-const CreateFileModal = dynamic(() => import('./CreateFileModal'));
-
-const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
-
-export type FileItemType = {
-  id: string; // fileId / raw Link
-  filename: string;
-  chunks: PushDatasetDataChunkProps[];
-  rawText: string; // raw text
-  icon: string;
-  tokens: number; // total tokens
-  type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
-  fileId?: string;
-  rawLink?: string;
-  metadata?: Record<string, any>;
-};
-
-export interface Props extends BoxProps {
-  fileExtension: string;
-  onPushFiles: (files: FileItemType[]) => void;
-  tipText?: string;
-  chunkLen?: number;
-  customSplitChar?: string;
-  overlapRatio?: number;
-  fileTemplate?: {
-    type: string;
-    filename: string;
-    value: string;
-  };
-  showUrlFetch?: boolean;
-  showCreateFile?: boolean;
-  tip?: string;
-}
-
-const FileSelect = ({
-  fileExtension,
-  onPushFiles,
-  tipText,
-  chunkLen = 500,
-  customSplitChar,
-  overlapRatio,
-  fileTemplate,
-  showUrlFetch = true,
-  showCreateFile = true,
-  tip,
-  ...props
-}: Props) => {
-  const { datasetDetail } = useDatasetStore();
-  const { Loading: FileSelectLoading } = useLoading();
-  const { t } = useTranslation();
-
-  const { toast } = useToast();
-
-  const { File: FileSelector, onOpen } = useSelectFile({
-    fileType: fileExtension,
-    multiple: true
-  });
-
-  const [isDragging, setIsDragging] = useState(false);
-  const [selectingText, setSelectingText] = useState<string>();
-
-  const {
-    isOpen: isOpenUrlFetch,
-    onOpen: onOpenUrlFetch,
-    onClose: onCloseUrlFetch
-  } = useDisclosure();
-  const {
-    isOpen: isOpenCreateFile,
-    onOpen: onOpenCreateFile,
-    onClose: onCloseCreateFile
-  } = useDisclosure();
-
-  // select file
-  const onSelectFile = useCallback(
-    async (files: File[]) => {
-      if (files.length >= 100) {
-        return toast({
-          status: 'warning',
-          title: t('common.file.Select file amount limit 100')
-        });
-      }
-
-      try {
-        for await (let file of files) {
-          const extension = file?.name?.split('.')?.pop()?.toLowerCase();
-
-          /* text file */
-          const icon = getFileIcon(file?.name);
-
-          // ts
-          if (!icon) continue;
-
-          // upload file
-          const filesId = await uploadFiles({
-            files: [file],
-            bucketName: 'dataset',
-            metadata: { datasetId: datasetDetail._id },
-            percentListen: (percent) => {
-              if (percent < 100) {
-                setSelectingText(
-                  t('file.Uploading', { name: file.name.slice(0, 30), percent }) || ''
-                );
-              } else {
-                setSelectingText(t('file.Parse', { name: file.name.slice(0, 30) }) || '');
-              }
-            }
-          });
-          const fileId = filesId[0];
-
-          /* QA csv file */
-          if (extension === 'csv') {
-            const { header, data } = await readCsvContent(file);
-            if (header[0] !== 'index' || header[1] !== 'content') {
-              throw new Error(t('core.dataset.import.Csv format error'));
-            }
-
-            const filterData = data
-              .filter((item) => item[0])
-              .map((item) => ({
-                q: item[0] || '',
-                a: item[1] || ''
-              }));
-
-            const fileItem: FileItemType = {
-              id: nanoid(),
-              filename: file.name,
-              icon,
-              tokens: filterData.reduce((sum, item) => sum + countPromptTokens(item.q), 0),
-              rawText: `${header.join(',')}\n${data
-                .map((item) => `"${item[0]}","${item[1]}"`)
-                .join('\n')}`,
-              chunks: filterData,
-              type: DatasetCollectionTypeEnum.file,
-              fileId
-            };
-
-            onPushFiles([fileItem]);
-            continue;
-          }
-
-          // parse and upload files
-          let { rawText } = await readFileRawContent({
-            file,
-            uploadBase64Controller: (base64Img) =>
-              getUploadBase64ImgController({
-                base64Img,
-                type: MongoImageTypeEnum.docImage,
-                metadata: {
-                  fileId
-                }
-              })
-          });
-
-          if (rawText) {
-            rawText = simpleText(rawText);
-            const { chunks, tokens } = splitText2Chunks({
-              text: rawText,
-              chunkLen,
-              overlapRatio,
-              customReg: customSplitChar ? [customSplitChar] : []
-            });
-
-            const fileItem: FileItemType = {
-              id: nanoid(),
-              filename: file.name,
-              icon,
-              rawText,
-              tokens,
-              type: DatasetCollectionTypeEnum.file,
-              fileId,
-              chunks: chunks.map((chunk) => ({
-                q: chunk,
-                a: ''
-              }))
-            };
-            onPushFiles([fileItem]);
-          }
-        }
-      } catch (error: any) {
-        console.log(error);
-        toast({
-          title: getErrText(error, t('common.file.Read File Error')),
-          status: 'error'
-        });
-      }
-      setSelectingText(undefined);
-    },
-    [chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
-  );
-  // link fetch
-  const onUrlFetch = useCallback(
-    (e: UrlFetchResponse) => {
-      const result: FileItemType[] = e.map<FileItemType>(({ url, content, selector }) => {
-        const { chunks, tokens } = splitText2Chunks({
-          text: content,
-          chunkLen,
-          overlapRatio,
-          customReg: customSplitChar ? [customSplitChar] : []
-        });
-        return {
-          id: nanoid(),
-          filename: url,
-          icon: '/imgs/files/link.svg',
-          rawText: content,
-          tokens,
-          type: DatasetCollectionTypeEnum.link,
-          rawLink: url,
-          chunks: chunks.map((chunk) => ({
-            q: chunk,
-            a: ''
-          })),
-          metadata: {
-            webPageSelector: selector
-          }
-        };
-      });
-      onPushFiles(result);
-    },
-    [chunkLen, customSplitChar, onPushFiles, overlapRatio]
-  );
-  // manual create file and copy data
-  const onCreateFile = useCallback(
-    async ({ filename, content }: { filename: string; content: string }) => {
-      content = simpleText(content);
-
-      // create virtual txt file
-      const txtBlob = new Blob([content], { type: 'text/plain' });
-      const txtFile = new File([txtBlob], `${filename}.txt`, {
-        type: txtBlob.type,
-        lastModified: new Date().getTime()
-      });
-      const fileIds = await uploadFiles({
-        files: [txtFile],
-        bucketName: 'dataset',
-        metadata: { datasetId: datasetDetail._id }
-      });
-
-      const { chunks, tokens } = splitText2Chunks({
-        text: content,
-        chunkLen,
-        overlapRatio,
-        customReg: customSplitChar ? [customSplitChar] : []
-      });
-
-      onPushFiles([
-        {
-          id: nanoid(),
-          filename,
-          icon: '/imgs/files/txt.svg',
-          rawText: content,
-          tokens,
-          type: DatasetCollectionTypeEnum.file,
-          fileId: fileIds[0],
-          chunks: chunks.map((chunk) => ({
-            q: chunk,
-            a: ''
-          }))
-        }
-      ]);
-    },
-    [chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio]
-  );
-
-  const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
-    e.preventDefault();
-    setIsDragging(true);
-  };
-
-  const handleDragLeave = (e: DragEvent<HTMLDivElement>) => {
-    e.preventDefault();
-    setIsDragging(false);
-  };
-
-  const handleDrop = useCallback(
-    async (e: DragEvent<HTMLDivElement>) => {
-      e.preventDefault();
-      setIsDragging(false);
-
-      const items = e.dataTransfer.items;
-      const fileList: File[] = [];
-
-      if (e.dataTransfer.items.length <= 1) {
-        const traverseFileTree = async (item: any) => {
-          return new Promise<void>((resolve, reject) => {
-            if (item.isFile) {
-              item.file((file: File) => {
-                fileList.push(file);
-                resolve();
-              });
-            } else if (item.isDirectory) {
-              const dirReader = item.createReader();
-              dirReader.readEntries(async (entries: any[]) => {
-                for (let i = 0; i < entries.length; i++) {
-                  await traverseFileTree(entries[i]);
-                }
-                resolve();
-              });
-            }
-          });
-        };
-
-        for (let i = 0; i < items.length; i++) {
-          const item = items[i].webkitGetAsEntry();
-          if (item) {
-            await traverseFileTree(item);
-          }
-        }
-      } else {
-        const files = Array.from(e.dataTransfer.files);
-        let isErr = files.some((item) => item.type === '');
-        if (isErr) {
-          return toast({
-            title: t('file.upload error description'),
-            status: 'error'
-          });
-        }
-
-        for (let i = 0; i < files.length; i++) {
-          fileList.push(files[i]);
-        }
-      }
-
-      onSelectFile(fileList);
-    },
-    [onSelectFile, t, toast]
-  );
-
-  const SelectTextStyles: BoxProps = {
-    ml: 1,
-    as: 'span',
-    cursor: 'pointer',
-    color: 'primary.600',
-    _hover: {
-      textDecoration: 'underline'
-    }
-  };
-
-  return (
-    <Box
-      display={'inline-block'}
-      textAlign={'center'}
-      bg={'myWhite.400'}
-      p={5}
-      borderRadius={'md'}
-      border={'1px dashed'}
-      borderColor={'myGray.300'}
-      w={'100%'}
-      position={'relative'}
-      {...props}
-      onDragEnter={handleDragEnter}
-      onDragOver={handleDragEnter}
-      onDragLeave={handleDragLeave}
-      onDrop={handleDrop}
-    >
-      <Flex justifyContent={'center'} alignItems={'center'}>
-        <MyIcon mr={1} name={'file/uploadFile'} w={'16px'} />
-        {isDragging ? (
-          t('file.Release the mouse to upload the file')
-        ) : (
-          <Box>
-            {t('file.Drag and drop')},
-            <MyTooltip label={t('file.max 10')}>
-              <Box {...SelectTextStyles} onClick={onOpen}>
-                {t('file.select a document')}
-              </Box>
-            </MyTooltip>
-            {showUrlFetch && (
-              <>
-                ,
-                <Box {...SelectTextStyles} onClick={onOpenUrlFetch}>
-                  {t('file.Fetch Url')}
-                </Box>
-              </>
-            )}
-            {showCreateFile && (
-              <>
-                ,
-                <Box {...SelectTextStyles} onClick={onOpenCreateFile}>
-                  {t('file.Create file')}
-                </Box>
-              </>
-            )}
-          </Box>
-        )}
-      </Flex>
-      <Box mt={1}>{t('file.support', { fileExtension: fileExtension })}</Box>
-      {tipText && (
-        <Box mt={1} fontSize={'sm'} color={'myGray.600'}>
-          {t(tipText)}
-        </Box>
-      )}
-      {!!fileTemplate && (
-        <Box
-          mt={1}
-          cursor={'pointer'}
-          textDecoration={'underline'}
-          color={'primary.500'}
-          fontSize={'12px'}
-          onClick={() =>
-            fileDownload({
-              text: fileTemplate.value,
-              type: fileTemplate.type,
-              filename: fileTemplate.filename
-            })
-          }
-        >
-          {t('file.Click to download file template', { name: fileTemplate.filename })}
-        </Box>
-      )}
-      {!!tip && <Box color={'myGray.500'}>{tip}</Box>}
-      {selectingText !== undefined && (
-        <FileSelectLoading loading text={selectingText} fixed={false} />
-      )}
-      <FileSelector onSelect={onSelectFile} />
-      {isOpenUrlFetch && <UrlFetchModal onClose={onCloseUrlFetch} onSuccess={onUrlFetch} />}
-      {isOpenCreateFile && <CreateFileModal onClose={onCloseCreateFile} onSuccess={onCreateFile} />}
-    </Box>
-  );
-};
-
-export default FileSelect;

projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx

@@ -1,138 +0,0 @@
-import React, { useMemo, useState } from 'react';
-import { Box, type BoxProps, Flex, useTheme, ModalCloseButton } from '@chakra-ui/react';
-import MyRadio from '@/components/common/MyRadio/index';
-import dynamic from 'next/dynamic';
-import ChunkImport from './Chunk';
-import { useTranslation } from 'next-i18next';
-
-const QAImport = dynamic(() => import('./QA'), {});
-const CsvImport = dynamic(() => import('./Csv'), {});
-import MyModal from '@/components/MyModal';
-import Provider from './Provider';
-import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
-
-export enum ImportTypeEnum {
-  chunk = 'chunk',
-  qa = 'qa',
-  csv = 'csv'
-}
-
-const ImportData = ({
-  datasetId,
-  parentId,
-  onClose,
-  uploadSuccess
-}: {
-  datasetId: string;
-  parentId: string;
-  onClose: () => void;
-  uploadSuccess: () => void;
-}) => {
-  const { t } = useTranslation();
-  const theme = useTheme();
-  const { datasetDetail } = useDatasetStore();
-  const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.chunk);
-  const vectorModel = datasetDetail.vectorModel;
-  const agentModel = datasetDetail.agentModel;
-
-  const typeMap = useMemo(() => {
-    const map = {
-      [ImportTypeEnum.chunk]: {
-        defaultChunkLen: vectorModel?.defaultToken || 500,
-        chunkOverlapRatio: 0.2,
-        inputPrice: vectorModel?.inputPrice || 0,
-        outputPrice: 0,
-        collectionTrainingType: TrainingModeEnum.chunk
-      },
-      [ImportTypeEnum.qa]: {
-        defaultChunkLen: agentModel?.maxContext * 0.55 || 8000,
-        chunkOverlapRatio: 0,
-        inputPrice: agentModel?.inputPrice || 0,
-        outputPrice: agentModel?.outputPrice || 0,
-        collectionTrainingType: TrainingModeEnum.qa
-      },
-      [ImportTypeEnum.csv]: {
-        defaultChunkLen: 0,
-        chunkOverlapRatio: 0,
-        inputPrice: vectorModel?.inputPrice || 0,
-        outputPrice: 0,
-        collectionTrainingType: TrainingModeEnum.chunk
-      }
-    };
-    return map[importType];
-  }, [
-    agentModel?.inputPrice,
-    agentModel?.maxContext,
-    agentModel?.outputPrice,
-    importType,
-    vectorModel?.defaultToken,
-    vectorModel?.inputPrice
-  ]);
-
-  const TitleStyle: BoxProps = {
-    fontWeight: 'bold',
-    fontSize: ['md', 'xl']
-  };
-
-  return (
-    <MyModal
-      iconSrc="/imgs/modal/import.svg"
-      title={<Box {...TitleStyle}>{t('dataset.data.File import')}</Box>}
-      isOpen
-      isCentered
-      maxW={['90vw', 'min(1440px,85vw)']}
-      w={['90vw', 'min(1440px,85vw)']}
-      h={'90vh'}
-    >
-      <ModalCloseButton onClick={onClose} />
-      <Flex mt={2} flexDirection={'column'} flex={'1 0 0'}>
-        <Box pb={[5, 7]} px={[4, 8]} borderBottom={theme.borders.base}>
-          <MyRadio
-            gridTemplateColumns={['repeat(1,1fr)', 'repeat(3,1fr)']}
-            list={[
-              {
-                icon: 'file/indexImport',
-                title: t('core.dataset.import.Chunk Split'),
-                desc: t('core.dataset.import.Chunk Split Tip'),
-                value: ImportTypeEnum.chunk
-              },
-              {
-                icon: 'file/qaImport',
-                title: t('core.dataset.import.QA Import'),
-                desc: t('core.dataset.import.QA Import Tip'),
-                value: ImportTypeEnum.qa
-              },
-              {
-                icon: 'file/csv',
-                title: t('core.dataset.import.CSV Import'),
-                desc: t('core.dataset.import.CSV Import Tip'),
-                value: ImportTypeEnum.csv
-              }
-            ]}
-            value={importType}
-            onChange={(e) => setImportType(e as `${ImportTypeEnum}`)}
-          />
-        </Box>
-
-        <Provider
-          {...typeMap}
-          vectorModel={vectorModel.model}
-          agentModel={agentModel.model}
-          datasetId={datasetDetail._id}
-          importType={importType}
-          parentId={parentId}
-          onUploadSuccess={uploadSuccess}
-        >
-          <Box flex={'1 0 0'} h={0}>
-            {importType === ImportTypeEnum.chunk && <ChunkImport />}
-            {importType === ImportTypeEnum.qa && <QAImport />}
-            {importType === ImportTypeEnum.csv && <CsvImport />}
-          </Box>
-        </Provider>
-      </Flex>
-    </MyModal>
-  );
-};
-
-export default ImportData;

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -1,502 +1,207 @@
-import React, {
-  type SetStateAction,
-  type Dispatch,
-  useContext,
-  useCallback,
-  createContext,
-  useState,
-  useMemo,
-  useEffect
-} from 'react';
-import FileSelect, { FileItemType, Props as FileSelectProps } from './FileSelect';
-import { useRequest } from '@/web/common/hooks/useRequest';
-import { postDatasetCollection } from '@/web/core/dataset/api';
+import React, { useContext, useCallback, createContext, useState, useMemo, useEffect } from 'react';
+
 import { formatModelPrice2Read } from '@fastgpt/global/support/wallet/bill/tools';
 import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
-import { hashStr } from '@fastgpt/global/common/string/tools';
-import { useToast } from '@/web/common/hooks/useToast';
-import { getErrText } from '@fastgpt/global/common/error/utils';
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
-import { Box, Flex, Image, useTheme } from '@chakra-ui/react';
-import { CloseIcon } from '@chakra-ui/icons';
-import DeleteIcon, { hoverDeleteStyles } from '@fastgpt/web/components/common/Icon/delete';
-import MyIcon from '@fastgpt/web/components/common/Icon';
-import { chunksUpload } from '@/web/core/dataset/utils';
-import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { useTranslation } from 'next-i18next';
-import { ImportTypeEnum } from './ImportModal';
+import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
+import { Prompt_AgentQA } from '@/global/core/prompt/agent';
+import { UseFormReturn, useForm } from 'react-hook-form';
+import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
+import { ImportSourceItemType } from '@/web/core/dataset/type';
 
-const filenameStyles = {
-  className: 'textEllipsis',
-  maxW: '400px'
+type ChunkSizeFieldType = 'embeddingChunkSize';
+export type FormType = {
+  mode: `${TrainingModeEnum}`;
+  way: `${ImportProcessWayEnum}`;
+  embeddingChunkSize: number;
+  customSplitChar: string;
+  qaPrompt: string;
+  webSelector: string;
 };
 
 type useImportStoreType = {
-  files: FileItemType[];
-  setFiles: Dispatch<SetStateAction<FileItemType[]>>;
-  previewFile: FileItemType | undefined;
-  setPreviewFile: Dispatch<SetStateAction<FileItemType | undefined>>;
-  successChunks: number;
-  setSuccessChunks: Dispatch<SetStateAction<number>>;
-  isUnselectedFile: boolean;
-  totalChunks: number;
-  totalTokens: number;
-  onclickUpload: (e?: { prompt?: string }) => void;
-  onReSplitChunks: () => void;
-  price: number;
-  uploading: boolean;
-  chunkLen: number;
-  chunkOverlapRatio: number;
-  setChunkLen: Dispatch<number>;
-  customSplitChar?: string;
-  setCustomSplitChar: Dispatch<string>;
+  parentId?: string;
+  processParamsForm: UseFormReturn<FormType, any>;
+  chunkSizeField?: ChunkSizeFieldType;
+  maxChunkSize: number;
+  minChunkSize: number;
+  showChunkInput: boolean;
+  showPromptInput: boolean;
+  sources: ImportSourceItemType[];
+  setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
   showRePreview: boolean;
-  setReShowRePreview: Dispatch<SetStateAction<boolean>>;
+  totalChunkChars: number;
+  totalChunks: number;
+  chunkSize: number;
+  predictPrice: number;
+  priceTip: string;
+  uploadRate: number;
+  splitSources2Chunks: () => void;
 };
 const StateContext = createContext<useImportStoreType>({
-  onclickUpload: function (e?: { prompt?: string }): void {
-    throw new Error('Function not implemented.');
-  },
-  uploading: false,
-  files: [],
-
-  previewFile: undefined,
-
-  successChunks: 0,
-
-  isUnselectedFile: false,
-  totalChunks: 0,
-  totalTokens: 0,
-  onReSplitChunks: function (): void {
-    throw new Error('Function not implemented.');
-  },
-  price: 0,
-  chunkLen: 0,
-  chunkOverlapRatio: 0,
-  customSplitChar: undefined,
-  setCustomSplitChar: function (value: string): void {
-    throw new Error('Function not implemented.');
-  },
-  setChunkLen: function (value: number): void {
-    throw new Error('Function not implemented.');
-  },
-  setFiles: function (value: React.SetStateAction<FileItemType[]>): void {
-    throw new Error('Function not implemented.');
-  },
-  setPreviewFile: function (value: React.SetStateAction<FileItemType | undefined>): void {
-    throw new Error('Function not implemented.');
-  },
-  setSuccessChunks: function (value: React.SetStateAction<number>): void {
+  processParamsForm: {} as any,
+  sources: [],
+  setSources: function (value: React.SetStateAction<ImportSourceItemType[]>): void {
     throw new Error('Function not implemented.');
   },
+  maxChunkSize: 0,
+  minChunkSize: 0,
+  showChunkInput: false,
+  showPromptInput: false,
+  chunkSizeField: 'embeddingChunkSize',
   showRePreview: false,
-  setReShowRePreview: function (value: React.SetStateAction<boolean>): void {
-    throw new Error('Function not implemented.');
-  }
+  totalChunkChars: 0,
+  totalChunks: 0,
+  chunkSize: 0,
+  predictPrice: 0,
+  priceTip: '',
+  uploadRate: 50,
+  splitSources2Chunks: () => {}
 });
+
 export const useImportStore = () => useContext(StateContext);
 
 const Provider = ({
-  datasetId,
+  dataset,
   parentId,
-  inputPrice,
-  outputPrice,
-  collectionTrainingType,
-  vectorModel,
-  agentModel,
-  defaultChunkLen = 500,
-  chunkOverlapRatio = 0.2,
-  importType,
-  onUploadSuccess,
   children
 }: {
-  datasetId: string;
-  parentId: string;
-  inputPrice: number;
-  outputPrice: number;
-  collectionTrainingType: `${TrainingModeEnum}`;
-  vectorModel: string;
-  agentModel: string;
-  defaultChunkLen: number;
-  chunkOverlapRatio: number;
-  importType: `${ImportTypeEnum}`;
-  onUploadSuccess: () => void;
+  dataset: DatasetItemType;
+  parentId?: string;
   children: React.ReactNode;
 }) => {
+  const vectorModel = dataset.vectorModel;
+  const agentModel = dataset.agentModel;
+
+  const processParamsForm = useForm<FormType>({
+    defaultValues: {
+      mode: TrainingModeEnum.chunk,
+      way: ImportProcessWayEnum.auto,
+      embeddingChunkSize: vectorModel?.defaultToken || 512,
+      customSplitChar: '',
+      qaPrompt: Prompt_AgentQA.description,
+      webSelector: ''
+    }
+  });
+
   const { t } = useTranslation();
-  const { toast } = useToast();
-  const [files, setFiles] = useState<FileItemType[]>([]);
-  const [successChunks, setSuccessChunks] = useState(0);
-  const [chunkLen, setChunkLen] = useState(defaultChunkLen);
-  const [customSplitChar, setCustomSplitChar] = useState<string>();
-  const [previewFile, setPreviewFile] = useState<FileItemType>();
-  const [showRePreview, setReShowRePreview] = useState(false);
+  const [sources, setSources] = useState<ImportSourceItemType[]>([]);
+  const [showRePreview, setShowRePreview] = useState(false);
 
-  const isUnselectedFile = useMemo(() => files.length === 0, [files]);
+  // watch form
+  const mode = processParamsForm.watch('mode');
+  const way = processParamsForm.watch('way');
+  const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize');
+  const customSplitChar = processParamsForm.watch('customSplitChar');
 
-  const totalChunks = useMemo(
-    () => files.reduce((sum, file) => sum + file.chunks.length, 0),
-    [files]
+  const modeStaticParams = {
+    [TrainingModeEnum.chunk]: {
+      chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType,
+      chunkOverlapRatio: 0.2,
+      maxChunkSize: vectorModel?.maxToken || 512,
+      minChunkSize: 100,
+      autoChunkSize: vectorModel?.defaultToken || 512,
+      chunkSize: embeddingChunkSize,
+      showChunkInput: true,
+      showPromptInput: false,
+      inputPrice: vectorModel.inputPrice,
+      outputPrice: 0,
+      priceTip: t('core.dataset.import.Embedding Estimated Price Tips', {
+        price: vectorModel.inputPrice
+      }),
+      uploadRate: 150
+    },
+    [TrainingModeEnum.qa]: {
+      chunkOverlapRatio: 0,
+      maxChunkSize: 8000,
+      minChunkSize: 3000,
+      autoChunkSize: agentModel.maxContext * 0.55 || 6000,
+      chunkSize: agentModel.maxContext * 0.55 || 6000,
+      showChunkInput: false,
+      showPromptInput: true,
+      inputPrice: agentModel.inputPrice,
+      outputPrice: agentModel.outputPrice,
+      priceTip: t('core.dataset.import.QA Estimated Price Tips', {
+        price: agentModel?.inputPrice
+      }),
+      uploadRate: 30
+    }
+  };
+  const selectModelStaticParam = useMemo(() => modeStaticParams[mode], [mode]);
+
+  const wayStaticPrams = {
+    [ImportProcessWayEnum.auto]: {
+      chunkSize: selectModelStaticParam.autoChunkSize,
+      customSplitChar: ''
+    },
+    [ImportProcessWayEnum.custom]: {
+      chunkSize: modeStaticParams[mode].chunkSize,
+      customSplitChar
+    }
+  };
+
+  const chunkSize = wayStaticPrams[way].chunkSize;
+
+  useEffect(() => {
+    setShowRePreview(true);
+  }, [mode, way, chunkSize, customSplitChar]);
+
+  const totalChunkChars = useMemo(
+    () => sources.reduce((sum, file) => sum + file.chunkChars, 0),
+    [sources]
   );
-
-  const totalTokens = useMemo(() => files.reduce((sum, file) => sum + file.tokens, 0), [files]);
-
-  const price = useMemo(() => {
-    if (collectionTrainingType === TrainingModeEnum.qa) {
-      const inputTotal = totalTokens * inputPrice;
-      const outputTotal = totalTokens * 0.5 * outputPrice;
+  const predictPrice = useMemo(() => {
+    if (mode === TrainingModeEnum.qa) {
+      const inputTotal = totalChunkChars * selectModelStaticParam.inputPrice;
+      const outputTotal = totalChunkChars * 0.5 * selectModelStaticParam.inputPrice;
 
       return formatModelPrice2Read(inputTotal + outputTotal);
     }
-    return formatModelPrice2Read(totalTokens * inputPrice);
-  }, [collectionTrainingType, inputPrice, outputPrice, totalTokens]);
+    return formatModelPrice2Read(totalChunkChars * selectModelStaticParam.inputPrice);
+  }, [mode, selectModelStaticParam.inputPrice, totalChunkChars]);
+  const totalChunks = useMemo(
+    () => sources.reduce((sum, file) => sum + file.chunks.length, 0),
+    [sources]
+  );
 
-  /* 
-    start upload data 
-    1. create training bill
-    2. create collection
-    3. upload chunks
-  */
-  const { mutate: onclickUpload, isLoading: uploading } = useRequest({
-    mutationFn: async (props?: { prompt?: string }) => {
-      const { prompt } = props || {};
-      let totalInsertion = 0;
-      for await (const file of files) {
-        // create training bill
-        const billId = await postCreateTrainingBill({
-          name: file.filename,
-          vectorModel,
-          agentModel
+  const splitSources2Chunks = useCallback(() => {
+    setSources((state) =>
+      state.map((file) => {
+        const { chunks, chars } = splitText2Chunks({
+          text: file.rawText,
+          chunkLen: chunkSize,
+          overlapRatio: selectModelStaticParam.chunkOverlapRatio,
+          customReg: customSplitChar ? [customSplitChar] : []
         });
 
-        // create a file collection and training bill
-        const collectionId = await postDatasetCollection({
-          datasetId,
-          parentId,
-          name: file.filename,
-          type: file.type,
-
-          trainingType: collectionTrainingType,
-          chunkSize: chunkLen,
-          chunkSplitter: customSplitChar,
-          qaPrompt: collectionTrainingType === TrainingModeEnum.qa ? prompt : '',
-
-          fileId: file.fileId,
-          rawLink: file.rawLink,
-
-          rawTextLength: file.rawText.length,
-          hashRawText: hashStr(file.rawText),
-          metadata: file.metadata
-        });
-
-        // upload chunks
-        const chunks = file.chunks;
-        const { insertLen } = await chunksUpload({
-          collectionId,
-          billId,
-          trainingMode: collectionTrainingType,
-          chunks,
-          onUploading: (insertLen) => {
-            setSuccessChunks((state) => state + insertLen);
-          },
-          prompt
-        });
-        totalInsertion += insertLen;
-      }
-      return totalInsertion;
-    },
-    onSuccess(num) {
-      toast({
-        title: t('core.dataset.import.Import Success Tip', { num }),
-        status: 'success'
-      });
-      onUploadSuccess();
-    },
-    errorToast: t('core.dataset.import.Import Failed')
-  });
-
-  const onReSplitChunks = useCallback(async () => {
-    try {
-      setPreviewFile(undefined);
-
-      setFiles((state) =>
-        state.map((file) => {
-          const { chunks, tokens } = splitText2Chunks({
-            text: file.rawText,
-            chunkLen,
-            overlapRatio: chunkOverlapRatio,
-            customReg: customSplitChar ? [customSplitChar] : []
-          });
-
-          return {
-            ...file,
-            tokens,
-            chunks: chunks.map((chunk) => ({
-              q: chunk,
-              a: ''
-            }))
-          };
-        })
-      );
-      setReShowRePreview(false);
-    } catch (error) {
-      toast({
-        status: 'warning',
-        title: getErrText(error, t('core.dataset.import.Set Chunk Error'))
-      });
-    }
-  }, [chunkLen, chunkOverlapRatio, customSplitChar, t, toast]);
-
-  const reset = useCallback(() => {
-    setFiles([]);
-    setSuccessChunks(0);
-    setChunkLen(defaultChunkLen);
-    setPreviewFile(undefined);
-    setReShowRePreview(false);
-  }, [defaultChunkLen]);
-
-  useEffect(() => {
-    reset();
-  }, [importType, reset]);
+        return {
+          ...file,
+          chunkChars: chars,
+          chunks: chunks.map((chunk) => ({
+            q: chunk,
+            a: ''
+          }))
+        };
+      })
+    );
+    setShowRePreview(false);
+  }, [chunkSize, customSplitChar, selectModelStaticParam.chunkOverlapRatio]);
 
   const value = {
-    files,
-    setFiles,
-    previewFile,
-    setPreviewFile,
-    successChunks,
-    setSuccessChunks,
-    isUnselectedFile,
-    totalChunks,
-    totalTokens,
-    price,
-    onReSplitChunks,
-    onclickUpload,
-    uploading,
-    chunkLen,
-    customSplitChar,
-    setCustomSplitChar,
-    chunkOverlapRatio,
-    setChunkLen,
+    parentId,
+    processParamsForm,
+    ...selectModelStaticParam,
+    sources,
+    setSources,
     showRePreview,
-    setReShowRePreview
+    totalChunkChars,
+    totalChunks,
+    chunkSize,
+    predictPrice,
+    splitSources2Chunks
   };
   return <StateContext.Provider value={value}>{children}</StateContext.Provider>;
 };
 
 export default React.memo(Provider);
-
-export const PreviewFileOrChunk = () => {
-  const theme = useTheme();
-  const { t } = useTranslation();
-  const { setFiles, previewFile, setPreviewFile, setReShowRePreview, totalChunks, files } =
-    useImportStore();
-
-  return (
-    <Box h={'100%'} w={'100%'}>
-      {!!previewFile ? (
-        <Box
-          position={'relative'}
-          display={['block', 'flex']}
-          h={'100%'}
-          flexDirection={'column'}
-          pt={[3, 6]}
-          bg={'myWhite.400'}
-        >
-          <Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'} {...filenameStyles}>
-            {previewFile.filename}
-          </Box>
-          <CloseIcon
-            position={'absolute'}
-            right={[4, 8]}
-            top={4}
-            cursor={'pointer'}
-            onClick={() => setPreviewFile(undefined)}
-          />
-          <Box
-            flex={'1 0 0'}
-            h={['auto', 0]}
-            overflow={'overlay'}
-            px={[4, 8]}
-            my={4}
-            // contentEditable
-            // dangerouslySetInnerHTML={{ __html: previewFile.rawText }}
-            fontSize={'sm'}
-            whiteSpace={'pre-wrap'}
-            wordBreak={'break-all'}
-            // onBlur={(e) => {
-            //   // @ts-ignore
-            //   const val = e.target.innerText;
-            //   setReShowRePreview(true);
-
-            //   setFiles((state) =>
-            //     state.map((file) =>
-            //       file.id === previewFile.id
-            //         ? {
-            //             ...file,
-            //             text: val
-            //           }
-            //         : file
-            //     )
-            //   );
-            // }}
-          >
-            {previewFile.rawText}
-          </Box>
-        </Box>
-      ) : (
-        <Box pt={[3, 6]}>
-          <Flex px={[4, 8]} alignItems={'center'}>
-            <Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
-              {t('core.dataset.import.Total Chunk Preview', { totalChunks })}
-            </Box>
-            {totalChunks > 50 && (
-              <Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
-                {t('core.dataset.import.Only Show First 50 Chunk')}
-              </Box>
-            )}
-          </Flex>
-          <Box px={[4, 8]} overflow={'overlay'}>
-            {files.map((file) =>
-              file.chunks.slice(0, 50).map((chunk, i) => (
-                <Box
-                  key={i}
-                  py={4}
-                  bg={'myWhite.500'}
-                  my={2}
-                  borderRadius={'md'}
-                  fontSize={'sm'}
-                  _hover={{ ...hoverDeleteStyles }}
-                >
-                  <Flex mb={1} px={4} userSelect={'none'}>
-                    <Box
-                      flexShrink={0}
-                      px={3}
-                      py={'1px'}
-                      border={theme.borders.base}
-                      borderRadius={'md'}
-                    >
-                      # {i + 1}
-                    </Box>
-                    <Box ml={2} fontSize={'sm'} color={'myhGray.500'} {...filenameStyles}>
-                      {file.filename}
-                    </Box>
-                    <Box flex={1} />
-                    <DeleteIcon
-                      onClick={() => {
-                        setFiles((state) =>
-                          state.map((stateFile) =>
-                            stateFile.id === file.id
-                              ? {
-                                  ...file,
-                                  chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
-                                }
-                              : stateFile
-                          )
-                        );
-                      }}
-                    />
-                  </Flex>
-                  <Box px={4} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
-                    {chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q}
-                  </Box>
-                </Box>
-              ))
-            )}
-          </Box>
-        </Box>
-      )}
-    </Box>
-  );
-};
-
-export const SelectorContainer = ({
-  fileExtension,
-  showUrlFetch,
-  showCreateFile,
-  fileTemplate,
-  tip,
-  children
-}: {
-  fileExtension: string;
-  showUrlFetch?: boolean;
-  showCreateFile?: boolean;
-  fileTemplate?: FileSelectProps['fileTemplate'];
-  tip?: string;
-  children: React.ReactNode;
-}) => {
-  const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen, chunkOverlapRatio } =
-    useImportStore();
-  return (
-    <Box
-      h={'100%'}
-      overflowY={'auto'}
-      flex={['auto', '1 0 400px']}
-      {...(isUnselectedFile
-        ? {}
-        : {
-            maxW: ['auto', '450px']
-          })}
-      p={[4, 8]}
-    >
-      <FileSelect
-        fileExtension={fileExtension}
-        onPushFiles={(files) => {
-          setFiles((state) => files.concat(state));
-        }}
-        chunkLen={chunkLen}
-        overlapRatio={chunkOverlapRatio}
-        showUrlFetch={showUrlFetch}
-        showCreateFile={showCreateFile}
-        fileTemplate={fileTemplate}
-        tip={tip}
-        py={isUnselectedFile ? '100px' : 5}
-      />
-      {!isUnselectedFile && (
-        <Box py={4} px={2} maxH={'400px'} overflowY={'auto'}>
-          {files.map((item) => (
-            <Flex
-              key={item.id}
-              w={'100%'}
-              _notLast={{ mb: 5 }}
-              px={5}
-              py={2}
-              boxShadow={'1px 1px 5px rgba(0,0,0,0.15)'}
-              borderRadius={'md'}
-              cursor={'pointer'}
-              position={'relative'}
-              alignItems={'center'}
-              _hover={{
-                bg: 'primary.50',
-                '& .delete': {
-                  display: 'block'
-                }
-              }}
-              onClick={() => setPreviewFile(item)}
-            >
-              <Image src={item.icon} w={'16px'} alt={''} />
-              <Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
-                {item.filename}
-              </Box>
-              <MyIcon
-                position={'absolute'}
-                right={3}
-                className="delete"
-                name={'delete'}
-                w={'16px'}
-                _hover={{ color: 'red.600' }}
-                display={['block', 'none']}
-                onClick={(e) => {
-                  e.stopPropagation();
-                  setPreviewFile(undefined);
-                  setFiles((state) => state.filter((file) => file.id !== item.id));
-                }}
-              />
-            </Flex>
-          ))}
-        </Box>
-      )}
-      {!isUnselectedFile && <>{children}</>}
-    </Box>
-  );
-};

projects/app/src/pages/dataset/detail/components/Import/QA.tsx

@@ -1,112 +0,0 @@
-import React, { useState } from 'react';
-import { Box, Flex, Button, Textarea, Grid } from '@chakra-ui/react';
-import { useConfirm } from '@/web/common/hooks/useConfirm';
-import MyTooltip from '@/components/MyTooltip';
-import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { Prompt_AgentQA } from '@/global/core/prompt/agent';
-import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
-import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-import { useTranslation } from 'next-i18next';
-
-const fileExtension = '.txt, .docx, .pdf, .md, .html';
-
-const QAImport = () => {
-  const { t } = useTranslation();
-  const { datasetDetail } = useDatasetStore();
-  const agentModel = datasetDetail.agentModel;
-
-  const {
-    successChunks,
-    totalChunks,
-    totalTokens,
-    isUnselectedFile,
-    price,
-    onclickUpload,
-    onReSplitChunks,
-    uploading,
-    showRePreview
-  } = useImportStore();
-
-  const { openConfirm, ConfirmModal } = useConfirm({
-    content: t('core.dataset.import.Import Tip')
-  });
-
-  const [prompt, setPrompt] = useState(Prompt_AgentQA.description);
-
-  return (
-    <Box display={['block', 'flex']} h={['auto', '100%']}>
-      <SelectorContainer fileExtension={fileExtension}>
-        {/* prompt */}
-        <Box p={3} bg={'myWhite.600'} borderRadius={'md'}>
-          <Box mb={1} fontWeight={'bold'}>
-            {t('core.dataset.collection.QA Prompt')}
-          </Box>
-          <Box whiteSpace={'pre-wrap'} fontSize={'sm'}>
-            <Textarea
-              defaultValue={prompt}
-              rows={8}
-              fontSize={'sm'}
-              onChange={(e) => {
-                setPrompt(e.target.value);
-              }}
-            />
-            <Box>{Prompt_AgentQA.fixedText}</Box>
-          </Box>
-        </Box>
-        {/* price */}
-        <Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
-          <Flex alignItems={'center'}>
-            <Box>{t('core.dataset.import.Total tokens')}：</Box>
-            <Box>{totalTokens}</Box>
-          </Flex>
-          {/* price */}
-          <Flex alignItems={'center'}>
-            <Box>
-              {t('core.dataset.import.Estimated Price')}
-              <MyTooltip
-                label={t('core.dataset.import.QA Estimated Price Tips', {
-                  inputPrice: agentModel?.inputPrice,
-                  outputPrice: agentModel?.outputPrice
-                })}
-                forceShow
-              >
-                <QuestionOutlineIcon ml={1} />
-              </MyTooltip>
-            </Box>
-            <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
-          </Flex>
-        </Grid>
-        <Flex mt={3}>
-          {showRePreview && (
-            <Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
-              {t('core.dataset.import.Re Preview')}
-            </Button>
-          )}
-          <Button
-            isDisabled={uploading}
-            onClick={() => {
-              onReSplitChunks();
-
-              openConfirm(() => onclickUpload({ prompt }))();
-            }}
-          >
-            {uploading ? (
-              <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
-            ) : (
-              t('common.Confirm Import')
-            )}
-          </Button>
-        </Flex>
-      </SelectorContainer>
-
-      {!isUnselectedFile && (
-        <Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
-          <PreviewFileOrChunk />
-        </Box>
-      )}
-      <ConfirmModal />
-    </Box>
-  );
-};
-
-export default QAImport;

projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx

@@ -1,94 +0,0 @@
-import React from 'react';
-import { useTranslation } from 'next-i18next';
-import MyModal from '@/components/MyModal';
-import { Box, Button, Input, Link, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
-import { useRequest } from '@/web/common/hooks/useRequest';
-import { postFetchUrls } from '@/web/common/tools/api';
-import { useForm } from 'react-hook-form';
-import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
-import { getDocPath } from '@/web/common/system/doc';
-import { feConfigs } from '@/web/common/system/staticData';
-
-const UrlFetchModal = ({
-  onClose,
-  onSuccess
-}: {
-  onClose: () => void;
-  onSuccess: (e: UrlFetchResponse) => void;
-}) => {
-  const { t } = useTranslation();
-  const { register, handleSubmit } = useForm({
-    defaultValues: {
-      urls: '',
-      selector: ''
-    }
-  });
-
-  const { mutate, isLoading } = useRequest({
-    mutationFn: async ({ urls, selector }: { urls: string; selector: string }) => {
-      const urlList = urls.split('\n').filter((e) => e);
-      const res = await postFetchUrls({
-        urlList,
-        selector
-      });
-
-      onSuccess(res);
-      onClose();
-    },
-    errorToast: t('core.dataset.import.Fetch Error')
-  });
-
-  return (
-    <MyModal
-      iconSrc="/imgs/modal/network.svg"
-      title={
-        <Box>
-          <Box>{t('file.Fetch Url')}</Box>
-          <Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'}>
-            {t('core.dataset.import.Fetch url tip')}
-          </Box>
-        </Box>
-      }
-      top={'15vh'}
-      isOpen
-      onClose={onClose}
-      w={'600px'}
-    >
-      <ModalBody>
-        <Box>
-          <Box fontWeight={'bold'}>{t('core.dataset.import.Fetch Url')}</Box>
-          <Textarea
-            {...register('urls', {
-              required: true
-            })}
-            rows={11}
-            whiteSpace={'nowrap'}
-            resize={'both'}
-            placeholder={t('core.dataset.import.Fetch url placeholder')}
-          />
-        </Box>
-        <Box mt={4}>
-          <Box fontWeight={'bold'}>
-            {t('core.dataset.website.Selector')}({t('common.choosable')})
-          </Box>
-          {feConfigs?.docUrl && (
-            <Link href={getDocPath('/docs/course/websync/#选择器如何使用')} target="_blank">
-              {t('core.dataset.website.Selector Course')}
-            </Link>
-          )}
-          <Input {...register('selector')} placeholder="body .content #document" />
-        </Box>
-      </ModalBody>
-      <ModalFooter>
-        <Button variant={'whiteBase'} mr={4} onClick={onClose}>
-          {t('common.Close')}
-        </Button>
-        <Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>
-          {t('common.Confirm')}
-        </Button>
-      </ModalFooter>
-    </MyModal>
-  );
-};
-
-export default UrlFetchModal;

projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx

@@ -0,0 +1,356 @@
+import React, { useEffect, useRef, useState } from 'react';
+import {
+  Box,
+  Flex,
+  NumberInput,
+  NumberInputField,
+  NumberInputStepper,
+  NumberIncrementStepper,
+  NumberDecrementStepper,
+  Input,
+  Button,
+  ModalBody,
+  ModalFooter,
+  Textarea,
+  useDisclosure
+} from '@chakra-ui/react';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { useTranslation } from 'next-i18next';
+import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
+import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
+import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
+import MyTooltip from '@/components/MyTooltip';
+import { useImportStore } from '../Provider';
+import { feConfigs } from '@/web/common/system/staticData';
+import Tag from '@/components/Tag';
+import MyModal from '@/components/MyModal';
+import { Prompt_AgentQA } from '@/global/core/prompt/agent';
+import Preview from '../components/Preview';
+
+function DataProcess({
+  showPreviewChunks = true,
+  goToNext
+}: {
+  showPreviewChunks: boolean;
+  goToNext: () => void;
+}) {
+  const { t } = useTranslation();
+  const {
+    processParamsForm,
+    sources,
+    chunkSizeField,
+    minChunkSize,
+    showChunkInput,
+    showPromptInput,
+    maxChunkSize,
+    totalChunkChars,
+    totalChunks,
+    predictPrice,
+    showRePreview,
+    splitSources2Chunks,
+    priceTip
+  } = useImportStore();
+  const { getValues, setValue, register } = processParamsForm;
+  const [refresh, setRefresh] = useState(false);
+
+  const {
+    isOpen: isOpenCustomPrompt,
+    onOpen: onOpenCustomPrompt,
+    onClose: onCloseCustomPrompt
+  } = useDisclosure();
+
+  useEffect(() => {
+    if (showPreviewChunks) {
+      splitSources2Chunks();
+    }
+  }, []);
+
+  return (
+    <Box h={'100%'} display={['block', 'flex']} gap={5}>
+      <Box flex={'1 0 0'} maxW={'600px'}>
+        <Flex fontWeight={'bold'} alignItems={'center'}>
+          <MyIcon name={'common/settingLight'} w={'20px'} />
+          <Box fontSize={'lg'}>{t('core.dataset.import.Data process params')}</Box>
+        </Flex>
+
+        <Flex mt={4} alignItems={'center'}>
+          <Box color={'myGray.600'} flex={'0 0 100px'}>
+            {t('core.dataset.import.Training mode')}
+          </Box>
+          <LeftRadio
+            list={Object.entries(TrainingTypeMap).map(([key, value]) => ({
+              title: t(value.label),
+              value: key,
+              tooltip: t(value.tooltip)
+            }))}
+            px={3}
+            py={2}
+            value={getValues('mode')}
+            onChange={(e) => {
+              setValue('mode', e);
+              setRefresh(!refresh);
+            }}
+            gridTemplateColumns={'1fr 1fr'}
+            defaultBg="white"
+            activeBg="white"
+          />
+        </Flex>
+        <Flex mt={5}>
+          <Box color={'myGray.600'} flex={'0 0 100px'}>
+            {t('core.dataset.import.Process way')}
+          </Box>
+          <LeftRadio
+            list={[
+              {
+                title: t('core.dataset.import.Auto process'),
+                desc: t('core.dataset.import.Auto process desc'),
+                value: ImportProcessWayEnum.auto
+              },
+              {
+                title: t('core.dataset.import.Custom process'),
+                desc: t('core.dataset.import.Custom process desc'),
+                value: ImportProcessWayEnum.custom,
+                children: getValues('way') === ImportProcessWayEnum.custom && (
+                  <Box mt={5}>
+                    {showChunkInput && chunkSizeField && (
+                      <Box>
+                        <Flex alignItems={'center'}>
+                          <Box>{t('core.dataset.import.Ideal chunk length')}</Box>
+                          <MyTooltip
+                            label={t('core.dataset.import.Ideal chunk length Tips')}
+                            forceShow
+                          >
+                            <MyIcon
+                              name={'common/questionLight'}
+                              ml={1}
+                              w={'14px'}
+                              color={'myGray.500'}
+                            />
+                          </MyTooltip>
+                        </Flex>
+                        <Box
+                          mt={1}
+                          css={{
+                            '& > span': {
+                              display: 'block'
+                            }
+                          }}
+                        >
+                          <MyTooltip
+                            label={t('core.dataset.import.Chunk Range', {
+                              min: minChunkSize,
+                              max: maxChunkSize
+                            })}
+                          >
+                            <NumberInput
+                              size={'sm'}
+                              step={100}
+                              min={minChunkSize}
+                              max={maxChunkSize}
+                              onChange={(e) => {
+                                setValue(chunkSizeField, +e);
+                              }}
+                            >
+                              <NumberInputField
+                                min={minChunkSize}
+                                max={maxChunkSize}
+                                {...register(chunkSizeField, {
+                                  min: minChunkSize,
+                                  max: maxChunkSize,
+                                  valueAsNumber: true
+                                })}
+                              />
+                              <NumberInputStepper>
+                                <NumberIncrementStepper />
+                                <NumberDecrementStepper />
+                              </NumberInputStepper>
+                            </NumberInput>
+                          </MyTooltip>
+                        </Box>
+                      </Box>
+                    )}
+
+                    <Box mt={3}>
+                      <Box>
+                        {t('core.dataset.import.Custom split char')}
+                        <MyTooltip
+                          label={t('core.dataset.import.Custom split char Tips')}
+                          forceShow
+                        >
+                          <MyIcon
+                            name={'common/questionLight'}
+                            ml={1}
+                            w={'14px'}
+                            color={'myGray.500'}
+                          />
+                        </MyTooltip>
+                      </Box>
+                      <Box mt={1}>
+                        <Input
+                          size={'sm'}
+                          bg={'myGray.50'}
+                          defaultValue={''}
+                          placeholder="\n;======;==SPLIT=="
+                          {...register('customSplitChar')}
+                        />
+                      </Box>
+                    </Box>
+
+                    {showPromptInput && (
+                      <Box mt={3}>
+                        <Box>{t('core.dataset.collection.QA Prompt')}</Box>
+                        <Box
+                          position={'relative'}
+                          py={2}
+                          px={3}
+                          bg={'myGray.50'}
+                          fontSize={'xs'}
+                          whiteSpace={'pre-wrap'}
+                          border={'1px'}
+                          borderColor={'borderColor.base'}
+                          borderRadius={'md'}
+                          maxH={'140px'}
+                          overflow={'auto'}
+                          _hover={{
+                            '& .mask': {
+                              display: 'block'
+                            }
+                          }}
+                        >
+                          {getValues('qaPrompt')}
+
+                          <Box
+                            display={'none'}
+                            className="mask"
+                            position={'absolute'}
+                            top={0}
+                            right={0}
+                            bottom={0}
+                            left={0}
+                            background={
+                              'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
+                            }
+                          >
+                            <Button
+                              size="xs"
+                              variant={'whiteBase'}
+                              leftIcon={<MyIcon name={'edit'} w={'13px'} />}
+                              color={'black'}
+                              position={'absolute'}
+                              right={2}
+                              bottom={2}
+                              onClick={onOpenCustomPrompt}
+                            >
+                              {t('core.dataset.import.Custom prompt')}
+                            </Button>
+                          </Box>
+                        </Box>
+                      </Box>
+                    )}
+                  </Box>
+                )
+              }
+            ]}
+            px={3}
+            py={3}
+            defaultBg="white"
+            activeBg="white"
+            value={getValues('way')}
+            w={'100%'}
+            onChange={(e) => {
+              setValue('way', e);
+              setRefresh(!refresh);
+            }}
+          ></LeftRadio>
+        </Flex>
+        {showPreviewChunks && (
+          <Flex mt={5} alignItems={'center'} pl={'100px'} gap={3}>
+            <Tag colorSchema={'gray'} py={'6px'} borderRadius={'md'} px={3}>
+              {t('core.dataset.Total chunks', { total: totalChunks })}
+            </Tag>
+            <Tag colorSchema={'gray'} py={'6px'} borderRadius={'md'} px={3}>
+              {t('core.Total chars', { total: totalChunkChars })}
+            </Tag>
+            {feConfigs?.show_pay && (
+              <MyTooltip label={priceTip}>
+                <Tag colorSchema={'gray'} py={'6px'} borderRadius={'md'} px={3}>
+                  {t('core.dataset.import.Estimated Price', { amount: predictPrice, unit: '元' })}
+                </Tag>
+              </MyTooltip>
+            )}
+          </Flex>
+        )}
+        <Flex mt={5} gap={3} justifyContent={'flex-end'}>
+          {showPreviewChunks && showRePreview && (
+            <Button variant={'primaryOutline'} onClick={splitSources2Chunks}>
+              {t('core.dataset.import.Re Preview')}
+            </Button>
+          )}
+          <Button
+            onClick={() => {
+              if (showRePreview) {
+                splitSources2Chunks();
+              }
+              goToNext();
+            }}
+          >
+            {t('common.Next Step')}
+          </Button>
+        </Flex>
+      </Box>
+      <Preview sources={sources} showPreviewChunks={showPreviewChunks} />
+
+      {isOpenCustomPrompt && (
+        <PromptTextarea
+          defaultValue={getValues('qaPrompt')}
+          onChange={(e) => {
+            setValue('qaPrompt', e);
+            setRefresh(!refresh);
+          }}
+          onClose={onCloseCustomPrompt}
+        />
+      )}
+    </Box>
+  );
+}
+
+export default React.memo(DataProcess);
+
+const PromptTextarea = ({
+  defaultValue,
+  onChange,
+  onClose
+}: {
+  defaultValue: string;
+  onChange: (e: string) => void;
+  onClose: () => void;
+}) => {
+  const ref = useRef<HTMLTextAreaElement>(null);
+  const { t } = useTranslation();
+
+  return (
+    <MyModal
+      isOpen
+      title={t('core.dataset.import.Custom prompt')}
+      iconSrc="modal/edit"
+      w={'600px'}
+      onClose={onClose}
+    >
+      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
+        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
+        <Box>{Prompt_AgentQA.fixedText}</Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button
+          onClick={() => {
+            const val = ref.current?.value || Prompt_AgentQA.description;
+            onChange(val);
+            onClose();
+          }}
+        >
+          {t('common.Confirm')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};

projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx

@@ -0,0 +1,30 @@
+import React from 'react';
+import { useImportStore } from '../Provider';
+import Preview from '../components/Preview';
+import { Box, Button, Flex } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+
+const PreviewData = ({
+  showPreviewChunks,
+  goToNext
+}: {
+  showPreviewChunks: boolean;
+  goToNext: () => void;
+}) => {
+  const { t } = useTranslation();
+  const { sources, setSources } = useImportStore();
+  console.log(sources);
+
+  return (
+    <Flex flexDirection={'column'} h={'100%'} maxW={'1080px'}>
+      <Box flex={'1 0 0 '}>
+        <Preview showPreviewChunks={showPreviewChunks} sources={sources} />
+      </Box>
+      <Flex mt={2} justifyContent={'flex-end'}>
+        <Button onClick={goToNext}>{t('common.Next Step')}</Button>
+      </Flex>
+    </Flex>
+  );
+};
+
+export default React.memo(PreviewData);

projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx

@@ -0,0 +1,298 @@
+import React, { useEffect, useState } from 'react';
+import {
+  Box,
+  TableContainer,
+  Table,
+  Thead,
+  Tr,
+  Th,
+  Td,
+  Tbody,
+  Progress,
+  Flex,
+  Button
+} from '@chakra-ui/react';
+import { useImportStore, type FormType } from '../Provider';
+import { useTranslation } from 'next-i18next';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { useRequest } from '@/web/common/hooks/useRequest';
+import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
+import { useDatasetStore } from '@/web/core/dataset/store/dataset';
+import { chunksUpload, fileCollectionCreate } from '@/web/core/dataset/utils';
+import { ImportSourceItemType } from '@/web/core/dataset/type';
+import { hashStr } from '@fastgpt/global/common/string/tools';
+import { useToast } from '@/web/common/hooks/useToast';
+import { useRouter } from 'next/router';
+import { TabEnum } from '../../../index';
+import { postCreateDatasetLinkCollection, postDatasetCollection } from '@/web/core/dataset/api';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { checkTeamDatasetSizeLimit } from '@/web/support/user/team/api';
+
+const Upload = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
+  const { t } = useTranslation();
+  const { toast } = useToast();
+  const router = useRouter();
+  const { datasetDetail } = useDatasetStore();
+  const { parentId, sources, processParamsForm, chunkSize, totalChunks, uploadRate } =
+    useImportStore();
+  const [uploadList, setUploadList] = useState<
+    (ImportSourceItemType & {
+      uploadedFileRate: number;
+      uploadedChunksRate: number;
+    })[]
+  >([]);
+
+  const { handleSubmit } = processParamsForm;
+
+  const { mutate: startUpload, isLoading } = useRequest({
+    mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: FormType) => {
+      if (uploadList.length === 0) return;
+
+      await checkTeamDatasetSizeLimit(totalChunks);
+
+      let totalInsertion = 0;
+
+      // Batch create collection and upload chunks
+      for await (const item of uploadList) {
+        const billId = await postCreateTrainingBill({
+          name: item.sourceName,
+          datasetId: datasetDetail._id
+        });
+
+        // create collection
+        const collectionId = await (async () => {
+          const commonParams = {
+            parentId,
+            trainingType: mode,
+            datasetId: datasetDetail._id,
+            chunkSize,
+            chunkSplitter: customSplitChar,
+            qaPrompt,
+
+            name: item.sourceName,
+            rawTextLength: item.rawText.length,
+            hashRawText: hashStr(item.rawText)
+          };
+          if (item.file) {
+            return fileCollectionCreate({
+              file: item.file,
+              data: {
+                ...commonParams,
+                collectionMetadata: {
+                  relatedImgId: item.id
+                }
+              },
+              percentListen: (e) => {
+                setUploadList((state) =>
+                  state.map((uploadItem) =>
+                    uploadItem.id === item.id
+                      ? {
+                          ...uploadItem,
+                          uploadedFileRate: e
+                        }
+                      : uploadItem
+                  )
+                );
+              }
+            });
+          } else if (item.link) {
+            const { collectionId } = await postCreateDatasetLinkCollection({
+              ...commonParams,
+              link: item.link,
+              metadata: {
+                webPageSelector: webSelector
+              }
+            });
+            setUploadList((state) =>
+              state.map((uploadItem) =>
+                uploadItem.id === item.id
+                  ? {
+                      ...uploadItem,
+                      uploadedFileRate: 100
+                    }
+                  : uploadItem
+              )
+            );
+            return collectionId;
+          } else if (item.rawText) {
+            // manual collection
+            return postDatasetCollection({
+              ...commonParams,
+              type: DatasetCollectionTypeEnum.virtual
+            });
+          }
+          return '';
+        })();
+
+        if (!collectionId) continue;
+
+        // upload chunks
+        const chunks = item.chunks;
+        const { insertLen } = await chunksUpload({
+          collectionId,
+          billId,
+          trainingMode: mode,
+          chunks,
+          rate: uploadRate,
+          onUploading: (e) => {
+            setUploadList((state) =>
+              state.map((uploadItem) =>
+                uploadItem.id === item.id
+                  ? {
+                      ...uploadItem,
+                      uploadedChunksRate: e
+                    }
+                  : uploadItem
+              )
+            );
+          },
+          prompt: qaPrompt
+        });
+        totalInsertion += insertLen;
+      }
+
+      return totalInsertion;
+    },
+    onSuccess(num) {
+      if (showPreviewChunks) {
+        toast({
+          title: t('core.dataset.import.Import Success Tip', { num }),
+          status: 'success'
+        });
+      } else {
+        toast({
+          title: t('core.dataset.import.Upload success'),
+          status: 'success'
+        });
+      }
+
+      // close import page
+      router.replace({
+        query: {
+          ...router.query,
+          currentTab: TabEnum.collectionCard
+        }
+      });
+    },
+    errorToast: t('common.file.Upload failed')
+  });
+
+  useEffect(() => {
+    setUploadList(
+      sources.map((item) => {
+        return {
+          ...item,
+          uploadedFileRate: item.file ? 0 : -1,
+          uploadedChunksRate: 0
+        };
+      })
+    );
+  }, []);
+
+  return (
+    <Box>
+      <TableContainer>
+        <Table variant={'simple'} fontSize={'sm'} draggable={false}>
+          <Thead draggable={false}>
+            <Tr bg={'myGray.100'} mb={2}>
+              <Th borderLeftRadius={'md'} overflow={'hidden'} borderBottom={'none'} py={4}>
+                {t('core.dataset.import.Source name')}
+              </Th>
+              {showPreviewChunks ? (
+                <>
+                  <Th borderBottom={'none'} py={4}>
+                    {t('core.dataset.Chunk amount')}
+                  </Th>
+                  <Th borderBottom={'none'} py={4}>
+                    {t('core.dataset.import.Upload file progress')}
+                  </Th>
+                  <Th borderRightRadius={'md'} overflow={'hidden'} borderBottom={'none'} py={4}>
+                    {t('core.dataset.import.Data file progress')}
+                  </Th>
+                </>
+              ) : (
+                <>
+                  <Th borderBottom={'none'} py={4}>
+                    {t('core.dataset.import.Upload status')}
+                  </Th>
+                </>
+              )}
+            </Tr>
+          </Thead>
+          <Tbody>
+            {uploadList.map((item) => (
+              <Tr key={item.id}>
+                <Td display={'flex'} alignItems={'center'}>
+                  <MyIcon name={item.icon as any} w={'16px'} mr={1} />
+                  {item.sourceName}
+                </Td>
+                {showPreviewChunks ? (
+                  <>
+                    <Td>{item.chunks.length}</Td>
+                    <Td>
+                      {item.uploadedFileRate === -1 ? (
+                        '-'
+                      ) : (
+                        <Flex alignItems={'center'} fontSize={'xs'}>
+                          <Progress
+                            value={item.uploadedFileRate}
+                            h={'6px'}
+                            w={'100%'}
+                            maxW={'210px'}
+                            size="sm"
+                            borderRadius={'20px'}
+                            colorScheme={'blue'}
+                            bg="myGray.200"
+                            hasStripe
+                            isAnimated
+                            mr={2}
+                          />
+                          {`${item.uploadedFileRate}%`}
+                        </Flex>
+                      )}
+                    </Td>
+                    <Td>
+                      <Flex alignItems={'center'} fontSize={'xs'}>
+                        <Progress
+                          value={item.uploadedChunksRate}
+                          h={'6px'}
+                          w={'100%'}
+                          maxW={'210px'}
+                          size="sm"
+                          borderRadius={'20px'}
+                          colorScheme={'purple'}
+                          bg="myGray.200"
+                          hasStripe
+                          isAnimated
+                          mr={2}
+                        />
+                        {`${item.uploadedChunksRate}%`}
+                      </Flex>
+                    </Td>
+                  </>
+                ) : (
+                  <>
+                    <Td color={item.uploadedFileRate === 100 ? 'green.600' : 'myGray.600'}>
+                      {item.uploadedFileRate === 100 ? t('common.Finish') : t('common.Waiting')}
+                    </Td>
+                  </>
+                )}
+              </Tr>
+            ))}
+          </Tbody>
+        </Table>
+      </TableContainer>
+
+      <Flex justifyContent={'flex-end'} mt={4}>
+        <Button isLoading={isLoading} onClick={handleSubmit((data) => startUpload(data))}>
+          {uploadList.length > 0
+            ? `${t('core.dataset.import.Total files', { total: uploadList.length })} | `
+            : ''}
+          {t('core.dataset.import.Start upload')}
+        </Button>
+      </Flex>
+    </Box>
+  );
+};
+
+export default Upload;

projects/app/src/pages/dataset/detail/components/Import/components/Preview.tsx

@@ -0,0 +1,134 @@
+import React, { useMemo, useState } from 'react';
+import { Box, Flex } from '@chakra-ui/react';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { useTranslation } from 'next-i18next';
+
+import RowTabs from '@fastgpt/web/components/common/Tabs/RowTabs';
+import { ImportSourceItemType } from '@/web/core/dataset/type';
+
+enum PreviewListEnum {
+  chunks = 'chunks',
+  sources = 'sources'
+}
+
+const Preview = ({
+  sources,
+  showPreviewChunks
+}: {
+  sources: ImportSourceItemType[];
+  showPreviewChunks: boolean;
+}) => {
+  const { t } = useTranslation();
+  const [previewListType, setPreviewListType] = useState(
+    showPreviewChunks ? PreviewListEnum.chunks : PreviewListEnum.sources
+  );
+
+  const chunks = useMemo(() => {
+    const oneSourceChunkLength = Math.max(4, Math.floor(50 / sources.length));
+    return sources
+      .map((source) =>
+        source.chunks.slice(0, oneSourceChunkLength).map((chunk, i) => ({
+          ...chunk,
+          chunkIndex: i + 1,
+          sourceName: source.sourceName,
+          sourceIcon: source.icon
+        }))
+      )
+      .flat();
+  }, [sources]);
+
+  return (
+    <Box h={'100%'} display={['block', 'flex']} flexDirection={'column'} flex={'1 0 0'}>
+      <Box>
+        <RowTabs
+          list={[
+            ...(showPreviewChunks
+              ? [
+                  {
+                    icon: 'common/viewLight',
+                    label: t('core.dataset.import.Preview chunks'),
+                    value: PreviewListEnum.chunks
+                  }
+                ]
+              : []),
+            {
+              icon: 'core/dataset/fileCollection',
+              label: t('core.dataset.import.Sources list'),
+              value: PreviewListEnum.sources
+            }
+          ]}
+          value={previewListType}
+          onChange={(e) => setPreviewListType(e as PreviewListEnum)}
+        />
+      </Box>
+      <Box mt={3} flex={'1 0 0'} overflow={'auto'}>
+        {previewListType === PreviewListEnum.chunks ? (
+          <>
+            {chunks.map((chunk, i) => (
+              <Box
+                key={i}
+                p={4}
+                bg={'white'}
+                mb={3}
+                borderRadius={'md'}
+                borderWidth={'1px'}
+                borderColor={'borderColor.low'}
+                boxShadow={'2'}
+                whiteSpace={'pre-wrap'}
+              >
+                <Flex mb={1} alignItems={'center'} fontSize={'sm'}>
+                  <Box
+                    flexShrink={0}
+                    px={1}
+                    color={'primary.600'}
+                    borderWidth={'1px'}
+                    borderColor={'primary.200'}
+                    bg={'primary.50'}
+                    borderRadius={'sm'}
+                  >
+                    # {chunk.chunkIndex}
+                  </Box>
+                  <Flex ml={2} fontWeight={'bold'} alignItems={'center'} gap={1}>
+                    <MyIcon name={chunk.sourceIcon as any} w={'14px'} />
+                    {chunk.sourceName}
+                  </Flex>
+                </Flex>
+                <Box fontSize={'xs'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
+                  <Box color={'myGray.900'}>{chunk.q}</Box>
+                  <Box color={'myGray.500'}>{chunk.a}</Box>
+                </Box>
+              </Box>
+            ))}
+          </>
+        ) : (
+          <>
+            {sources.map((source) => (
+              <Flex
+                key={source.id}
+                bg={'white'}
+                p={4}
+                borderRadius={'md'}
+                borderWidth={'1px'}
+                borderColor={'borderColor.low'}
+                boxShadow={'2'}
+                mb={3}
+              >
+                <MyIcon name={source.icon as any} w={'16px'} />
+                <Box mx={1} flex={'1 0 0'} className="textEllipsis">
+                  {source.sourceName}
+                </Box>
+                {showPreviewChunks && (
+                  <Box>
+                    {t('core.dataset.import.File chunk amount', { amount: source.chunks.length })}
+                  </Box>
+                )}
+              </Flex>
+            ))}
+          </>
+        )}
+      </Box>
+    </Box>
+  );
+};
+
+export default React.memo(Preview);

projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx

@@ -0,0 +1,28 @@
+import React from 'react';
+import MyModal from '@/components/MyModal';
+import { ModalBody } from '@chakra-ui/react';
+
+export type PreviewRawTextProps = {
+  icon: string;
+  title: string;
+  rawText: string;
+};
+
+const PreviewRawText = ({
+  icon,
+  title,
+  rawText,
+  onClose
+}: PreviewRawTextProps & {
+  onClose: () => void;
+}) => {
+  return (
+    <MyModal isOpen onClose={onClose} iconSrc={icon} title={title}>
+      <ModalBody whiteSpace={'pre-wrap'} overflowY={'auto'}>
+        {rawText}
+      </ModalBody>
+    </MyModal>
+  );
+};
+
+export default PreviewRawText;

projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx

@@ -0,0 +1,103 @@
+import React, { useEffect } from 'react';
+import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
+
+import dynamic from 'next/dynamic';
+import { useImportStore } from '../Provider';
+import { useTranslation } from 'next-i18next';
+import { useForm } from 'react-hook-form';
+import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
+import Loading from '@/components/Loading';
+
+const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
+  loading: () => <Loading fixed={false} />
+});
+const Upload = dynamic(() => import('../commonProgress/Upload'));
+
+const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => {
+  return (
+    <>
+      {activeStep === 0 && <CustomTextInput goToNext={goToNext} />}
+      {activeStep === 1 && <DataProcess showPreviewChunks goToNext={goToNext} />}
+      {activeStep === 2 && <Upload showPreviewChunks />}
+    </>
+  );
+};
+
+export default React.memo(CustomTet);
+
+const CustomTextInput = ({ goToNext }: { goToNext: () => void }) => {
+  const { t } = useTranslation();
+  const { sources, setSources } = useImportStore();
+  const { register, reset, handleSubmit } = useForm({
+    defaultValues: {
+      name: '',
+      value: ''
+    }
+  });
+
+  useEffect(() => {
+    const source = sources[0];
+    if (source) {
+      reset({
+        name: source.sourceName,
+        value: source.rawText
+      });
+    }
+  }, []);
+
+  return (
+    <Box maxW={['100%', '800px']}>
+      <Box display={['block', 'flex']} alignItems={'center'}>
+        <Box flex={'0 0 120px'} fontSize={'sm'}>
+          {t('core.dataset.collection.Collection name')}
+        </Box>
+        <Input
+          flex={'1 0 0'}
+          maxW={['100%', '350px']}
+          {...register('name', {
+            required: true
+          })}
+          placeholder={t('core.dataset.collection.Collection name')}
+          bg={'myGray.50'}
+        />
+      </Box>
+      <Box display={['block', 'flex']} alignItems={'flex-start'} mt={5}>
+        <Box flex={'0 0 120px'} fontSize={'sm'}>
+          {t('core.dataset.collection.Collection raw text')}
+        </Box>
+        <Textarea
+          flex={'1 0 0'}
+          w={'100%'}
+          rows={15}
+          placeholder={t('core.dataset.collection.Collection raw text')}
+          {...register('value', {
+            required: true
+          })}
+          bg={'myGray.50'}
+        />
+      </Box>
+      <Flex mt={5} justifyContent={'flex-end'}>
+        <Button
+          onClick={handleSubmit((data) => {
+            const fileId = getNanoid(32);
+
+            setSources([
+              {
+                id: fileId,
+                rawText: data.value,
+                chunks: [],
+                chunkChars: 0,
+                sourceName: data.name,
+                icon: 'file/fill/manual'
+              }
+            ]);
+            goToNext();
+          })}
+        >
+          {t('common.Next Step')}
+        </Button>
+      </Flex>
+    </Box>
+  );
+};

projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx

@@ -0,0 +1,147 @@
+import React, { useEffect } from 'react';
+import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
+
+import dynamic from 'next/dynamic';
+import { useImportStore } from '../Provider';
+import { useTranslation } from 'next-i18next';
+import { useForm } from 'react-hook-form';
+import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants';
+import { feConfigs } from '@/web/common/system/staticData';
+import { getDocPath } from '@/web/common/system/doc';
+import Loading from '@/components/Loading';
+
+const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
+  loading: () => <Loading fixed={false} />
+});
+const Upload = dynamic(() => import('../commonProgress/Upload'));
+
+const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => {
+  return (
+    <>
+      {activeStep === 0 && <CustomLinkImport goToNext={goToNext} />}
+      {activeStep === 1 && <DataProcess showPreviewChunks={false} goToNext={goToNext} />}
+      {activeStep === 2 && <Upload showPreviewChunks={false} />}
+    </>
+  );
+};
+
+export default React.memo(LinkCollection);
+
+const CustomLinkImport = ({ goToNext }: { goToNext: () => void }) => {
+  const { t } = useTranslation();
+  const { sources, setSources, processParamsForm } = useImportStore();
+  const { register, reset, handleSubmit, watch } = useForm({
+    defaultValues: {
+      link: ''
+    }
+  });
+
+  const link = watch('link');
+  const linkList = link.split('\n').filter((item) => item);
+
+  useEffect(() => {
+    reset({
+      link: sources
+        .map((item) => item.link)
+        .filter((item) => item)
+        .join('\n')
+    });
+  }, []);
+
+  return (
+    <Box maxW={['100%', '800px']}>
+      <Box display={['block', 'flex']} alignItems={'flex-start'} mt={1}>
+        <Box flex={'0 0 100px'} fontSize={'sm'}>
+          {t('core.dataset.import.Link name')}
+        </Box>
+        <Textarea
+          flex={'1 0 0'}
+          w={'100%'}
+          rows={10}
+          placeholder={t('core.dataset.import.Link name placeholder')}
+          bg={'myGray.50'}
+          overflowX={'auto'}
+          whiteSpace={'nowrap'}
+          {...register('link', {
+            required: true
+          })}
+        />
+      </Box>
+      <Box display={['block', 'flex']} alignItems={'center'} mt={4}>
+        <Box flex={'0 0 100px'} fontSize={'sm'}>
+          {t('core.dataset.website.Selector')}
+          <Box color={'myGray.500'} fontSize={'sm'}>
+            {feConfigs?.docUrl && (
+              <Link href={getDocPath('/docs/course/websync/#选择器如何使用')} target="_blank">
+                {t('core.dataset.website.Selector Course')}
+              </Link>
+            )}
+          </Box>
+        </Box>
+        <Input
+          flex={'1 0 0'}
+          maxW={['100%', '350px']}
+          {...processParamsForm.register('webSelector')}
+          placeholder={'body .content #document'}
+          bg={'myGray.50'}
+        />
+      </Box>
+
+      <Flex my={4} flexWrap={'wrap'} gap={4} alignItems={'center'} pl={[0, '100px']}>
+        {linkList.map((item, i) => (
+          <Flex
+            key={`${item}-${i}`}
+            alignItems={'center'}
+            px={4}
+            py={2}
+            borderRadius={'md'}
+            bg={'myGray.100'}
+          >
+            <MyIcon name={LinkCollectionIcon} w={'16px'} />
+            <Box ml={1} mr={3} wordBreak={'break-all'}>
+              {item}
+            </Box>
+            <MyIcon
+              name={'common/closeLight'}
+              w={'14px'}
+              color={'myGray.500'}
+              cursor={'pointer'}
+              onClick={() => {
+                const newLinkList = linkList.filter((link, index) => index !== i);
+                reset({
+                  link: newLinkList.join('\n')
+                });
+              }}
+            />
+          </Flex>
+        ))}
+      </Flex>
+      <Flex mt={5} justifyContent={'flex-end'}>
+        <Button
+          onClick={handleSubmit((data) => {
+            const newLinkList = data.link.split('\n').filter((item) => item);
+
+            setSources(
+              newLinkList.map((link) => ({
+                id: getNanoid(32),
+                link,
+                rawText: '',
+                chunks: [],
+                chunkChars: 0,
+                sourceName: link,
+                icon: LinkCollectionIcon
+              }))
+            );
+
+            goToNext();
+          })}
+        >
+          {t('common.Next Step')}
+        </Button>
+      </Flex>
+    </Box>
+  );
+};

projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx

@@ -0,0 +1,177 @@
+import React, { useEffect, useMemo, useState } from 'react';
+import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
+import { Box, Button, Flex } from '@chakra-ui/react';
+import { ImportSourceItemType } from '@/web/core/dataset/type.d';
+import FileSelector, { type SelectFileItemType } from '@/web/core/dataset/components/FileSelector';
+import { getFileIcon } from '@fastgpt/global/common/file/icon';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { formatFileSize } from '@fastgpt/global/common/file/tools';
+import { useTranslation } from 'next-i18next';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
+import { useRequest } from '@/web/common/hooks/useRequest';
+import { readFileRawContent } from '@fastgpt/web/common/file/read';
+import { getUploadBase64ImgController } from '@/web/common/file/controller';
+import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
+import MyTooltip from '@/components/MyTooltip';
+import type { PreviewRawTextProps } from '../components/PreviewRawText';
+import { useImportStore } from '../Provider';
+import { feConfigs } from '@/web/common/system/staticData';
+
+import dynamic from 'next/dynamic';
+import Loading from '@/components/Loading';
+
+const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
+  loading: () => <Loading fixed={false} />
+});
+const Upload = dynamic(() => import('../commonProgress/Upload'));
+const PreviewRawText = dynamic(() => import('../components/PreviewRawText'));
+
+type FileItemType = ImportSourceItemType & { file: File };
+const fileType = '.txt, .docx, .pdf, .md, .html';
+const maxSelectFileCount = 1000;
+
+const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {
+  return (
+    <>
+      {activeStep === 0 && <SelectFile goToNext={goToNext} />}
+      {activeStep === 1 && <DataProcess showPreviewChunks goToNext={goToNext} />}
+      {activeStep === 2 && <Upload showPreviewChunks />}
+    </>
+  );
+};
+
+export default React.memo(FileLocal);
+
+const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) {
+  const { t } = useTranslation();
+  const { sources, setSources } = useImportStore();
+  // @ts-ignore
+  const [selectFiles, setSelectFiles] = useState<FileItemType[]>(sources);
+  const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
+
+  const [previewRaw, setPreviewRaw] = useState<PreviewRawTextProps>();
+
+  useEffect(() => {
+    setSources(successFiles);
+  }, [successFiles]);
+
+  const { mutate: onSelectFile, isLoading } = useRequest({
+    mutationFn: async (files: SelectFileItemType[]) => {
+      {
+        for await (const selectFile of files) {
+          const { file, folderPath } = selectFile;
+          const relatedId = getNanoid(32);
+
+          const { rawText } = await (() => {
+            try {
+              return readFileRawContent({
+                file,
+                uploadBase64Controller: (base64Img) =>
+                  getUploadBase64ImgController({
+                    base64Img,
+                    type: MongoImageTypeEnum.collectionImage,
+                    metadata: {
+                      relatedId
+                    }
+                  })
+              });
+            } catch (error) {
+              return { rawText: '' };
+            }
+          })();
+
+          const item: FileItemType = {
+            id: relatedId,
+            file,
+            rawText,
+            chunks: [],
+            chunkChars: 0,
+            sourceFolderPath: folderPath,
+            sourceName: file.name,
+            sourceSize: formatFileSize(file.size),
+            icon: getFileIcon(file.name),
+            errorMsg: rawText.length === 0 ? t('common.file.Empty file tip') : ''
+          };
+
+          setSelectFiles((state) => {
+            const results = [item].concat(state).slice(0, maxSelectFileCount);
+            return results;
+          });
+        }
+      }
+    }
+  });
+
+  return (
+    <Box>
+      <FileSelector
+        isLoading={isLoading}
+        fileType={fileType}
+        multiple
+        maxCount={maxSelectFileCount}
+        maxSize={(feConfigs?.uploadFileMaxSize || 500) * 1024 * 1024}
+        onSelectFile={onSelectFile}
+      />
+
+      {/* render files */}
+      <Flex my={4} flexWrap={'wrap'} gap={5} alignItems={'center'}>
+        {selectFiles.map((item) => (
+          <MyTooltip key={item.id} label={t('core.dataset.import.Preview raw text')}>
+            <Flex
+              alignItems={'center'}
+              px={4}
+              py={3}
+              borderRadius={'md'}
+              bg={'myGray.100'}
+              cursor={'pointer'}
+              onClick={() =>
+                setPreviewRaw({
+                  icon: item.icon,
+                  title: item.sourceName,
+                  rawText: item.rawText.slice(0, 10000)
+                })
+              }
+            >
+              <MyIcon name={item.icon as any} w={'16px'} />
+              <Box ml={1} mr={3}>
+                {item.sourceName}
+              </Box>
+              <Box mr={1} fontSize={'xs'} color={'myGray.500'}>
+                {item.sourceSize}
+                {item.rawText.length > 0 && (
+                  <>,{t('common.Number of words', { amount: item.rawText.length })}</>
+                )}
+              </Box>
+              {item.errorMsg && (
+                <MyTooltip label={item.errorMsg}>
+                  <MyIcon name={'common/errorFill'} w={'14px'} mr={3} />
+                </MyTooltip>
+              )}
+              <MyIcon
+                name={'common/closeLight'}
+                w={'14px'}
+                color={'myGray.500'}
+                cursor={'pointer'}
+                onClick={(e) => {
+                  e.stopPropagation();
+                  setSelectFiles((state) => state.filter((file) => file.id !== item.id));
+                }}
+              />
+            </Flex>
+          </MyTooltip>
+        ))}
+      </Flex>
+
+      <Box textAlign={'right'}>
+        <Button isDisabled={successFiles.length === 0 || isLoading} onClick={goToNext}>
+          {selectFiles.length > 0
+            ? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
+            : ''}
+          {t('common.Next Step')}
+        </Button>
+      </Box>
+
+      {previewRaw && <PreviewRawText {...previewRaw} onClose={() => setPreviewRaw(undefined)} />}
+    </Box>
+  );
+});

projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx

@@ -0,0 +1,168 @@
+import React, { useEffect, useMemo, useState } from 'react';
+import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
+import { Box, Button, Flex } from '@chakra-ui/react';
+import { ImportSourceItemType } from '@/web/core/dataset/type.d';
+import FileSelector, { type SelectFileItemType } from '@/web/core/dataset/components/FileSelector';
+import { getFileIcon } from '@fastgpt/global/common/file/icon';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { formatFileSize } from '@fastgpt/global/common/file/tools';
+import { useTranslation } from 'next-i18next';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
+import { useRequest } from '@/web/common/hooks/useRequest';
+import MyTooltip from '@/components/MyTooltip';
+import { useImportStore } from '../Provider';
+import { feConfigs } from '@/web/common/system/staticData';
+
+import dynamic from 'next/dynamic';
+import { fileDownload, readCsvContent } from '@/web/common/file/utils';
+
+const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
+const Upload = dynamic(() => import('../commonProgress/Upload'));
+
+type FileItemType = ImportSourceItemType & { file: File };
+const fileType = '.csv';
+const maxSelectFileCount = 1000;
+
+const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {
+  return (
+    <>
+      {activeStep === 0 && <SelectFile goToNext={goToNext} />}
+      {activeStep === 1 && <PreviewData showPreviewChunks goToNext={goToNext} />}
+      {activeStep === 2 && <Upload showPreviewChunks />}
+    </>
+  );
+};
+
+export default React.memo(FileLocal);
+
+const csvTemplate = `index,content
+"必填内容","可选内容。CSV 中请注意内容不能包含双引号，双引号是列分割符号"
+"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
+"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
+
+const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) {
+  const { t } = useTranslation();
+  const { sources, setSources } = useImportStore();
+  // @ts-ignore
+  const [selectFiles, setSelectFiles] = useState<FileItemType[]>(sources);
+  const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
+
+  useEffect(() => {
+    setSources(successFiles);
+  }, [successFiles]);
+
+  const { mutate: onSelectFile, isLoading } = useRequest({
+    mutationFn: async (files: SelectFileItemType[]) => {
+      {
+        for await (const selectFile of files) {
+          const { file, folderPath } = selectFile;
+          const { header, data } = await readCsvContent(file);
+
+          const filterData: FileItemType['chunks'] = data
+            .filter((item) => item[0])
+            .map((item, i) => ({
+              q: item[0] || '',
+              a: item[1] || '',
+              chunkIndex: i
+            }));
+
+          const item: FileItemType = {
+            id: getNanoid(32),
+            file,
+            rawText: '',
+            chunks: filterData,
+            chunkChars: 0,
+            sourceFolderPath: folderPath,
+            sourceName: file.name,
+            sourceSize: formatFileSize(file.size),
+            icon: getFileIcon(file.name),
+            errorMsg:
+              header[0] !== 'index' || header[1] !== 'content' || filterData.length === 0
+                ? t('core.dataset.import.Csv format error')
+                : ''
+          };
+
+          setSelectFiles((state) => {
+            const results = [item].concat(state).slice(0, 10);
+            return results;
+          });
+        }
+      }
+    },
+    errorToast: t('common.file.Select failed')
+  });
+
+  return (
+    <Box>
+      <FileSelector
+        multiple
+        maxCount={maxSelectFileCount}
+        maxSize={(feConfigs?.uploadFileMaxSize || 500) * 1024 * 1024}
+        isLoading={isLoading}
+        fileType={fileType}
+        onSelectFile={onSelectFile}
+      />
+
+      <Box
+        mt={4}
+        color={'primary.600'}
+        textDecoration={'underline'}
+        cursor={'pointer'}
+        onClick={() =>
+          fileDownload({
+            text: csvTemplate,
+            type: 'text/csv;charset=utf-8',
+            filename: 'template.csv'
+          })
+        }
+      >
+        {t('core.dataset.import.Down load csv template')}
+      </Box>
+
+      {/* render files */}
+      <Flex my={4} flexWrap={'wrap'} gap={5} alignItems={'center'}>
+        {selectFiles.map((item) => (
+          <Flex
+            key={item.id}
+            alignItems={'center'}
+            px={4}
+            py={2}
+            borderRadius={'md'}
+            bg={'myGray.100'}
+          >
+            <MyIcon name={item.icon as any} w={'16px'} />
+            <Box ml={1} mr={3}>
+              {item.sourceName}
+            </Box>
+            <Box mr={1} fontSize={'xs'} color={'myGray.500'}>
+              {item.sourceSize}
+            </Box>
+            {item.errorMsg && (
+              <MyTooltip label={item.errorMsg}>
+                <MyIcon name={'common/errorFill'} w={'14px'} mr={3} />
+              </MyTooltip>
+            )}
+            <MyIcon
+              name={'common/closeLight'}
+              w={'14px'}
+              color={'myGray.500'}
+              cursor={'pointer'}
+              onClick={() => {
+                setSelectFiles((state) => state.filter((file) => file.id !== item.id));
+              }}
+            />
+          </Flex>
+        ))}
+      </Flex>
+
+      <Box textAlign={'right'}>
+        <Button isDisabled={successFiles.length === 0 || isLoading} onClick={goToNext}>
+          {selectFiles.length > 0
+            ? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
+            : ''}
+          {t('common.Next Step')}
+        </Button>
+      </Box>
+    </Box>
+  );
+});

projects/app/src/pages/dataset/detail/components/Import/index.tsx

@@ -0,0 +1,154 @@
+import React, { useMemo } from 'react';
+import { Box, Button, Flex, IconButton } from '@chakra-ui/react';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { useTranslation } from 'next-i18next';
+import { useRouter } from 'next/router';
+import { TabEnum } from '../../index';
+import { useMyStep } from '@fastgpt/web/hooks/useStep';
+import dynamic from 'next/dynamic';
+import Provider from './Provider';
+import { useDatasetStore } from '@/web/core/dataset/store/dataset';
+
+const FileLocal = dynamic(() => import('./diffSource/FileLocal'));
+const FileLink = dynamic(() => import('./diffSource/FileLink'));
+const FileCustomText = dynamic(() => import('./diffSource/FileCustomText'));
+const TableLocal = dynamic(() => import('./diffSource/TableLocal'));
+
+export enum ImportDataSourceEnum {
+  fileLocal = 'fileLocal',
+  fileLink = 'fileLink',
+  fileCustom = 'fileCustom',
+
+  tableLocal = 'tableLocal'
+}
+
+const ImportDataset = () => {
+  const { t } = useTranslation();
+  const router = useRouter();
+  const { datasetDetail } = useDatasetStore();
+  const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as {
+    source: `${ImportDataSourceEnum}`;
+    parentId?: string;
+  };
+
+  const modeSteps: Record<`${ImportDataSourceEnum}`, { title: string }[]> = {
+    [ImportDataSourceEnum.fileLocal]: [
+      {
+        title: t('core.dataset.import.Select file')
+      },
+      {
+        title: t('core.dataset.import.Data Preprocessing')
+      },
+      {
+        title: t('core.dataset.import.Upload data')
+      }
+    ],
+    [ImportDataSourceEnum.fileLink]: [
+      {
+        title: t('core.dataset.import.Select file')
+      },
+      {
+        title: t('core.dataset.import.Data Preprocessing')
+      },
+      {
+        title: t('core.dataset.import.Upload data')
+      }
+    ],
+    [ImportDataSourceEnum.fileCustom]: [
+      {
+        title: t('core.dataset.import.Select file')
+      },
+      {
+        title: t('core.dataset.import.Data Preprocessing')
+      },
+      {
+        title: t('core.dataset.import.Upload data')
+      }
+    ],
+    [ImportDataSourceEnum.tableLocal]: [
+      {
+        title: t('core.dataset.import.Select file')
+      },
+      {
+        title: t('core.dataset.import.Data Preprocessing')
+      },
+      {
+        title: t('core.dataset.import.Upload data')
+      }
+    ]
+  };
+  const steps = modeSteps[source];
+
+  const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({
+    defaultStep: 0,
+    steps
+  });
+
+  const ImportComponent = useMemo(() => {
+    if (source === ImportDataSourceEnum.fileLocal) return FileLocal;
+    if (source === ImportDataSourceEnum.fileLink) return FileLink;
+    if (source === ImportDataSourceEnum.fileCustom) return FileCustomText;
+    if (source === ImportDataSourceEnum.tableLocal) return TableLocal;
+  }, [source]);
+
+  return ImportComponent ? (
+    <Flex flexDirection={'column'} bg={'white'} h={'100%'} px={[2, 9]} py={[2, 5]}>
+      <Flex>
+        {activeStep === 0 ? (
+          <Flex alignItems={'center'}>
+            <IconButton
+              icon={<MyIcon name={'common/backFill'} w={'14px'} />}
+              aria-label={''}
+              size={'smSquare'}
+              w={'26px'}
+              h={'26px'}
+              borderRadius={'50%'}
+              variant={'whiteBase'}
+              mr={2}
+              onClick={() =>
+                router.replace({
+                  query: {
+                    ...router.query,
+                    currentTab: TabEnum.collectionCard
+                  }
+                })
+              }
+            />
+            {t('common.Exit')}
+          </Flex>
+        ) : (
+          <Button
+            variant={'whiteBase'}
+            leftIcon={<MyIcon name={'common/backFill'} w={'14px'} />}
+            onClick={goToPrevious}
+          >
+            {t('common.Last Step')}
+          </Button>
+        )}
+        <Box flex={1} />
+      </Flex>
+      {/* step */}
+      <Box
+        mt={4}
+        mb={5}
+        px={3}
+        py={[2, 4]}
+        bg={'myGray.50'}
+        borderWidth={'1px'}
+        borderColor={'borderColor.low'}
+        borderRadius={'md'}
+      >
+        <Box maxW={['100%', '900px']} mx={'auto'}>
+          <MyStep />
+        </Box>
+      </Box>
+      <Provider dataset={datasetDetail} parentId={parentId}>
+        <Box flex={'1 0 0'} overflow={'auto'} position={'relative'}>
+          <ImportComponent activeStep={activeStep} goToNext={goToNext} />
+        </Box>
+      </Provider>
+    </Flex>
+  ) : null;
+};
+
+export default React.memo(ImportDataset);

projects/app/src/pages/dataset/detail/components/Import/sourceSelector/FileSourceSelector.tsx

@@ -0,0 +1,65 @@
+import React, { useState } from 'react';
+import MyModal from '@/components/MyModal';
+import { ModalBody, ModalFooter, Button } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
+import { ImportDataSourceEnum } from '..';
+import { useRouter } from 'next/router';
+import { TabEnum } from '../../..';
+
+const FileModeSelector = ({ onClose }: { onClose: () => void }) => {
+  const { t } = useTranslation();
+  const router = useRouter();
+  const [value, setValue] = useState<`${ImportDataSourceEnum}`>(ImportDataSourceEnum.fileLocal);
+
+  return (
+    <MyModal
+      isOpen
+      onClose={onClose}
+      iconSrc="modal/selectSource"
+      title={t('core.dataset.import.Select source')}
+      w={'600px'}
+    >
+      <ModalBody px={6} py={4}>
+        <LeftRadio
+          list={[
+            {
+              title: t('core.dataset.import.Local file'),
+              desc: t('core.dataset.import.Local file desc'),
+              value: ImportDataSourceEnum.fileLocal
+            },
+            {
+              title: t('core.dataset.import.Web link'),
+              desc: t('core.dataset.import.Web link desc'),
+              value: ImportDataSourceEnum.fileLink
+            },
+            {
+              title: t('core.dataset.import.Custom text'),
+              desc: t('core.dataset.import.Custom text desc'),
+              value: ImportDataSourceEnum.fileCustom
+            }
+          ]}
+          value={value}
+          onChange={setValue}
+        />
+      </ModalBody>
+      <ModalFooter>
+        <Button
+          onClick={() =>
+            router.replace({
+              query: {
+                ...router.query,
+                currentTab: TabEnum.import,
+                source: value
+              }
+            })
+          }
+        >
+          {t('common.Confirm')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};
+
+export default FileModeSelector;

projects/app/src/pages/dataset/detail/components/Info.tsx

@@ -14,7 +14,7 @@ import MyTooltip from '@/components/MyTooltip';
 import { useTranslation } from 'next-i18next';
 import PermissionRadio from '@/components/support/permission/Radio';
 import MySelect from '@/components/Select';
-import { qaModelList } from '@/web/common/system/staticData';
+import { qaModelList, vectorModelList } from '@/web/common/system/staticData';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
 
@@ -119,39 +119,44 @@ const Info = ({ datasetId }: { datasetId: string }) => {
         </Box>
         <Input flex={[1, '0 0 300px']} maxLength={30} {...register('name')} />
       </Flex>
-      <Flex mt={8} w={'100%'} alignItems={'center'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          {t('core.ai.model.Vector Model')}
-        </Box>
-        <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
-      </Flex>
+      {vectorModelList.length > 1 && (
+        <Flex mt={8} w={'100%'} alignItems={'center'}>
+          <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+            {t('core.ai.model.Vector Model')}
+          </Box>
+          <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
+        </Flex>
+      )}
       <Flex mt={8} w={'100%'} alignItems={'center'}>
         <Box flex={['0 0 90px', '0 0 160px']} w={0}>
           {t('core.Max Token')}
         </Box>
         <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
       </Flex>
-      <Flex mt={6} alignItems={'center'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          {t('core.ai.model.Dataset Agent Model')}
-        </Box>
-        <Box flex={[1, '0 0 300px']}>
-          <MySelect
-            w={'100%'}
-            value={getValues('agentModel').model}
-            list={qaModelList.map((item) => ({
-              label: item.name,
-              value: item.model
-            }))}
-            onchange={(e) => {
-              const agentModel = qaModelList.find((item) => item.model === e);
-              if (!agentModel) return;
-              setValue('agentModel', agentModel);
-              setRefresh((state) => !state);
-            }}
-          />
-        </Box>
-      </Flex>
+      {qaModelList.length > 1 && (
+        <Flex mt={6} alignItems={'center'}>
+          <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+            {t('core.ai.model.Dataset Agent Model')}
+          </Box>
+          <Box flex={[1, '0 0 300px']}>
+            <MySelect
+              w={'100%'}
+              value={getValues('agentModel').model}
+              list={qaModelList.map((item) => ({
+                label: item.name,
+                value: item.model
+              }))}
+              onchange={(e) => {
+                const agentModel = qaModelList.find((item) => item.model === e);
+                if (!agentModel) return;
+                setValue('agentModel', agentModel);
+                setRefresh((state) => !state);
+              }}
+            />
+          </Box>
+        </Flex>
+      )}
+
       <Flex mt={8} alignItems={'center'} w={'100%'}>
         <Box flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</Box>
         <Textarea flex={[1, '0 0 300px']} {...register('intro')} placeholder={t('common.Intro')} />

projects/app/src/pages/dataset/detail/components/InputDataModal.tsx

@@ -1,6 +1,6 @@
 import React, { useMemo, useState } from 'react';
 import { Box, Flex, Button, Textarea, useTheme, Grid } from '@chakra-ui/react';
-import { useFieldArray, useForm } from 'react-hook-form';
+import { UseFormRegister, useFieldArray, useForm } from 'react-hook-form';
 import {
   postInsertData2Dataset,
   putDatasetDataById,
@@ -20,7 +20,7 @@ import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
 import { vectorModelList } from '@/web/common/system/staticData';
-import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constants';
 import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
 import SideTabs from '@/components/SideTabs';
 import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
@@ -29,6 +29,8 @@ import { getDocPath } from '@/web/common/system/doc';
 import RawSourceBox from '@/components/core/dataset/RawSourceBox';
 import MyBox from '@/components/common/MyBox';
 import { getErrText } from '@fastgpt/global/common/error/utils';
+import RowTabs from '@fastgpt/web/components/common/Tabs/RowTabs';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 
 export type InputDataType = {
   q: string;
@@ -124,7 +126,7 @@ const InputDataModal = ({
       onError(err) {
         toast({
           status: 'error',
-          title: getErrText(err)
+          title: t(getErrText(err))
         });
         onClose();
       }
@@ -261,51 +263,7 @@ const InputDataModal = ({
             {currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}</>}
           </Box>
           <Box flex={1} px={5} overflow={'auto'}>
-            {currentTab === TabEnum.content && (
-              <>
-                <Box>
-                  <Flex alignItems={'center'}>
-                    <Box>
-                      <Box as="span" color={'red.600'}>
-                        *
-                      </Box>
-                      {t('core.dataset.data.Data Content')}
-                    </Box>
-                    <MyTooltip label={t('core.dataset.data.Data Content Tip')}>
-                      <QuestionOutlineIcon ml={1} />
-                    </MyTooltip>
-                  </Flex>
-                  <Textarea
-                    mt={1}
-                    placeholder={t('core.dataset.data.Data Content Placeholder', { maxToken })}
-                    maxLength={maxToken}
-                    rows={12}
-                    bg={'myWhite.400'}
-                    {...register(`q`, {
-                      required: true
-                    })}
-                  />
-                </Box>
-                <Box mt={5}>
-                  <Flex>
-                    <Box>{t('core.dataset.data.Auxiliary Data')}</Box>
-                    <MyTooltip label={t('core.dataset.data.Auxiliary Data Tip')}>
-                      <QuestionOutlineIcon ml={1} />
-                    </MyTooltip>
-                  </Flex>
-                  <Textarea
-                    mt={1}
-                    placeholder={t('core.dataset.data.Auxiliary Data Placeholder', {
-                      maxToken: maxToken * 1.5
-                    })}
-                    bg={'myWhite.400'}
-                    rows={12}
-                    maxLength={maxToken * 1.5}
-                    {...register('a')}
-                  />
-                </Box>
-              </>
-            )}
+            {currentTab === TabEnum.content && <InputTab maxToken={maxToken} register={register} />}
             {currentTab === TabEnum.index && (
               <Grid gridTemplateColumns={['1fr', '1fr 1fr']} gridGap={4}>
                 {indexes.map((index, i) => (
@@ -382,6 +340,7 @@ const InputDataModal = ({
               </Grid>
             )}
           </Box>
+          {/* footer */}
           <Flex justifyContent={'flex-end'} px={5} mt={4}>
             <Button variant={'whiteBase'} mr={3} onClick={onClose}>
               {t('common.Close')}
@@ -404,3 +363,80 @@ const InputDataModal = ({
 };
 
 export default React.memo(InputDataModal);
+
+enum InputTypeEnum {
+  q = 'q',
+  a = 'a'
+}
+const InputTab = ({
+  maxToken,
+  register
+}: {
+  maxToken: number;
+  register: UseFormRegister<InputDataType>;
+}) => {
+  const { t } = useTranslation();
+  const { isPc } = useSystemStore();
+  const [inputType, setInputType] = useState(InputTypeEnum.q);
+
+  return (
+    <Box>
+      <RowTabs
+        list={[
+          {
+            label: (
+              <Flex alignItems={'center'}>
+                <Box as="span" color={'red.600'}>
+                  *
+                </Box>
+                {t('core.dataset.data.Main Content')}
+                <MyTooltip label={t('core.dataset.data.Data Content Tip')}>
+                  <QuestionOutlineIcon ml={1} />
+                </MyTooltip>
+              </Flex>
+            ),
+            value: InputTypeEnum.q
+          },
+          {
+            label: (
+              <Flex alignItems={'center'}>
+                {t('core.dataset.data.Auxiliary Data')}
+                <MyTooltip label={t('core.dataset.data.Auxiliary Data Tip')}>
+                  <QuestionOutlineIcon ml={1} />
+                </MyTooltip>
+              </Flex>
+            ),
+            value: InputTypeEnum.a
+          }
+        ]}
+        value={inputType}
+        onChange={(e) => setInputType(e as InputTypeEnum)}
+      />
+
+      <Box mt={3}>
+        {inputType === InputTypeEnum.q && (
+          <Textarea
+            placeholder={t('core.dataset.data.Data Content Placeholder', { maxToken })}
+            maxLength={maxToken}
+            rows={isPc ? 24 : 12}
+            bg={'myWhite.400'}
+            {...register(`q`, {
+              required: true
+            })}
+          />
+        )}
+        {inputType === InputTypeEnum.a && (
+          <Textarea
+            placeholder={t('core.dataset.data.Auxiliary Data Placeholder', {
+              maxToken: maxToken * 1.5
+            })}
+            bg={'myWhite.400'}
+            rows={isPc ? 24 : 12}
+            maxLength={maxToken * 1.5}
+            {...register('a')}
+          />
+        )}
+      </Box>
+    </Box>
+  );
+};

projects/app/src/pages/dataset/detail/components/Test.tsx

@@ -5,7 +5,6 @@ import {
   Button,
   Flex,
   useTheme,
-  Grid,
   useDisclosure,
   Table,
   Thead,
@@ -28,7 +27,10 @@ import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useTranslation } from 'next-i18next';
 import { SearchTestResponse } from '@/global/core/dataset/api';
-import { DatasetSearchModeEnum, DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetSearchModeEnum,
+  DatasetSearchModeMap
+} from '@fastgpt/global/core/dataset/constants';
 import dynamic from 'next/dynamic';
 import { useForm } from 'react-hook-form';
 import MySelect from '@/components/Select';
@@ -97,6 +99,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
           title: t('dataset.test.noResult')
         });
       }
+
       const testItem: SearchTestStoreItemType = {
         id: nanoid(),
         datasetId,
@@ -389,7 +392,7 @@ const TestHistories = React.memo(function TestHistories({
             })}
             onClick={() => setDatasetTestItem(item)}
           >
-            <Box flex={'0 0 80px'}>
+            <Box flex={'0 0 auto'} mr={2}>
               {DatasetSearchModeMap[item.searchMode] ? (
                 <Flex alignItems={'center'} fontWeight={'500'} color={'myGray.500'}>
                   <MyIcon
@@ -406,7 +409,11 @@ const TestHistories = React.memo(function TestHistories({
             <Box flex={1} mr={2} wordBreak={'break-all'} fontWeight={'400'}>
               {item.text}
             </Box>
-            <Box flex={'0 0 70px'}>{formatTimeToChatTime(item.time)}</Box>
+            <Box flex={'0 0 70px'}>
+              {formatTimeToChatTime(item.time).includes('.')
+                ? t(formatTimeToChatTime(item.time))
+                : formatTimeToChatTime(item.time)}
+            </Box>
             <MyTooltip label={t('core.dataset.test.delete test history')}>
               <Box w={'14px'} h={'14px'}>
                 <MyIcon