4.6.3-alpha1 (#529)

projects/app/src/pages/dataset/detail/components/CollectionCard.tsx

@@ -36,7 +36,11 @@ import MyMenu from '@/components/MyMenu';
 import { useEditTitle } from '@/web/common/hooks/useEditTitle';
 import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
 import EmptyTip from '@/components/EmptyTip';
-import { FolderAvatarSrc, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  FolderAvatarSrc,
+  DatasetCollectionTypeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
 import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
 import { TabEnum } from '..';
@@ -347,7 +351,7 @@ const CollectionCard = () => {
               <Th>#</Th>
               <Th>{t('common.Name')}</Th>
               <Th>{t('dataset.collections.Data Amount')}</Th>
-              <Th>{t('common.Time')}</Th>
+              <Th>{t('core.dataset.Sync Time')}</Th>
               <Th>{t('common.Status')}</Th>
               <Th />
             </Tr>

projects/app/src/pages/dataset/detail/components/DataCard.tsx

@@ -1,5 +1,20 @@
 import React, { useCallback, useState, useRef, useMemo } from 'react';
-import { Box, Card, IconButton, Flex, Grid, Button } from '@chakra-ui/react';
+import {
+  Box,
+  Card,
+  IconButton,
+  Flex,
+  Grid,
+  Button,
+  useTheme,
+  Drawer,
+  DrawerBody,
+  DrawerFooter,
+  DrawerHeader,
+  DrawerOverlay,
+  DrawerContent,
+  useDisclosure
+} from '@chakra-ui/react';
 import { usePagination } from '@/web/common/hooks/usePagination';
 import {
   getDatasetDataList,
@@ -23,12 +38,23 @@ import { TabEnum } from '..';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
 import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import {
+  DatasetCollectionTypeMap,
+  DatasetCollectionTrainingTypeMap
+} from '@fastgpt/global/core/dataset/constant';
+import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
+import { formatFileSize } from '@fastgpt/global/common/file/tools';
+import { getFileAndOpen } from '@/web/core/dataset/utils';
+import MyTooltip from '@/components/MyTooltip';
 
 const DataCard = () => {
   const BoxRef = useRef<HTMLDivElement>(null);
+  const theme = useTheme();
   const lastSearch = useRef('');
   const router = useRouter();
   const { userInfo } = useUserStore();
+  const { isPc } = useSystemStore();
   const { collectionId = '' } = router.query as { collectionId: string };
   const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
   const { t } = useTranslation();
@@ -37,6 +63,7 @@ const DataCard = () => {
   const { openConfirm, ConfirmModal } = useConfirm({
     content: t('dataset.Confirm to delete the data')
   });
+  const { isOpen, onOpen, onClose } = useDisclosure();
 
   const {
     data: datasetDataList,
@@ -81,6 +108,43 @@ const DataCard = () => {
     [collection?.canWrite, userInfo?.team?.role]
   );
 
+  const metadataList = useMemo(
+    () =>
+      collection
+        ? [
+            {
+              label: t('core.dataset.collection.metadata.source'),
+              value: t(DatasetCollectionTypeMap[collection.type]?.name)
+            },
+            {
+              label: t('core.dataset.collection.metadata.source name'),
+              value: collection.file?.filename || collection?.rawLink || collection?.name
+            },
+            {
+              label: t('core.dataset.collection.metadata.source size'),
+              value: collection.file ? formatFileSize(collection.file.length) : '-'
+            },
+            {
+              label: t('core.dataset.collection.metadata.Createtime'),
+              value: formatTime2YMDHM(collection.createTime)
+            },
+            {
+              label: t('core.dataset.collection.metadata.Updatetime'),
+              value: formatTime2YMDHM(collection.updateTime)
+            },
+            {
+              label: t('core.dataset.collection.metadata.Training Type'),
+              value: t(DatasetCollectionTrainingTypeMap[collection.trainingType]?.label)
+            },
+            {
+              label: t('core.dataset.collection.metadata.Chunk Size'),
+              value: collection.chunkSize || '-'
+            }
+          ]
+        : [],
+    [collection, t]
+  );
+
   return (
     <Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
       <Flex alignItems={'center'}>
@@ -106,7 +170,7 @@ const DataCard = () => {
           <Box lineHeight={1.2}>
             <RawSourceText
               sourceName={collection?.name}
-              sourceId={collection?.metadata?.fileId || collection?.metadata?.rawLink}
+              sourceId={collection?.fileId || collection?.rawLink}
               fontSize={['md', 'lg']}
               color={'black'}
               textDecoration={'none'}
@@ -122,7 +186,7 @@ const DataCard = () => {
         {canWrite && (
           <Box>
             <Button
-              ml={2}
+              mx={2}
               variant={'base'}
               size={['sm', 'md']}
               onClick={() => {
@@ -137,6 +201,17 @@ const DataCard = () => {
             </Button>
           </Box>
         )}
+        {isPc && (
+          <MyTooltip label={t('core.dataset.collection.metadata.Read Metadata')}>
+            <IconButton
+              variant={'base'}
+              size={['sm', 'md']}
+              icon={<MyIcon name={'menu'} w={'18px'} />}
+              aria-label={''}
+              onClick={onOpen}
+            />
+          </MyTooltip>
+        )}
       </Flex>
       <Flex my={3} alignItems={'center'}>
         <Box>
@@ -178,16 +253,23 @@ const DataCard = () => {
         gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
         gridGap={4}
       >
-        {datasetDataList.map((item) => (
+        {datasetDataList.map((item, index) => (
           <Card
             key={item._id}
             cursor={'pointer'}
-            pt={3}
+            p={3}
             userSelect={'none'}
             boxShadow={'none'}
-            _hover={{ boxShadow: 'lg', '& .delete': { display: 'flex' } }}
-            border={'1px solid '}
-            borderColor={'myGray.200'}
+            bg={'myWhite.500'}
+            border={theme.borders.sm}
+            position={'relative'}
+            overflow={'hidden'}
+            _hover={{
+              borderColor: 'myGray.200',
+              boxShadow: 'lg',
+              bg: 'white',
+              '& .footer': { h: 'auto', p: 3 }
+            }}
             onClick={() => {
               if (!collection) return;
               setEditInputData({
@@ -198,57 +280,113 @@ const DataCard = () => {
               });
             }}
           >
-            <Box
-              h={'95px'}
-              overflow={'hidden'}
-              wordBreak={'break-all'}
-              px={3}
-              py={1}
-              fontSize={'13px'}
-            >
-              <Box color={'myGray.1000'} mb={2}>
-                {item.q}
+            <Flex zIndex={1} alignItems={'center'} justifyContent={'space-between'}>
+              <Box border={theme.borders.base} px={2} fontSize={'sm'} mr={1} borderRadius={'md'}>
+                # {index + 1}
               </Box>
-              <Box color={'myGray.600'}>{item.a}</Box>
-            </Box>
-            <Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
-              <Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
+              <Box className={'textEllipsis'} color={'myGray.500'} fontSize={'xs'}>
                 ID:{item._id}
               </Box>
-              {canWrite && (
-                <IconButton
-                  className="delete"
-                  display={['flex', 'none']}
-                  icon={<DeleteIcon />}
-                  variant={'base'}
-                  colorScheme={'gray'}
-                  aria-label={'delete'}
-                  size={'xs'}
-                  borderRadius={'md'}
-                  _hover={{ color: 'red.600' }}
-                  onClick={(e) => {
-                    e.stopPropagation();
-                    openConfirm(async () => {
-                      try {
-                        setIsLoading(true);
-                        await delOneDatasetDataById(item._id);
-                        getData(pageNum);
-                      } catch (error) {
-                        toast({
-                          title: getErrText(error),
-                          status: 'error'
-                        });
-                      }
-                      setIsLoading(false);
-                    })();
-                  }}
-                />
-              )}
             </Flex>
+            <Box
+              maxH={'135px'}
+              overflow={'hidden'}
+              wordBreak={'break-all'}
+              pt={1}
+              pb={3}
+              fontSize={'13px'}
+            >
+              <Box color={'black'} mb={1}>
+                {item.q}
+              </Box>
+              <Box color={'myGray.700'}>{item.a}</Box>
+
+              <Flex
+                className="footer"
+                position={'absolute'}
+                top={0}
+                bottom={0}
+                left={0}
+                right={0}
+                h={'0'}
+                overflow={'hidden'}
+                p={0}
+                bg={'linear-gradient(to top, white,white 20%, rgba(255,255,255,0) 60%)'}
+                alignItems={'flex-end'}
+                fontSize={'sm'}
+              >
+                <Flex alignItems={'center'}>
+                  <MyIcon name="common/text/t" w={'14px'} mr={1} color={'myGray.500'} />
+                  {item.q.length + (item.a?.length || 0)}
+                </Flex>
+                <Box flex={1} />
+                {canWrite && (
+                  <IconButton
+                    display={'flex'}
+                    icon={<DeleteIcon />}
+                    variant={'base'}
+                    colorScheme={'gray'}
+                    aria-label={'delete'}
+                    size={'xs'}
+                    borderRadius={'md'}
+                    _hover={{ color: 'red.600' }}
+                    onClick={(e) => {
+                      e.stopPropagation();
+                      openConfirm(async () => {
+                        try {
+                          setIsLoading(true);
+                          await delOneDatasetDataById(item._id);
+                          getData(pageNum);
+                        } catch (error) {
+                          toast({
+                            title: getErrText(error),
+                            status: 'error'
+                          });
+                        }
+                        setIsLoading(false);
+                      })();
+                    }}
+                  />
+                )}
+              </Flex>
+            </Box>
           </Card>
         ))}
       </Grid>
 
+      {/* metadata drawer */}
+      <Drawer isOpen={isOpen} placement="right" size={'md'} onClose={onClose}>
+        <DrawerOverlay />
+        <DrawerContent>
+          <DrawerHeader>{t('core.dataset.collection.metadata.metadata')}</DrawerHeader>
+
+          <DrawerBody>
+            {metadataList.map((item) => (
+              <Flex key={item.label} alignItems={'center'} mb={5}>
+                <Box color={'myGray.500'} w={'100px'}>
+                  {item.label}
+                </Box>
+                <Box>{item.value}</Box>
+              </Flex>
+            ))}
+            {collection?.sourceId && (
+              <Button
+                variant={'base'}
+                onClick={() => collection.sourceId && getFileAndOpen(collection.sourceId)}
+              >
+                {t('core.dataset.collection.metadata.read source')}
+              </Button>
+            )}
+          </DrawerBody>
+
+          <DrawerFooter>
+            <Button variant={'base'} onClick={onClose}>
+              {t('common.Close')}
+            </Button>
+          </DrawerFooter>
+        </DrawerContent>
+      </Drawer>
+
       {total > pageSize && (
         <Flex mt={2} justifyContent={'center'}>
           <Pagination />

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -19,7 +19,6 @@ import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
 import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
-import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
@@ -40,7 +39,8 @@ export type FileItemType = {
   icon: string;
   tokens: number; // total tokens
   type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
-  metadata: DatasetCollectionSchemaType['metadata'];
+  fileId?: string;
+  rawLink?: string;
 };
 
 export interface Props extends BoxProps {
@@ -157,9 +157,7 @@ const FileSelect = ({
                 .join('\n')}`,
               chunks: filterData,
               type: DatasetCollectionTypeEnum.file,
-              metadata: {
-                fileId
-              }
+              fileId
             };
 
             onPushFiles([fileItem]);
@@ -195,9 +193,7 @@ const FileSelect = ({
               text,
               tokens: splitRes.tokens,
               type: DatasetCollectionTypeEnum.file,
-              metadata: {
-                fileId
-              },
+              fileId,
               chunks: splitRes.chunks.map((chunk) => ({
                 q: chunk,
                 a: ''
@@ -220,7 +216,7 @@ const FileSelect = ({
   // link fetch
   const onUrlFetch = useCallback(
     (e: FetchResultItem[]) => {
-      const result: FileItemType[] = e.map(({ url, content }) => {
+      const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
         const splitRes = splitText2Chunks({
           text: content,
           chunkLen,
@@ -233,9 +229,7 @@ const FileSelect = ({
           text: content,
           tokens: splitRes.tokens,
           type: DatasetCollectionTypeEnum.link,
-          metadata: {
-            rawLink: url
-          },
+          rawLink: url,
           chunks: splitRes.chunks.map((chunk) => ({
             q: chunk,
             a: ''
@@ -277,9 +271,7 @@ const FileSelect = ({
           text: content,
           tokens: splitRes.tokens,
           type: DatasetCollectionTypeEnum.file,
-          metadata: {
-            fileId: fileIds[0]
-          },
+          fileId: fileIds[0],
           chunks: splitRes.chunks.map((chunk) => ({
             q: chunk,
             a: ''

projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx

@@ -11,7 +11,10 @@ import MyModal from '@/components/MyModal';
 import Provider from './Provider';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { qaModelList } from '@/web/common/system/staticData';
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionTrainingModeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constant';
 
 export enum ImportTypeEnum {
   chunk = 'chunk',
@@ -43,19 +46,22 @@ const ImportData = ({
         defaultChunkLen: vectorModel?.defaultToken || 500,
         chunkOverlapRatio: 0.2,
         unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.chunk
+        mode: TrainingModeEnum.chunk,
+        collectionTrainingType: DatasetCollectionTrainingModeEnum.chunk
       },
       [ImportTypeEnum.qa]: {
         defaultChunkLen: agentModel?.maxContext * 0.6 || 8000,
         chunkOverlapRatio: 0,
         unitPrice: agentModel?.price || 3,
-        mode: TrainingModeEnum.qa
+        mode: TrainingModeEnum.qa,
+        collectionTrainingType: DatasetCollectionTrainingModeEnum.qa
       },
       [ImportTypeEnum.csv]: {
-        defaultChunkLen: vectorModel?.defaultToken || 500,
+        defaultChunkLen: 0,
         chunkOverlapRatio: 0,
         unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.chunk
+        mode: TrainingModeEnum.chunk,
+        collectionTrainingType: DatasetCollectionTrainingModeEnum.manual
       }
     };
     return map[importType];

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -15,7 +15,10 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { useToast } from '@/web/common/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionTrainingModeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import { Box, Flex, Image, useTheme } from '@chakra-ui/react';
 import { CloseIcon } from '@chakra-ui/icons';
 import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
@@ -92,6 +95,7 @@ const Provider = ({
   parentId,
   unitPrice,
   mode,
+  collectionTrainingType,
   vectorModel,
   agentModel,
   defaultChunkLen = 500,
@@ -104,6 +108,7 @@ const Provider = ({
   parentId: string;
   unitPrice: number;
   mode: `${TrainingModeEnum}`;
+  collectionTrainingType: `${DatasetCollectionTrainingModeEnum}`;
   vectorModel: string;
   agentModel: string;
   defaultChunkLen: number;
@@ -150,7 +155,10 @@ const Provider = ({
           parentId,
           name: file.filename,
           type: file.type,
-          metadata: file.metadata
+          fileId: file.fileId,
+          rawLink: file.rawLink,
+          chunkSize: chunkLen,
+          trainingType: collectionTrainingType
         });
 
         // upload data

projects/app/src/pages/dataset/detail/components/InputDataModal.tsx

@@ -427,7 +427,11 @@ export function RawSourceText({
         {...props}
       >
         <Image src={icon} alt="" w={['14px', '16px']} mr={2} />
-        <Box maxW={['200px', '300px']} className={props.className ?? 'textEllipsis'}>
+        <Box
+          maxW={['200px', '300px']}
+          className={props.className ?? 'textEllipsis'}
+          wordBreak={'break-all'}
+        >
           {sourceName || t('common.UnKnow Source')}
         </Box>
       </Box>