mirror of
https://github.com/labring/FastGPT.git
synced 2025-08-05 22:55:27 +00:00
v4.5.1 (#417)
This commit is contained in:
@@ -0,0 +1,601 @@
|
||||
import React, { useCallback, useState, useRef, useMemo, useEffect } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
TableContainer,
|
||||
Table,
|
||||
Thead,
|
||||
Tr,
|
||||
Th,
|
||||
Td,
|
||||
Tbody,
|
||||
Image,
|
||||
MenuButton,
|
||||
useTheme,
|
||||
useDisclosure,
|
||||
ModalFooter,
|
||||
Button
|
||||
} from '@chakra-ui/react';
|
||||
import {
|
||||
getDatasetCollections,
|
||||
delDatasetCollectionById,
|
||||
putDatasetCollectionById,
|
||||
postDatasetCollection,
|
||||
getDatasetCollectionPathById
|
||||
} from '@/web/core/dataset/api';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { debounce } from 'lodash';
|
||||
import { useConfirm } from '@/web/common/hooks/useConfirm';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import MyInput from '@/components/MyInput';
|
||||
import dayjs from 'dayjs';
|
||||
import { useRequest } from '@/web/common/hooks/useRequest';
|
||||
import { useLoading } from '@/web/common/hooks/useLoading';
|
||||
import { useRouter } from 'next/router';
|
||||
import { usePagination } from '@/web/common/hooks/usePagination';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import MyMenu from '@/components/MyMenu';
|
||||
import { useEditTitle } from '@/web/common/hooks/useEditTitle';
|
||||
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
|
||||
import EmptyTip from '@/components/EmptyTip';
|
||||
import { AddIcon } from '@chakra-ui/icons';
|
||||
import { FolderAvatarSrc, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
|
||||
|
||||
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
|
||||
import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
|
||||
import { TabEnum } from '..';
|
||||
import ParentPath from '@/components/common/ParentPaths';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useDrag } from '@/web/common/hooks/useDrag';
|
||||
import SelectCollections from '@/web/core/dataset/components/SelectCollections';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
|
||||
const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
|
||||
|
||||
const CollectionCard = () => {
|
||||
const BoxRef = useRef<HTMLDivElement>(null);
|
||||
const theme = useTheme();
|
||||
const lastSearch = useRef('');
|
||||
const router = useRouter();
|
||||
const { toast } = useToast();
|
||||
const { parentId = '', datasetId } = router.query as { parentId: string; datasetId: string };
|
||||
const { t } = useTranslation();
|
||||
const { Loading } = useLoading();
|
||||
const { isPc } = useSystemStore();
|
||||
const [searchText, setSearchText] = useState('');
|
||||
const { setLoading } = useSystemStore();
|
||||
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
|
||||
|
||||
const { openConfirm, ConfirmModal } = useConfirm({
|
||||
content: t('dataset.Confirm to delete the file')
|
||||
});
|
||||
const {
|
||||
isOpen: isOpenFileImportModal,
|
||||
onOpen: onOpenFileImportModal,
|
||||
onClose: onCloseFileImportModal
|
||||
} = useDisclosure();
|
||||
const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
|
||||
useEditTitle({
|
||||
title: t('dataset.Create Virtual File'),
|
||||
tip: t('dataset.Virtual File Tip')
|
||||
});
|
||||
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
|
||||
title: t('Rename')
|
||||
});
|
||||
|
||||
const { editFolderData, setEditFolderData } = useEditFolder();
|
||||
const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
|
||||
|
||||
const {
|
||||
data: collections,
|
||||
Pagination,
|
||||
total,
|
||||
getData,
|
||||
isLoading,
|
||||
pageNum,
|
||||
pageSize
|
||||
} = usePagination<DatasetCollectionsListItemType>({
|
||||
api: getDatasetCollections,
|
||||
pageSize: 20,
|
||||
params: {
|
||||
datasetId,
|
||||
parentId,
|
||||
searchText
|
||||
},
|
||||
defaultRequest: false,
|
||||
onChange() {
|
||||
if (BoxRef.current) {
|
||||
BoxRef.current.scrollTop = 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const { moveDataId, setMoveDataId, dragStartId, setDragStartId, dragTargetId, setDragTargetId } =
|
||||
useDrag();
|
||||
|
||||
// change search
|
||||
const debounceRefetch = useCallback(
|
||||
debounce(() => {
|
||||
getData(1);
|
||||
lastSearch.current = searchText;
|
||||
}, 300),
|
||||
[]
|
||||
);
|
||||
|
||||
// add file icon
|
||||
const formatCollections = useMemo(
|
||||
() =>
|
||||
collections.map((collection) => {
|
||||
const icon = getCollectionIcon(collection.type, collection.name);
|
||||
|
||||
return {
|
||||
...collection,
|
||||
icon,
|
||||
...(collection.trainingAmount > 0
|
||||
? {
|
||||
statusText: t('dataset.collections.Collection Embedding', {
|
||||
total: collection.trainingAmount
|
||||
}),
|
||||
color: 'myGray.500'
|
||||
}
|
||||
: {
|
||||
statusText: t('dataset.collections.Ready'),
|
||||
color: 'green.500'
|
||||
})
|
||||
};
|
||||
}),
|
||||
[collections, t]
|
||||
);
|
||||
const hasTrainingData = useMemo(
|
||||
() => !!formatCollections.find((item) => item.trainingAmount > 0),
|
||||
[formatCollections]
|
||||
);
|
||||
|
||||
const { mutate: onCreateVirtualFile } = useRequest({
|
||||
mutationFn: ({ name }: { name: string }) => {
|
||||
setLoading(true);
|
||||
return postDatasetCollection({
|
||||
parentId,
|
||||
datasetId,
|
||||
name,
|
||||
type: DatasetCollectionTypeEnum.virtual
|
||||
});
|
||||
},
|
||||
onSuccess() {
|
||||
getData(pageNum);
|
||||
},
|
||||
onSettled() {
|
||||
setLoading(false);
|
||||
},
|
||||
successToast: t('dataset.collections.Create Virtual File Success'),
|
||||
errorToast: t('common.Create Virtual File Failed')
|
||||
});
|
||||
const { mutate: onUpdateCollectionName } = useRequest({
|
||||
mutationFn: ({ collectionId, name }: { collectionId: string; name: string }) => {
|
||||
return putDatasetCollectionById({
|
||||
id: collectionId,
|
||||
name
|
||||
});
|
||||
},
|
||||
onSuccess() {
|
||||
getData(pageNum);
|
||||
},
|
||||
|
||||
successToast: t('common.Rename Success'),
|
||||
errorToast: t('common.Rename Failed')
|
||||
});
|
||||
const { mutate: onDelCollection } = useRequest({
|
||||
mutationFn: (collectionId: string) => {
|
||||
setLoading(true);
|
||||
return delDatasetCollectionById({
|
||||
collectionId
|
||||
});
|
||||
},
|
||||
onSuccess() {
|
||||
getData(pageNum);
|
||||
},
|
||||
onSettled() {
|
||||
setLoading(false);
|
||||
},
|
||||
successToast: t('common.Delete Success'),
|
||||
errorToast: t('common.Delete Failed')
|
||||
});
|
||||
|
||||
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
|
||||
getDatasetCollectionPathById(parentId)
|
||||
);
|
||||
|
||||
useQuery(['loadDatasetDetail', datasetId], () => loadDatasetDetail(datasetId));
|
||||
useQuery(
|
||||
['refreshCollection'],
|
||||
() => {
|
||||
getData(1);
|
||||
return null;
|
||||
},
|
||||
{
|
||||
refetchInterval: 6000,
|
||||
enabled: hasTrainingData
|
||||
}
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
getData(1);
|
||||
}, [parentId]);
|
||||
|
||||
return (
|
||||
<Box ref={BoxRef} py={[1, 3]} h={'100%'} overflow={'overlay'}>
|
||||
<Flex px={[2, 5]} alignItems={['flex-start', 'center']}>
|
||||
<Box flex={1}>
|
||||
<ParentPath
|
||||
paths={paths.map((path, i) => ({
|
||||
parentId: path.parentId,
|
||||
parentName: i === paths.length - 1 ? `${path.parentName}(${total})` : path.parentName
|
||||
}))}
|
||||
FirstPathDom={
|
||||
<Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
|
||||
{t('common.File')}({total})
|
||||
</Box>
|
||||
}
|
||||
onClick={(e) => {
|
||||
router.replace({
|
||||
query: {
|
||||
...router.query,
|
||||
parentId: e
|
||||
}
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
{isPc && (
|
||||
<Flex alignItems={'center'} mr={2}>
|
||||
<MyInput
|
||||
leftIcon={
|
||||
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
|
||||
}
|
||||
w={['100%', '250px']}
|
||||
size={['sm', 'md']}
|
||||
placeholder={t('common.Search') || ''}
|
||||
value={searchText}
|
||||
onChange={(e) => {
|
||||
setSearchText(e.target.value);
|
||||
debounceRefetch();
|
||||
}}
|
||||
onBlur={() => {
|
||||
if (searchText === lastSearch.current) return;
|
||||
getData(1);
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
if (searchText === lastSearch.current) return;
|
||||
if (e.key === 'Enter') {
|
||||
getData(1);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
)}
|
||||
<MyMenu
|
||||
offset={[-10, 10]}
|
||||
width={120}
|
||||
Button={
|
||||
<MenuButton
|
||||
_hover={{
|
||||
color: 'myBlue.600'
|
||||
}}
|
||||
fontSize={['sm', 'md']}
|
||||
>
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
border={theme.borders.base}
|
||||
px={5}
|
||||
py={2}
|
||||
borderRadius={'md'}
|
||||
cursor={'pointer'}
|
||||
h={['28px', '35px']}
|
||||
>
|
||||
<AddIcon mr={2} />
|
||||
<Box>{t('Create New')}</Box>
|
||||
</Flex>
|
||||
</MenuButton>
|
||||
}
|
||||
menuList={[
|
||||
{
|
||||
child: (
|
||||
<Flex>
|
||||
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
|
||||
{t('Folder')}
|
||||
</Flex>
|
||||
),
|
||||
onClick: () => setEditFolderData({})
|
||||
},
|
||||
{
|
||||
child: (
|
||||
<Flex>
|
||||
<Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
|
||||
{t('dataset.Create Virtual File')}
|
||||
</Flex>
|
||||
),
|
||||
onClick: () => {
|
||||
onOpenCreateVirtualFileModal({
|
||||
defaultVal: '',
|
||||
onSuccess: (name) => onCreateVirtualFile({ name })
|
||||
});
|
||||
}
|
||||
},
|
||||
{
|
||||
child: (
|
||||
<Flex>
|
||||
<Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
|
||||
{t('dataset.File Input')}
|
||||
</Flex>
|
||||
),
|
||||
onClick: onOpenFileImportModal
|
||||
}
|
||||
]}
|
||||
/>
|
||||
</Flex>
|
||||
<TableContainer mt={[0, 3]} position={'relative'} minH={'50vh'}>
|
||||
<Table variant={'simple'} fontSize={'sm'} draggable={false}>
|
||||
<Thead draggable={false}>
|
||||
<Tr>
|
||||
<Th>{t('common.Name')}</Th>
|
||||
<Th>{t('dataset.collections.Data Amount')}</Th>
|
||||
<Th>{t('common.Time')}</Th>
|
||||
<Th>{t('common.Status')}</Th>
|
||||
<Th />
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{formatCollections.map((collection) => (
|
||||
<Tr
|
||||
key={collection._id}
|
||||
_hover={{ bg: 'myWhite.600' }}
|
||||
cursor={'pointer'}
|
||||
data-drag-id={
|
||||
collection.type === DatasetCollectionTypeEnum.folder ? collection._id : undefined
|
||||
}
|
||||
bg={dragTargetId === collection._id ? 'myBlue.200' : ''}
|
||||
userSelect={'none'}
|
||||
onDragStart={(e) => {
|
||||
setDragStartId(collection._id);
|
||||
}}
|
||||
onDragOver={(e) => {
|
||||
e.preventDefault();
|
||||
const targetId = e.currentTarget.getAttribute('data-drag-id');
|
||||
if (!targetId) return;
|
||||
DatasetCollectionTypeEnum.folder && setDragTargetId(targetId);
|
||||
}}
|
||||
onDragLeave={(e) => {
|
||||
e.preventDefault();
|
||||
setDragTargetId(undefined);
|
||||
}}
|
||||
onDrop={async (e) => {
|
||||
e.preventDefault();
|
||||
if (!dragTargetId || !dragStartId || dragTargetId === dragStartId) return;
|
||||
// update parentId
|
||||
try {
|
||||
await putDatasetCollectionById({
|
||||
id: dragStartId,
|
||||
parentId: dragTargetId
|
||||
});
|
||||
getData(pageNum);
|
||||
} catch (error) {}
|
||||
setDragTargetId(undefined);
|
||||
}}
|
||||
title={
|
||||
collection.type === DatasetCollectionTypeEnum.folder
|
||||
? t('dataset.collections.Click to view folder')
|
||||
: t('dataset.collections.Click to view file')
|
||||
}
|
||||
onClick={() => {
|
||||
if (collection.type === DatasetCollectionTypeEnum.folder) {
|
||||
router.replace({
|
||||
query: {
|
||||
...router.query,
|
||||
parentId: collection._id
|
||||
}
|
||||
});
|
||||
} else {
|
||||
router.replace({
|
||||
query: {
|
||||
...router.query,
|
||||
collectionId: collection._id,
|
||||
currentTab: TabEnum.dataCard
|
||||
}
|
||||
});
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Td maxW={['200px', '300px']} draggable>
|
||||
<Flex alignItems={'center'}>
|
||||
<Image src={collection.icon} w={'16px'} mr={2} alt={''} />
|
||||
<MyTooltip label={t('common.folder.Drag Tip')} shouldWrapChildren={false}>
|
||||
<Box fontWeight={'bold'} className="textEllipsis">
|
||||
{collection.name}
|
||||
</Box>
|
||||
</MyTooltip>
|
||||
</Flex>
|
||||
</Td>
|
||||
<Td fontSize={'md'}>{collection.dataAmount ?? '-'}</Td>
|
||||
<Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
|
||||
<Td>
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
_before={{
|
||||
content: '""',
|
||||
w: '10px',
|
||||
h: '10px',
|
||||
mr: 2,
|
||||
borderRadius: 'lg',
|
||||
bg: collection?.color
|
||||
}}
|
||||
>
|
||||
{collection?.statusText}
|
||||
</Flex>
|
||||
</Td>
|
||||
<Td onClick={(e) => e.stopPropagation()}>
|
||||
<MyMenu
|
||||
width={100}
|
||||
Button={
|
||||
<MenuButton
|
||||
w={'22px'}
|
||||
h={'22px'}
|
||||
borderRadius={'md'}
|
||||
_hover={{
|
||||
color: 'myBlue.600',
|
||||
'& .icon': {
|
||||
bg: 'myGray.100'
|
||||
}
|
||||
}}
|
||||
>
|
||||
<MyIcon
|
||||
className="icon"
|
||||
name={'more'}
|
||||
h={'16px'}
|
||||
w={'16px'}
|
||||
px={1}
|
||||
py={1}
|
||||
borderRadius={'md'}
|
||||
cursor={'pointer'}
|
||||
/>
|
||||
</MenuButton>
|
||||
}
|
||||
menuList={[
|
||||
{
|
||||
child: (
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={'moveLight'} w={'14px'} mr={2} />
|
||||
{t('Move')}
|
||||
</Flex>
|
||||
),
|
||||
onClick: () => setMoveCollectionData({ collectionId: collection._id })
|
||||
},
|
||||
{
|
||||
child: (
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={'edit'} w={'14px'} mr={2} />
|
||||
{t('Rename')}
|
||||
</Flex>
|
||||
),
|
||||
onClick: () =>
|
||||
onOpenEditTitleModal({
|
||||
defaultVal: collection.name,
|
||||
onSuccess: (newName) => {
|
||||
onUpdateCollectionName({
|
||||
collectionId: collection._id,
|
||||
name: newName
|
||||
});
|
||||
}
|
||||
})
|
||||
},
|
||||
{
|
||||
child: (
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon
|
||||
mr={1}
|
||||
name={'delete'}
|
||||
w={'14px'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
/>
|
||||
<Box>{t('common.Delete')}</Box>
|
||||
</Flex>
|
||||
),
|
||||
onClick: () =>
|
||||
openConfirm(
|
||||
() => {
|
||||
onDelCollection(collection._id);
|
||||
},
|
||||
undefined,
|
||||
collection.type === DatasetCollectionTypeEnum.folder
|
||||
? t('dataset.collections.Confirm to delete the folder')
|
||||
: t('dataset.Confirm to delete the file')
|
||||
)()
|
||||
}
|
||||
]}
|
||||
/>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
<Loading loading={isLoading && collections.length === 0} fixed={false} />
|
||||
{total > pageSize && (
|
||||
<Flex mt={2} justifyContent={'center'}>
|
||||
<Pagination />
|
||||
</Flex>
|
||||
)}
|
||||
{total === 0 && <EmptyTip text="数据集空空如也" />}
|
||||
</TableContainer>
|
||||
|
||||
<ConfirmModal />
|
||||
<EditTitleModal />
|
||||
<EditCreateVirtualFileModal />
|
||||
{isOpenFileImportModal && (
|
||||
<FileImportModal
|
||||
datasetId={datasetId}
|
||||
parentId={parentId}
|
||||
uploadSuccess={() => {
|
||||
getData(1);
|
||||
onCloseFileImportModal();
|
||||
}}
|
||||
onClose={onCloseFileImportModal}
|
||||
/>
|
||||
)}
|
||||
|
||||
{!!editFolderData && (
|
||||
<EditFolderModal
|
||||
onClose={() => setEditFolderData(undefined)}
|
||||
editCallback={async (name) => {
|
||||
try {
|
||||
if (editFolderData.id) {
|
||||
await putDatasetCollectionById({
|
||||
id: editFolderData.id,
|
||||
name
|
||||
});
|
||||
} else {
|
||||
await postDatasetCollection({
|
||||
parentId,
|
||||
datasetId,
|
||||
name,
|
||||
type: DatasetCollectionTypeEnum.folder
|
||||
});
|
||||
}
|
||||
getData(pageNum);
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
}}
|
||||
isEdit={!!editFolderData.id}
|
||||
name={editFolderData.name}
|
||||
/>
|
||||
)}
|
||||
|
||||
{!!moveCollectionData && (
|
||||
<SelectCollections
|
||||
datasetId={datasetId}
|
||||
type="folder"
|
||||
defaultSelectedId={[moveCollectionData.collectionId]}
|
||||
onClose={() => setMoveCollectionData(undefined)}
|
||||
onSuccess={async ({ parentId }) => {
|
||||
await putDatasetCollectionById({
|
||||
id: moveCollectionData.collectionId,
|
||||
parentId
|
||||
});
|
||||
getData(pageNum);
|
||||
setMoveCollectionData(undefined);
|
||||
toast({
|
||||
status: 'success',
|
||||
title: t('common.folder.Move Success')
|
||||
});
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(CollectionCard);
|
267
projects/app/src/pages/dataset/detail/components/DataCard.tsx
Normal file
267
projects/app/src/pages/dataset/detail/components/DataCard.tsx
Normal file
@@ -0,0 +1,267 @@
|
||||
import React, { useCallback, useState, useRef, useMemo } from 'react';
|
||||
import { Box, Card, IconButton, Flex, Grid, Image, Button } from '@chakra-ui/react';
|
||||
import { usePagination } from '@/web/common/hooks/usePagination';
|
||||
import {
|
||||
getDatasetDataList,
|
||||
delOneDatasetDataById,
|
||||
getDatasetCollectionById
|
||||
} from '@/web/core/dataset/api';
|
||||
import { DeleteIcon } from '@chakra-ui/icons';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import { debounce } from 'lodash';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { useConfirm } from '@/web/common/hooks/useConfirm';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useRouter } from 'next/router';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import MyInput from '@/components/MyInput';
|
||||
import { useLoading } from '@/web/common/hooks/useLoading';
|
||||
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
import InputDataModal, { RawSourceText, type InputDataType } from '../components/InputDataModal';
|
||||
import type { DatasetDataListItemType } from '@/global/core/dataset/response.d';
|
||||
import { TabEnum } from '..';
|
||||
|
||||
const DataCard = () => {
|
||||
const BoxRef = useRef<HTMLDivElement>(null);
|
||||
const lastSearch = useRef('');
|
||||
const router = useRouter();
|
||||
const { collectionId = '' } = router.query as { collectionId: string };
|
||||
const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
|
||||
const { t } = useTranslation();
|
||||
const [searchText, setSearchText] = useState('');
|
||||
const { toast } = useToast();
|
||||
const [isDeleting, setIsDeleting] = useState(false);
|
||||
const { openConfirm, ConfirmModal } = useConfirm({
|
||||
content: t('dataset.Confirm to delete the data')
|
||||
});
|
||||
|
||||
const {
|
||||
data: datasetDataList,
|
||||
Pagination,
|
||||
total,
|
||||
getData,
|
||||
pageNum,
|
||||
pageSize
|
||||
} = usePagination<DatasetDataListItemType>({
|
||||
api: getDatasetDataList,
|
||||
pageSize: 24,
|
||||
params: {
|
||||
collectionId,
|
||||
searchText
|
||||
},
|
||||
onChange() {
|
||||
setIsLoading(false);
|
||||
if (BoxRef.current) {
|
||||
BoxRef.current.scrollTop = 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const [editInputData, setEditInputData] = useState<InputDataType>();
|
||||
|
||||
// get first page data
|
||||
const getFirstData = useCallback(
|
||||
debounce(() => {
|
||||
getData(1);
|
||||
lastSearch.current = searchText;
|
||||
}, 300),
|
||||
[]
|
||||
);
|
||||
|
||||
// get file info
|
||||
const { data: collection } = useQuery(['getDatasetCollectionById', collectionId], () =>
|
||||
getDatasetCollectionById(collectionId)
|
||||
);
|
||||
const fileIcon = useMemo(
|
||||
() => getCollectionIcon(collection?.type, collection?.name),
|
||||
[collection?.name, collection?.type]
|
||||
);
|
||||
|
||||
return (
|
||||
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
|
||||
<Flex alignItems={'center'}>
|
||||
<IconButton
|
||||
mr={3}
|
||||
icon={<MyIcon name={'backFill'} w={['14px', '18px']} color={'myBlue.600'} />}
|
||||
bg={'white'}
|
||||
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
|
||||
size={'sm'}
|
||||
borderRadius={'50%'}
|
||||
aria-label={''}
|
||||
onClick={() =>
|
||||
router.replace({
|
||||
query: {
|
||||
datasetId: router.query.datasetId,
|
||||
parentId: router.query.parentId,
|
||||
currentTab: TabEnum.collectionCard
|
||||
}
|
||||
})
|
||||
}
|
||||
/>
|
||||
<Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
|
||||
<Image src={fileIcon || '/imgs/files/file.svg'} w={'16px'} mr={2} alt={''} />
|
||||
<RawSourceText
|
||||
sourceName={collection?.name}
|
||||
sourceId={collection?.metadata?.fileId || collection?.metadata?.rawLink}
|
||||
fontSize={['md', 'lg']}
|
||||
color={'black'}
|
||||
textDecoration={'none'}
|
||||
/>
|
||||
</Flex>
|
||||
<Box>
|
||||
<Button
|
||||
ml={2}
|
||||
variant={'base'}
|
||||
size={['sm', 'md']}
|
||||
onClick={() => {
|
||||
if (!collection) return;
|
||||
setEditInputData({
|
||||
datasetId: collection.datasetId,
|
||||
collectionId: collection._id,
|
||||
sourceId: collection.metadata?.fileId || collection.metadata?.rawLink,
|
||||
sourceName: collection.name
|
||||
});
|
||||
}}
|
||||
>
|
||||
{t('dataset.Insert Data')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Flex>
|
||||
<Flex my={3} alignItems={'center'}>
|
||||
<Box>
|
||||
<Box as={'span'} fontSize={['md', 'lg']}>
|
||||
{total}组
|
||||
</Box>
|
||||
</Box>
|
||||
<Box flex={1} mr={1} />
|
||||
<MyInput
|
||||
leftIcon={
|
||||
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
|
||||
}
|
||||
w={['200px', '300px']}
|
||||
placeholder="根据匹配知识,预期答案和来源进行搜索"
|
||||
value={searchText}
|
||||
onChange={(e) => {
|
||||
setSearchText(e.target.value);
|
||||
getFirstData();
|
||||
}}
|
||||
onBlur={() => {
|
||||
if (searchText === lastSearch.current) return;
|
||||
getFirstData();
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
if (searchText === lastSearch.current) return;
|
||||
if (e.key === 'Enter') {
|
||||
getFirstData();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
<Grid
|
||||
minH={'100px'}
|
||||
gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)']}
|
||||
gridGap={4}
|
||||
>
|
||||
{datasetDataList.map((item) => (
|
||||
<Card
|
||||
key={item.id}
|
||||
cursor={'pointer'}
|
||||
pt={3}
|
||||
userSelect={'none'}
|
||||
boxShadow={'none'}
|
||||
_hover={{ boxShadow: 'lg', '& .delete': { display: 'flex' } }}
|
||||
border={'1px solid '}
|
||||
borderColor={'myGray.200'}
|
||||
onClick={() => {
|
||||
if (!collection) return;
|
||||
setEditInputData({
|
||||
id: item.id,
|
||||
datasetId: collection.datasetId,
|
||||
collectionId: collection._id,
|
||||
q: item.q,
|
||||
a: item.a,
|
||||
sourceId: collection.metadata?.fileId || collection.metadata?.rawLink,
|
||||
sourceName: collection.name
|
||||
});
|
||||
}}
|
||||
>
|
||||
<Box
|
||||
h={'95px'}
|
||||
overflow={'hidden'}
|
||||
wordBreak={'break-all'}
|
||||
px={3}
|
||||
py={1}
|
||||
fontSize={'13px'}
|
||||
>
|
||||
<Box color={'myGray.1000'} mb={2}>
|
||||
{item.q}
|
||||
</Box>
|
||||
<Box color={'myGray.600'}>{item.a}</Box>
|
||||
</Box>
|
||||
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
|
||||
<Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
|
||||
ID:{item.id}
|
||||
</Box>
|
||||
<IconButton
|
||||
className="delete"
|
||||
display={['flex', 'none']}
|
||||
icon={<DeleteIcon />}
|
||||
variant={'base'}
|
||||
colorScheme={'gray'}
|
||||
aria-label={'delete'}
|
||||
size={'xs'}
|
||||
borderRadius={'md'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
isLoading={isDeleting}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
openConfirm(async () => {
|
||||
try {
|
||||
setIsDeleting(true);
|
||||
await delOneDatasetDataById(item.id);
|
||||
getData(pageNum);
|
||||
} catch (error) {
|
||||
toast({
|
||||
title: getErrText(error),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setIsDeleting(false);
|
||||
})();
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
</Card>
|
||||
))}
|
||||
</Grid>
|
||||
|
||||
{total > pageSize && (
|
||||
<Flex mt={2} justifyContent={'center'}>
|
||||
<Pagination />
|
||||
</Flex>
|
||||
)}
|
||||
{total === 0 && (
|
||||
<Flex flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
|
||||
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
|
||||
<Box mt={2} color={'myGray.500'}>
|
||||
知识库空空如也
|
||||
</Box>
|
||||
</Flex>
|
||||
)}
|
||||
|
||||
{editInputData !== undefined && collection && (
|
||||
<InputDataModal
|
||||
datasetId={collection.datasetId}
|
||||
defaultValues={editInputData}
|
||||
onClose={() => setEditInputData(undefined)}
|
||||
onSuccess={() => getData(pageNum)}
|
||||
/>
|
||||
)}
|
||||
<ConfirmModal />
|
||||
<Loading fixed={false} />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(DataCard);
|
@@ -0,0 +1,125 @@
|
||||
import React, { useState } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
Button,
|
||||
NumberInput,
|
||||
NumberInputField,
|
||||
NumberInputStepper,
|
||||
NumberIncrementStepper,
|
||||
NumberDecrementStepper
|
||||
} from '@chakra-ui/react';
|
||||
import { useConfirm } from '@/web/common/hooks/useConfirm';
|
||||
import { formatPrice } from '@fastgpt/global/common/bill/tools';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
|
||||
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
|
||||
|
||||
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
|
||||
|
||||
const ChunkImport = () => {
|
||||
const { datasetDetail } = useDatasetStore();
|
||||
const vectorModel = datasetDetail.vectorModel;
|
||||
const unitPrice = vectorModel?.price || 0.2;
|
||||
|
||||
const {
|
||||
chunkLen,
|
||||
setChunkLen,
|
||||
successChunks,
|
||||
totalChunks,
|
||||
isUnselectedFile,
|
||||
price,
|
||||
onclickUpload,
|
||||
onReSplitChunks,
|
||||
uploading,
|
||||
showRePreview,
|
||||
setReShowRePreview
|
||||
} = useImportStore();
|
||||
|
||||
const { openConfirm, ConfirmModal } = useConfirm({
|
||||
content: `该任务无法终止,需要一定时间生成索引,请确认导入。如果余额不足,未完成的任务会被暂停,充值后可继续进行。`
|
||||
});
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} h={['auto', '100%']}>
|
||||
<SelectorContainer fileExtension={fileExtension}>
|
||||
{/* chunk size */}
|
||||
<Flex py={4} alignItems={'center'}>
|
||||
<Box>
|
||||
段落长度
|
||||
<MyTooltip
|
||||
label={
|
||||
'按结束标点符号进行分段。前后段落会有 20% 的内容重叠。\n中文文档建议不要超过1000,英文不要超过1500'
|
||||
}
|
||||
forceShow
|
||||
>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Box
|
||||
flex={1}
|
||||
css={{
|
||||
'& > span': {
|
||||
display: 'block'
|
||||
}
|
||||
}}
|
||||
>
|
||||
<MyTooltip label={`范围: 100~${datasetDetail.vectorModel.maxToken}`}>
|
||||
<NumberInput
|
||||
ml={4}
|
||||
defaultValue={chunkLen}
|
||||
min={100}
|
||||
max={datasetDetail.vectorModel.maxToken}
|
||||
step={10}
|
||||
onChange={(e) => {
|
||||
setChunkLen(+e);
|
||||
setReShowRePreview(true);
|
||||
}}
|
||||
>
|
||||
<NumberInputField />
|
||||
<NumberInputStepper>
|
||||
<NumberIncrementStepper />
|
||||
<NumberDecrementStepper />
|
||||
</NumberInputStepper>
|
||||
</NumberInput>
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
</Flex>
|
||||
{/* price */}
|
||||
<Flex py={4} alignItems={'center'}>
|
||||
<Box>
|
||||
预估价格
|
||||
<MyTooltip
|
||||
label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}
|
||||
forceShow
|
||||
>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Box ml={4}>{price}元</Box>
|
||||
</Flex>
|
||||
<Flex mt={3}>
|
||||
{showRePreview && (
|
||||
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
|
||||
重新生成预览
|
||||
</Button>
|
||||
)}
|
||||
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
|
||||
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
|
||||
</Button>
|
||||
</Flex>
|
||||
</SelectorContainer>
|
||||
|
||||
{!isUnselectedFile && (
|
||||
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
|
||||
<PreviewFileOrChunk />
|
||||
</Box>
|
||||
)}
|
||||
<ConfirmModal />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default ChunkImport;
|
@@ -0,0 +1,66 @@
|
||||
import React from 'react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import MyModal from '@/components/MyModal';
|
||||
import { Box, Input, Textarea, ModalBody, ModalFooter, Button } from '@chakra-ui/react';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { useRequest } from '@/web/common/hooks/useRequest';
|
||||
|
||||
const CreateFileModal = ({
|
||||
onClose,
|
||||
onSuccess
|
||||
}: {
|
||||
onClose: () => void;
|
||||
onSuccess: (e: { filename: string; content: string }) => Promise<void>;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const { register, handleSubmit } = useForm({
|
||||
defaultValues: {
|
||||
filename: '',
|
||||
content: ''
|
||||
}
|
||||
});
|
||||
|
||||
const { mutate, isLoading } = useRequest({
|
||||
mutationFn: () => handleSubmit(onSuccess)(),
|
||||
onSuccess: () => {
|
||||
onClose();
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<MyModal title={t('file.Create File')} isOpen w={'600px'} top={'15vh'}>
|
||||
<ModalBody>
|
||||
<Box mb={1} fontSize={'sm'}>
|
||||
文件名
|
||||
</Box>
|
||||
<Input
|
||||
mb={5}
|
||||
{...register('filename', {
|
||||
required: '文件名不能为空'
|
||||
})}
|
||||
/>
|
||||
<Box mb={1} fontSize={'sm'}>
|
||||
文件内容
|
||||
</Box>
|
||||
<Textarea
|
||||
{...register('content', {
|
||||
required: '文件内容不能为空'
|
||||
})}
|
||||
rows={12}
|
||||
whiteSpace={'nowrap'}
|
||||
resize={'both'}
|
||||
/>
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button variant={'base'} mr={4} onClick={onClose}>
|
||||
取消
|
||||
</Button>
|
||||
<Button isLoading={isLoading} onClick={mutate}>
|
||||
确认
|
||||
</Button>
|
||||
</ModalFooter>
|
||||
</MyModal>
|
||||
);
|
||||
};
|
||||
|
||||
export default CreateFileModal;
|
@@ -0,0 +1,36 @@
|
||||
import React from 'react';
|
||||
import { Box, Flex, Button } from '@chakra-ui/react';
|
||||
import { useConfirm } from '@/web/common/hooks/useConfirm';
|
||||
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
|
||||
|
||||
const fileExtension = '.csv';
|
||||
|
||||
const CsvImport = () => {
|
||||
const { successChunks, totalChunks, isUnselectedFile, onclickUpload, uploading } =
|
||||
useImportStore();
|
||||
|
||||
const { openConfirm, ConfirmModal } = useConfirm({
|
||||
content: `该任务无法终止,需要一定时间生成索引,请确认导入。如果余额不足,未完成的任务会被暂停,充值后可继续进行。`
|
||||
});
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} h={['auto', '100%']}>
|
||||
<SelectorContainer fileExtension={fileExtension} showUrlFetch={false}>
|
||||
<Flex mt={3}>
|
||||
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
|
||||
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
|
||||
</Button>
|
||||
</Flex>
|
||||
</SelectorContainer>
|
||||
|
||||
{!isUnselectedFile && (
|
||||
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
|
||||
<PreviewFileOrChunk />
|
||||
</Box>
|
||||
)}
|
||||
<ConfirmModal />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default CsvImport;
|
@@ -0,0 +1,424 @@
|
||||
import MyIcon from '@/components/Icon';
|
||||
import { useLoading } from '@/web/common/hooks/useLoading';
|
||||
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import { splitText2Chunks } from '@/global/common/string/tools';
|
||||
import { simpleText } from '@fastgpt/global/common/string/tools';
|
||||
import {
|
||||
uploadFiles,
|
||||
fileDownload,
|
||||
readCsvContent,
|
||||
readTxtContent,
|
||||
readPdfContent,
|
||||
readDocContent
|
||||
} from '@/web/common/file/utils';
|
||||
import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
|
||||
import { DragEvent, useCallback, useState } from 'react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import dynamic from 'next/dynamic';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
|
||||
import type {
|
||||
DatasetChunkItemType,
|
||||
DatasetCollectionSchemaType
|
||||
} from '@fastgpt/global/core/dataset/type';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
import { getFileIcon } from '@fastgpt/global/common/file/icon';
|
||||
import { countPromptTokens } from '@/global/common/tiktoken';
|
||||
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
|
||||
|
||||
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
|
||||
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
|
||||
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
const csvTemplate = `index,content\n"被索引的内容","对应的答案。CSV 中请注意内容不能包含双引号,双引号是列分割符号"\n"什么是 laf","laf 是一个云函数开发平台……",""\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
|
||||
|
||||
export type FileItemType = {
|
||||
id: string; // fileId / raw Link
|
||||
filename: string;
|
||||
chunks: DatasetChunkItemType[];
|
||||
text: string; // raw text
|
||||
icon: string;
|
||||
tokens: number; // total tokens
|
||||
type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
|
||||
metadata: DatasetCollectionSchemaType['metadata'];
|
||||
};
|
||||
|
||||
interface Props extends BoxProps {
|
||||
fileExtension: string;
|
||||
onPushFiles: (files: FileItemType[]) => void;
|
||||
tipText?: string;
|
||||
chunkLen?: number;
|
||||
isCsv?: boolean;
|
||||
showUrlFetch?: boolean;
|
||||
showCreateFile?: boolean;
|
||||
}
|
||||
|
||||
const FileSelect = ({
|
||||
fileExtension,
|
||||
onPushFiles,
|
||||
tipText,
|
||||
chunkLen = 500,
|
||||
isCsv = false,
|
||||
showUrlFetch = true,
|
||||
showCreateFile = true,
|
||||
...props
|
||||
}: Props) => {
|
||||
const { datasetDetail } = useDatasetStore();
|
||||
const { Loading: FileSelectLoading } = useLoading();
|
||||
const { t } = useTranslation();
|
||||
|
||||
const { toast } = useToast();
|
||||
|
||||
const { File: FileSelector, onOpen } = useSelectFile({
|
||||
fileType: fileExtension,
|
||||
multiple: true
|
||||
});
|
||||
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const [selectingText, setSelectingText] = useState<string>();
|
||||
|
||||
const {
|
||||
isOpen: isOpenUrlFetch,
|
||||
onOpen: onOpenUrlFetch,
|
||||
onClose: onCloseUrlFetch
|
||||
} = useDisclosure();
|
||||
const {
|
||||
isOpen: isOpenCreateFile,
|
||||
onOpen: onOpenCreateFile,
|
||||
onClose: onCloseCreateFile
|
||||
} = useDisclosure();
|
||||
|
||||
// select file
|
||||
const onSelectFile = useCallback(
|
||||
async (files: File[]) => {
|
||||
try {
|
||||
for await (let file of files) {
|
||||
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
|
||||
|
||||
/* text file */
|
||||
const icon = getFileIcon(file?.name);
|
||||
|
||||
// ts
|
||||
if (!icon) continue;
|
||||
|
||||
// upload file
|
||||
const filesId = await uploadFiles([file], { datasetId: datasetDetail._id }, (percent) => {
|
||||
if (percent < 100) {
|
||||
setSelectingText(
|
||||
t('file.Uploading', { name: file.name.slice(0, 30), percent }) || ''
|
||||
);
|
||||
} else {
|
||||
setSelectingText(t('file.Parse', { name: file.name.slice(0, 30) }) || '');
|
||||
}
|
||||
});
|
||||
const fileId = filesId[0];
|
||||
|
||||
/* csv file */
|
||||
if (extension === 'csv') {
|
||||
const { header, data } = await readCsvContent(file);
|
||||
if (header[0] !== 'index' || header[1] !== 'content') {
|
||||
throw new Error('csv 文件格式有误,请确保 index 和 content 两列');
|
||||
}
|
||||
|
||||
const filterData = data
|
||||
.filter((item) => item[0])
|
||||
.map((item) => ({
|
||||
q: item[0] || '',
|
||||
a: item[1] || ''
|
||||
}));
|
||||
|
||||
const fileItem: FileItemType = {
|
||||
id: nanoid(),
|
||||
filename: file.name,
|
||||
icon,
|
||||
tokens: filterData.reduce((sum, item) => sum + countPromptTokens(item.q), 0),
|
||||
text: '',
|
||||
chunks: filterData,
|
||||
type: DatasetCollectionTypeEnum.file,
|
||||
metadata: {
|
||||
fileId
|
||||
}
|
||||
};
|
||||
|
||||
onPushFiles([fileItem]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// parse and upload files
|
||||
let text = await (async () => {
|
||||
switch (extension) {
|
||||
case 'txt':
|
||||
case 'md':
|
||||
return readTxtContent(file);
|
||||
case 'pdf':
|
||||
return readPdfContent(file);
|
||||
case 'doc':
|
||||
case 'docx':
|
||||
return readDocContent(file);
|
||||
}
|
||||
return '';
|
||||
})();
|
||||
|
||||
if (text) {
|
||||
text = simpleText(text);
|
||||
const splitRes = splitText2Chunks({
|
||||
text,
|
||||
maxLen: chunkLen
|
||||
});
|
||||
|
||||
const fileItem: FileItemType = {
|
||||
id: nanoid(),
|
||||
filename: file.name,
|
||||
icon,
|
||||
text,
|
||||
tokens: splitRes.tokens,
|
||||
type: DatasetCollectionTypeEnum.file,
|
||||
metadata: {
|
||||
fileId
|
||||
},
|
||||
chunks: splitRes.chunks.map((chunk) => ({
|
||||
q: chunk,
|
||||
a: ''
|
||||
}))
|
||||
};
|
||||
onPushFiles([fileItem]);
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.log(error);
|
||||
toast({
|
||||
title: getErrText(error, '解析文件失败'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setSelectingText(undefined);
|
||||
},
|
||||
[chunkLen, datasetDetail._id, onPushFiles, t, toast]
|
||||
);
|
||||
// link fetch
|
||||
const onUrlFetch = useCallback(
|
||||
(e: FetchResultItem[]) => {
|
||||
const result: FileItemType[] = e.map(({ url, content }) => {
|
||||
const splitRes = splitText2Chunks({
|
||||
text: content,
|
||||
maxLen: chunkLen
|
||||
});
|
||||
return {
|
||||
id: nanoid(),
|
||||
filename: url,
|
||||
icon: '/imgs/files/link.svg',
|
||||
text: content,
|
||||
tokens: splitRes.tokens,
|
||||
type: DatasetCollectionTypeEnum.link,
|
||||
metadata: {
|
||||
rawLink: url
|
||||
},
|
||||
chunks: splitRes.chunks.map((chunk) => ({
|
||||
q: chunk,
|
||||
a: ''
|
||||
}))
|
||||
};
|
||||
});
|
||||
onPushFiles(result);
|
||||
},
|
||||
[chunkLen, onPushFiles]
|
||||
);
|
||||
// manual create file and copy data
|
||||
const onCreateFile = useCallback(
|
||||
async ({ filename, content }: { filename: string; content: string }) => {
|
||||
content = simpleText(content);
|
||||
|
||||
// create virtual txt file
|
||||
const txtBlob = new Blob([content], { type: 'text/plain' });
|
||||
const txtFile = new File([txtBlob], `${filename}.txt`, {
|
||||
type: txtBlob.type,
|
||||
lastModified: new Date().getTime()
|
||||
});
|
||||
const fileIds = await uploadFiles([txtFile], { datasetId: datasetDetail._id });
|
||||
|
||||
const splitRes = splitText2Chunks({
|
||||
text: content,
|
||||
maxLen: chunkLen
|
||||
});
|
||||
|
||||
onPushFiles([
|
||||
{
|
||||
id: nanoid(),
|
||||
filename,
|
||||
icon: '/imgs/files/txt.svg',
|
||||
text: content,
|
||||
tokens: splitRes.tokens,
|
||||
type: DatasetCollectionTypeEnum.file,
|
||||
metadata: {
|
||||
fileId: fileIds[0]
|
||||
},
|
||||
chunks: splitRes.chunks.map((chunk) => ({
|
||||
q: chunk,
|
||||
a: ''
|
||||
}))
|
||||
}
|
||||
]);
|
||||
},
|
||||
[chunkLen, datasetDetail._id, onPushFiles]
|
||||
);
|
||||
|
||||
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(true);
|
||||
};
|
||||
|
||||
const handleDragLeave = (e: DragEvent<HTMLDivElement>) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
};
|
||||
|
||||
const handleDrop = useCallback(
|
||||
async (e: DragEvent<HTMLDivElement>) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
|
||||
const items = e.dataTransfer.items;
|
||||
const fileList: File[] = [];
|
||||
|
||||
if (e.dataTransfer.items.length <= 1) {
|
||||
const traverseFileTree = async (item: any) => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
if (item.isFile) {
|
||||
item.file((file: File) => {
|
||||
fileList.push(file);
|
||||
resolve();
|
||||
});
|
||||
} else if (item.isDirectory) {
|
||||
const dirReader = item.createReader();
|
||||
dirReader.readEntries(async (entries: any[]) => {
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
await traverseFileTree(entries[i]);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const item = items[i].webkitGetAsEntry();
|
||||
if (item) {
|
||||
await traverseFileTree(item);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const files = Array.from(e.dataTransfer.files);
|
||||
let isErr = files.some((item) => item.type === '');
|
||||
if (isErr) {
|
||||
return toast({
|
||||
title: t('file.upload error description'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
fileList.push(files[i]);
|
||||
}
|
||||
}
|
||||
|
||||
onSelectFile(fileList);
|
||||
},
|
||||
[onSelectFile, t, toast]
|
||||
);
|
||||
|
||||
const SelectTextStyles: BoxProps = {
|
||||
ml: 1,
|
||||
as: 'span',
|
||||
cursor: 'pointer',
|
||||
color: 'myBlue.700',
|
||||
_hover: {
|
||||
textDecoration: 'underline'
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Box
|
||||
display={'inline-block'}
|
||||
textAlign={'center'}
|
||||
bg={'myWhite.400'}
|
||||
p={5}
|
||||
borderRadius={'lg'}
|
||||
border={'1px dashed'}
|
||||
borderColor={'myGray.300'}
|
||||
w={'100%'}
|
||||
position={'relative'}
|
||||
{...props}
|
||||
onDragEnter={handleDragEnter}
|
||||
onDragOver={handleDragEnter}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}
|
||||
>
|
||||
<Flex justifyContent={'center'} alignItems={'center'}>
|
||||
<MyIcon mr={1} name={'uploadFile'} w={'16px'} />
|
||||
{isDragging ? (
|
||||
t('file.Release the mouse to upload the file')
|
||||
) : (
|
||||
<Box>
|
||||
{t('file.Drag and drop')},
|
||||
<MyTooltip label={t('file.max 10')}>
|
||||
<Box {...SelectTextStyles} onClick={onOpen}>
|
||||
{t('file.select a document')}
|
||||
</Box>
|
||||
</MyTooltip>
|
||||
{showUrlFetch && (
|
||||
<>
|
||||
,
|
||||
<Box {...SelectTextStyles} onClick={onOpenUrlFetch}>
|
||||
{t('file.Fetch Url')}
|
||||
</Box>
|
||||
</>
|
||||
)}
|
||||
{showCreateFile && (
|
||||
<>
|
||||
,
|
||||
<Box {...SelectTextStyles} onClick={onOpenCreateFile}>
|
||||
{t('file.Create file')}
|
||||
</Box>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
</Flex>
|
||||
<Box mt={1}>{t('file.support', { fileExtension: fileExtension })}</Box>
|
||||
{tipText && (
|
||||
<Box mt={1} fontSize={'sm'} color={'myGray.600'}>
|
||||
{t(tipText)}
|
||||
</Box>
|
||||
)}
|
||||
{isCsv && (
|
||||
<Box
|
||||
mt={1}
|
||||
cursor={'pointer'}
|
||||
textDecoration={'underline'}
|
||||
color={'myBlue.600'}
|
||||
fontSize={'12px'}
|
||||
onClick={() =>
|
||||
fileDownload({
|
||||
text: csvTemplate,
|
||||
type: 'text/csv',
|
||||
filename: 'template.csv'
|
||||
})
|
||||
}
|
||||
>
|
||||
{t('file.Click to download CSV template')}
|
||||
</Box>
|
||||
)}
|
||||
{selectingText !== undefined && (
|
||||
<FileSelectLoading loading text={selectingText} fixed={false} />
|
||||
)}
|
||||
<FileSelector onSelect={onSelectFile} />
|
||||
{isOpenUrlFetch && <UrlFetchModal onClose={onCloseUrlFetch} onSuccess={onUrlFetch} />}
|
||||
{isOpenCreateFile && <CreateFileModal onClose={onCloseCreateFile} onSuccess={onCreateFile} />}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default FileSelect;
|
@@ -0,0 +1,123 @@
|
||||
import React, { useMemo, useState } from 'react';
|
||||
import { Box, type BoxProps, Flex, useTheme, ModalCloseButton } from '@chakra-ui/react';
|
||||
import MyRadio from '@/components/Radio/index';
|
||||
import dynamic from 'next/dynamic';
|
||||
import ChunkImport from './Chunk';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
const QAImport = dynamic(() => import('./QA'), {});
|
||||
const CsvImport = dynamic(() => import('./Csv'), {});
|
||||
import MyModal from '@/components/MyModal';
|
||||
import Provider from './Provider';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
import { qaModelList } from '@/web/common/system/staticData';
|
||||
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
|
||||
|
||||
export enum ImportTypeEnum {
|
||||
index = 'index',
|
||||
qa = 'qa',
|
||||
csv = 'csv'
|
||||
}
|
||||
|
||||
const ImportData = ({
|
||||
datasetId,
|
||||
parentId,
|
||||
onClose,
|
||||
uploadSuccess
|
||||
}: {
|
||||
datasetId: string;
|
||||
parentId: string;
|
||||
onClose: () => void;
|
||||
uploadSuccess: () => void;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const theme = useTheme();
|
||||
const { datasetDetail } = useDatasetStore();
|
||||
const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.index);
|
||||
|
||||
const typeMap = useMemo(() => {
|
||||
const vectorModel = datasetDetail.vectorModel;
|
||||
const qaModel = qaModelList[0];
|
||||
const map = {
|
||||
[ImportTypeEnum.index]: {
|
||||
defaultChunkLen: vectorModel?.defaultToken || 500,
|
||||
unitPrice: vectorModel?.price || 0.2,
|
||||
mode: TrainingModeEnum.index
|
||||
},
|
||||
[ImportTypeEnum.qa]: {
|
||||
defaultChunkLen: qaModel?.maxToken * 0.5 || 8000,
|
||||
unitPrice: qaModel?.price || 3,
|
||||
mode: TrainingModeEnum.qa
|
||||
},
|
||||
[ImportTypeEnum.csv]: {
|
||||
defaultChunkLen: vectorModel?.defaultToken || 500,
|
||||
unitPrice: vectorModel?.price || 0.2,
|
||||
mode: TrainingModeEnum.index
|
||||
}
|
||||
};
|
||||
return map[importType];
|
||||
}, [datasetDetail.vectorModel, importType]);
|
||||
|
||||
const TitleStyle: BoxProps = {
|
||||
fontWeight: 'bold',
|
||||
fontSize: ['md', 'xl']
|
||||
};
|
||||
|
||||
return (
|
||||
<MyModal
|
||||
title={<Box {...TitleStyle}>{t('dataset.data.File import')}</Box>}
|
||||
isOpen
|
||||
isCentered
|
||||
maxW={['90vw', '85vw']}
|
||||
w={['90vw', '85vw']}
|
||||
h={'90vh'}
|
||||
>
|
||||
<ModalCloseButton onClick={onClose} />
|
||||
<Flex flexDirection={'column'} flex={'1 0 0'}>
|
||||
<Box pb={[5, 7]} px={[4, 8]} borderBottom={theme.borders.base}>
|
||||
<MyRadio
|
||||
gridTemplateColumns={['repeat(1,1fr)', 'repeat(3, 350px)']}
|
||||
list={[
|
||||
{
|
||||
icon: 'indexImport',
|
||||
title: '直接分段',
|
||||
desc: '选择文本文件,直接将其按分段进行处理',
|
||||
value: ImportTypeEnum.index
|
||||
},
|
||||
{
|
||||
icon: 'qaImport',
|
||||
title: 'QA拆分',
|
||||
desc: '选择文本文件,让大模型自动生成问答对',
|
||||
value: ImportTypeEnum.qa
|
||||
},
|
||||
{
|
||||
icon: 'csvImport',
|
||||
title: 'CSV 导入',
|
||||
desc: '批量导入问答对,是最精准的数据',
|
||||
value: ImportTypeEnum.csv
|
||||
}
|
||||
]}
|
||||
value={importType}
|
||||
onChange={(e) => setImportType(e as `${ImportTypeEnum}`)}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Provider
|
||||
{...typeMap}
|
||||
importType={importType}
|
||||
datasetId={datasetId}
|
||||
parentId={parentId}
|
||||
onUploadSuccess={uploadSuccess}
|
||||
>
|
||||
<Box flex={'1 0 0'} h={0}>
|
||||
{importType === ImportTypeEnum.index && <ChunkImport />}
|
||||
{importType === ImportTypeEnum.qa && <QAImport />}
|
||||
{importType === ImportTypeEnum.csv && <CsvImport />}
|
||||
</Box>
|
||||
</Provider>
|
||||
</Flex>
|
||||
</MyModal>
|
||||
);
|
||||
};
|
||||
|
||||
export default ImportData;
|
@@ -0,0 +1,473 @@
|
||||
import React, {
|
||||
type SetStateAction,
|
||||
type Dispatch,
|
||||
useContext,
|
||||
useCallback,
|
||||
createContext,
|
||||
useState,
|
||||
useMemo,
|
||||
useEffect
|
||||
} from 'react';
|
||||
import FileSelect, { FileItemType } from './FileSelect';
|
||||
import { useRequest } from '@/web/common/hooks/useRequest';
|
||||
import { postDatasetCollection } from '@/web/core/dataset/api';
|
||||
import { formatPrice } from '@fastgpt/global/common/bill/tools';
|
||||
import { splitText2Chunks } from '@/global/common/string/tools';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
|
||||
import { Box, Flex, Image, useTheme } from '@chakra-ui/react';
|
||||
import { CloseIcon } from '@chakra-ui/icons';
|
||||
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import { chunksUpload } from '@/web/core/dataset/utils';
|
||||
import { postCreateTrainingBill } from '@/web/common/bill/api';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { ImportTypeEnum } from './ImportModal';
|
||||
|
||||
const filenameStyles = {
|
||||
className: 'textEllipsis',
|
||||
maxW: '400px'
|
||||
};
|
||||
|
||||
type useImportStoreType = {
|
||||
files: FileItemType[];
|
||||
setFiles: Dispatch<SetStateAction<FileItemType[]>>;
|
||||
previewFile: FileItemType | undefined;
|
||||
setPreviewFile: Dispatch<SetStateAction<FileItemType | undefined>>;
|
||||
successChunks: number;
|
||||
setSuccessChunks: Dispatch<SetStateAction<number>>;
|
||||
isUnselectedFile: boolean;
|
||||
totalChunks: number;
|
||||
onclickUpload: (e: { files: FileItemType[] }) => void;
|
||||
onReSplitChunks: () => void;
|
||||
price: number;
|
||||
uploading: boolean;
|
||||
chunkLen: number;
|
||||
setChunkLen: Dispatch<number>;
|
||||
showRePreview: boolean;
|
||||
setReShowRePreview: Dispatch<SetStateAction<boolean>>;
|
||||
};
|
||||
const StateContext = createContext<useImportStoreType>({
|
||||
onclickUpload: function (e: { files: FileItemType[] }): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
uploading: false,
|
||||
files: [],
|
||||
|
||||
previewFile: undefined,
|
||||
|
||||
successChunks: 0,
|
||||
|
||||
isUnselectedFile: false,
|
||||
totalChunks: 0,
|
||||
onReSplitChunks: function (): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
price: 0,
|
||||
chunkLen: 0,
|
||||
setChunkLen: function (value: number): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
setFiles: function (value: React.SetStateAction<FileItemType[]>): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
setPreviewFile: function (value: React.SetStateAction<FileItemType | undefined>): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
setSuccessChunks: function (value: React.SetStateAction<number>): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
showRePreview: false,
|
||||
setReShowRePreview: function (value: React.SetStateAction<boolean>): void {
|
||||
throw new Error('Function not implemented.');
|
||||
}
|
||||
});
|
||||
export const useImportStore = () => useContext(StateContext);
|
||||
|
||||
const Provider = ({
|
||||
datasetId,
|
||||
parentId,
|
||||
unitPrice,
|
||||
mode,
|
||||
defaultChunkLen = 500,
|
||||
importType,
|
||||
onUploadSuccess,
|
||||
children
|
||||
}: {
|
||||
datasetId: string;
|
||||
parentId: string;
|
||||
unitPrice: number;
|
||||
mode: `${TrainingModeEnum}`;
|
||||
defaultChunkLen: number;
|
||||
importType: `${ImportTypeEnum}`;
|
||||
onUploadSuccess: () => void;
|
||||
children: React.ReactNode;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const { toast } = useToast();
|
||||
const [files, setFiles] = useState<FileItemType[]>([]);
|
||||
const [successChunks, setSuccessChunks] = useState(0);
|
||||
const [chunkLen, setChunkLen] = useState(defaultChunkLen);
|
||||
const [previewFile, setPreviewFile] = useState<FileItemType>();
|
||||
const [showRePreview, setReShowRePreview] = useState(false);
|
||||
|
||||
const isUnselectedFile = useMemo(() => files.length === 0, [files]);
|
||||
|
||||
const totalChunks = useMemo(
|
||||
() => files.reduce((sum, file) => sum + file.chunks.length, 0),
|
||||
[files]
|
||||
);
|
||||
|
||||
const price = useMemo(() => {
|
||||
return formatPrice(files.reduce((sum, file) => sum + file.tokens, 0) * unitPrice);
|
||||
}, [files, unitPrice]);
|
||||
|
||||
/* start upload data */
|
||||
const { mutate: onclickUpload, isLoading: uploading } = useRequest({
|
||||
mutationFn: async () => {
|
||||
let totalInsertion = 0;
|
||||
for await (const file of files) {
|
||||
const chunks = file.chunks;
|
||||
// create a file collection and training bill
|
||||
const [collectionId, billId] = await Promise.all([
|
||||
postDatasetCollection({
|
||||
datasetId,
|
||||
parentId,
|
||||
name: file.filename,
|
||||
type: file.type,
|
||||
metadata: file.metadata
|
||||
}),
|
||||
postCreateTrainingBill({
|
||||
name: t('dataset.collections.Create Training Data', { filename: file.filename })
|
||||
})
|
||||
]);
|
||||
// upload data
|
||||
const { insertLen } = await chunksUpload({
|
||||
collectionId,
|
||||
billId,
|
||||
chunks,
|
||||
mode,
|
||||
onUploading: (insertLen) => {
|
||||
setSuccessChunks((state) => state + insertLen);
|
||||
}
|
||||
});
|
||||
totalInsertion += insertLen;
|
||||
}
|
||||
return totalInsertion;
|
||||
},
|
||||
onSuccess(num) {
|
||||
toast({
|
||||
title: `共成功导入 ${num} 组数据,请耐心等待训练.`,
|
||||
status: 'success'
|
||||
});
|
||||
onUploadSuccess();
|
||||
},
|
||||
errorToast: '导入文件失败'
|
||||
});
|
||||
|
||||
const onReSplitChunks = useCallback(async () => {
|
||||
try {
|
||||
setFiles((state) =>
|
||||
state.map((file) => {
|
||||
const splitRes = splitText2Chunks({
|
||||
text: file.text,
|
||||
maxLen: chunkLen
|
||||
});
|
||||
|
||||
return {
|
||||
...file,
|
||||
tokens: splitRes.tokens,
|
||||
chunks: splitRes.chunks.map((chunk) => ({
|
||||
q: chunk,
|
||||
a: ''
|
||||
}))
|
||||
};
|
||||
})
|
||||
);
|
||||
setReShowRePreview(false);
|
||||
} catch (error) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: getErrText(error, '文本分段异常')
|
||||
});
|
||||
}
|
||||
}, [chunkLen, toast]);
|
||||
|
||||
const reset = useCallback(() => {
|
||||
setFiles([]);
|
||||
setSuccessChunks(0);
|
||||
setChunkLen(defaultChunkLen);
|
||||
setPreviewFile(undefined);
|
||||
setReShowRePreview(false);
|
||||
}, [defaultChunkLen]);
|
||||
|
||||
useEffect(() => {
|
||||
reset();
|
||||
}, [importType, reset]);
|
||||
|
||||
const value = {
|
||||
files,
|
||||
setFiles,
|
||||
previewFile,
|
||||
setPreviewFile,
|
||||
successChunks,
|
||||
setSuccessChunks,
|
||||
isUnselectedFile,
|
||||
totalChunks,
|
||||
price,
|
||||
onReSplitChunks,
|
||||
onclickUpload,
|
||||
uploading,
|
||||
chunkLen,
|
||||
setChunkLen,
|
||||
showRePreview,
|
||||
setReShowRePreview
|
||||
};
|
||||
return <StateContext.Provider value={value}>{children}</StateContext.Provider>;
|
||||
};
|
||||
|
||||
export default React.memo(Provider);
|
||||
|
||||
export const PreviewFileOrChunk = () => {
|
||||
const theme = useTheme();
|
||||
const { setFiles, previewFile, setPreviewFile, setReShowRePreview, totalChunks, files } =
|
||||
useImportStore();
|
||||
|
||||
return (
|
||||
<Box h={'100%'} w={'100%'}>
|
||||
{!!previewFile ? (
|
||||
<Box
|
||||
position={'relative'}
|
||||
display={['block', 'flex']}
|
||||
h={'100%'}
|
||||
flexDirection={'column'}
|
||||
pt={[3, 6]}
|
||||
bg={'myWhite.400'}
|
||||
>
|
||||
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'} {...filenameStyles}>
|
||||
{previewFile.filename}
|
||||
</Box>
|
||||
<CloseIcon
|
||||
position={'absolute'}
|
||||
right={[4, 8]}
|
||||
top={4}
|
||||
cursor={'pointer'}
|
||||
onClick={() => setPreviewFile(undefined)}
|
||||
/>
|
||||
<Box
|
||||
flex={'1 0 0'}
|
||||
h={['auto', 0]}
|
||||
overflow={'overlay'}
|
||||
px={[4, 8]}
|
||||
my={4}
|
||||
contentEditable
|
||||
dangerouslySetInnerHTML={{ __html: previewFile.text }}
|
||||
fontSize={'sm'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
wordBreak={'break-all'}
|
||||
onBlur={(e) => {
|
||||
// @ts-ignore
|
||||
const val = e.target.innerText;
|
||||
setReShowRePreview(true);
|
||||
|
||||
setFiles((state) =>
|
||||
state.map((file) =>
|
||||
file.id === previewFile.id
|
||||
? {
|
||||
...file,
|
||||
text: val
|
||||
}
|
||||
: file
|
||||
)
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
) : (
|
||||
<Box pt={[3, 6]}>
|
||||
<Flex px={[4, 8]} alignItems={'center'}>
|
||||
<Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
|
||||
分段预览({totalChunks}组)
|
||||
</Box>
|
||||
{totalChunks > 50 && (
|
||||
<Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
|
||||
仅展示部分
|
||||
</Box>
|
||||
)}
|
||||
</Flex>
|
||||
<Box px={[4, 8]} overflow={'overlay'}>
|
||||
{files.map((file) =>
|
||||
file.chunks.slice(0, 50).map((chunk, i) => (
|
||||
<Box
|
||||
key={i}
|
||||
py={4}
|
||||
bg={'myWhite.500'}
|
||||
my={2}
|
||||
borderRadius={'md'}
|
||||
fontSize={'sm'}
|
||||
_hover={{ ...hoverDeleteStyles }}
|
||||
>
|
||||
<Flex mb={1} px={4} userSelect={'none'}>
|
||||
<Box
|
||||
flexShrink={0}
|
||||
px={3}
|
||||
py={'1px'}
|
||||
border={theme.borders.base}
|
||||
borderRadius={'md'}
|
||||
>
|
||||
# {i + 1}
|
||||
</Box>
|
||||
<Box ml={2} fontSize={'sm'} color={'myhGray.500'} {...filenameStyles}>
|
||||
{file.filename}
|
||||
</Box>
|
||||
<Box flex={1} />
|
||||
<DeleteIcon
|
||||
onClick={() => {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
<Box
|
||||
px={4}
|
||||
fontSize={'sm'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
wordBreak={'break-all'}
|
||||
contentEditable={!chunk.a}
|
||||
dangerouslySetInnerHTML={{
|
||||
__html: chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q
|
||||
}}
|
||||
onBlur={(e) => {
|
||||
// @ts-ignore
|
||||
const val = e.target.innerText;
|
||||
|
||||
/* delete file */
|
||||
if (val === '') {
|
||||
setFiles((state) =>
|
||||
state.map((stateFile) =>
|
||||
stateFile.id === file.id
|
||||
? {
|
||||
...file,
|
||||
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
} else {
|
||||
// update chunk
|
||||
setFiles((stateFiles) =>
|
||||
stateFiles.map((stateFile) =>
|
||||
file.id === stateFile.id
|
||||
? {
|
||||
...stateFile,
|
||||
chunks: stateFile.chunks.map((chunk, index) => ({
|
||||
...chunk,
|
||||
index: i === index ? val : chunk.q
|
||||
}))
|
||||
}
|
||||
: stateFile
|
||||
)
|
||||
);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
))
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export const SelectorContainer = ({
|
||||
fileExtension,
|
||||
showUrlFetch,
|
||||
showCreateFile,
|
||||
children
|
||||
}: {
|
||||
fileExtension: string;
|
||||
showUrlFetch?: boolean;
|
||||
showCreateFile?: boolean;
|
||||
children: React.ReactNode;
|
||||
}) => {
|
||||
const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen } = useImportStore();
|
||||
return (
|
||||
<Box
|
||||
h={'100%'}
|
||||
overflowY={'auto'}
|
||||
flex={['auto', '1 0 400px']}
|
||||
{...(isUnselectedFile
|
||||
? {}
|
||||
: {
|
||||
maxW: ['auto', '500px']
|
||||
})}
|
||||
p={[4, 8]}
|
||||
>
|
||||
<FileSelect
|
||||
fileExtension={fileExtension}
|
||||
onPushFiles={(files) => {
|
||||
setFiles((state) => files.concat(state));
|
||||
}}
|
||||
chunkLen={chunkLen}
|
||||
showUrlFetch={showUrlFetch}
|
||||
showCreateFile={showCreateFile}
|
||||
py={isUnselectedFile ? '100px' : 5}
|
||||
/>
|
||||
{!isUnselectedFile && (
|
||||
<Box py={4} px={2} maxH={'400px'} overflowY={'auto'}>
|
||||
{files.map((item) => (
|
||||
<Flex
|
||||
key={item.id}
|
||||
w={'100%'}
|
||||
_notLast={{ mb: 5 }}
|
||||
px={5}
|
||||
py={2}
|
||||
boxShadow={'1px 1px 5px rgba(0,0,0,0.15)'}
|
||||
borderRadius={'md'}
|
||||
cursor={'pointer'}
|
||||
position={'relative'}
|
||||
alignItems={'center'}
|
||||
_hover={{
|
||||
bg: 'myBlue.100',
|
||||
'& .delete': {
|
||||
display: 'block'
|
||||
}
|
||||
}}
|
||||
onClick={() => setPreviewFile(item)}
|
||||
>
|
||||
<Image src={item.icon} w={'16px'} alt={''} />
|
||||
<Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
|
||||
{item.filename}
|
||||
</Box>
|
||||
<MyIcon
|
||||
position={'absolute'}
|
||||
right={3}
|
||||
className="delete"
|
||||
name={'delete'}
|
||||
w={'16px'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
display={['block', 'none']}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
))}
|
||||
</Box>
|
||||
)}
|
||||
{!isUnselectedFile && <>{children}</>}
|
||||
</Box>
|
||||
);
|
||||
};
|
100
projects/app/src/pages/dataset/detail/components/Import/QA.tsx
Normal file
100
projects/app/src/pages/dataset/detail/components/Import/QA.tsx
Normal file
@@ -0,0 +1,100 @@
|
||||
import React, { useState, useMemo } from 'react';
|
||||
import { Box, Flex, Button, Input } from '@chakra-ui/react';
|
||||
import { useConfirm } from '@/web/common/hooks/useConfirm';
|
||||
import { formatPrice } from '@fastgpt/global/common/bill/tools';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons';
|
||||
import { Prompt_AgentQA } from '@/global/core/prompt/agent';
|
||||
import { replaceVariable } from '@/global/common/string/tools';
|
||||
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
|
||||
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
|
||||
|
||||
const QAImport = () => {
|
||||
const { datasetDetail } = useDatasetStore();
|
||||
const vectorModel = datasetDetail.vectorModel;
|
||||
const unitPrice = vectorModel?.price || 0.2;
|
||||
|
||||
const {
|
||||
successChunks,
|
||||
totalChunks,
|
||||
isUnselectedFile,
|
||||
price,
|
||||
onclickUpload,
|
||||
onReSplitChunks,
|
||||
uploading,
|
||||
showRePreview
|
||||
} = useImportStore();
|
||||
|
||||
const { openConfirm, ConfirmModal } = useConfirm({
|
||||
content: `该任务无法终止!导入后会自动调用大模型生成问答对,会有一些细节丢失,请确认!如果余额不足,未完成的任务会被暂停。`
|
||||
});
|
||||
|
||||
const [prompt, setPrompt] = useState('');
|
||||
|
||||
const previewQAPrompt = useMemo(() => {
|
||||
return replaceVariable(Prompt_AgentQA.prompt, {
|
||||
theme: prompt || Prompt_AgentQA.defaultTheme
|
||||
});
|
||||
}, [prompt]);
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} h={['auto', '100%']}>
|
||||
<SelectorContainer fileExtension={fileExtension}>
|
||||
{/* prompt */}
|
||||
<Box py={5}>
|
||||
<Box mb={2}>
|
||||
QA 拆分引导词{' '}
|
||||
<MyTooltip label={previewQAPrompt} forceShow>
|
||||
<InfoOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Flex alignItems={'center'} fontSize={'sm'}>
|
||||
<Box mr={2}>文件主题</Box>
|
||||
<Input
|
||||
fontSize={'sm'}
|
||||
flex={1}
|
||||
placeholder={Prompt_AgentQA.defaultTheme}
|
||||
bg={'myWhite.500'}
|
||||
defaultValue={prompt}
|
||||
onChange={(e) => setPrompt(e.target.value || '')}
|
||||
/>
|
||||
</Flex>
|
||||
</Box>
|
||||
{/* price */}
|
||||
<Flex py={5} alignItems={'center'}>
|
||||
<Box>
|
||||
预估价格
|
||||
<MyTooltip
|
||||
label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}
|
||||
forceShow
|
||||
>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Box ml={4}>{price}元</Box>
|
||||
</Flex>
|
||||
<Flex mt={3}>
|
||||
{showRePreview && (
|
||||
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
|
||||
重新生成预览
|
||||
</Button>
|
||||
)}
|
||||
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
|
||||
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
|
||||
</Button>
|
||||
</Flex>
|
||||
</SelectorContainer>
|
||||
|
||||
{!isUnselectedFile && (
|
||||
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
|
||||
<PreviewFileOrChunk />
|
||||
</Box>
|
||||
)}
|
||||
<ConfirmModal />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default QAImport;
|
@@ -0,0 +1,67 @@
|
||||
import React, { useRef } from 'react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import MyModal from '@/components/MyModal';
|
||||
import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
|
||||
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
|
||||
import { useRequest } from '@/web/common/hooks/useRequest';
|
||||
import { postFetchUrls } from '@/web/common/plugin/api';
|
||||
|
||||
const UrlFetchModal = ({
|
||||
onClose,
|
||||
onSuccess
|
||||
}: {
|
||||
onClose: () => void;
|
||||
onSuccess: (e: FetchResultItem[]) => void;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const Dom = useRef<HTMLTextAreaElement>(null);
|
||||
|
||||
const { mutate, isLoading } = useRequest({
|
||||
mutationFn: async () => {
|
||||
const val = Dom.current?.value || '';
|
||||
const urls = val.split('\n').filter((e) => e);
|
||||
const res = await postFetchUrls(urls);
|
||||
|
||||
onSuccess(res);
|
||||
onClose();
|
||||
},
|
||||
errorToast: '获取链接失败'
|
||||
});
|
||||
|
||||
return (
|
||||
<MyModal
|
||||
title={
|
||||
<>
|
||||
<Box>{t('file.Fetch Url')}</Box>
|
||||
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
|
||||
目前仅支持读取静态链接,请注意检查结果
|
||||
</Box>
|
||||
</>
|
||||
}
|
||||
top={'15vh'}
|
||||
isOpen
|
||||
onClose={onClose}
|
||||
w={'600px'}
|
||||
>
|
||||
<ModalBody>
|
||||
<Textarea
|
||||
ref={Dom}
|
||||
rows={12}
|
||||
whiteSpace={'nowrap'}
|
||||
resize={'both'}
|
||||
placeholder={'最多10个链接,每行一个。'}
|
||||
/>
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button variant={'base'} mr={4} onClick={onClose}>
|
||||
取消
|
||||
</Button>
|
||||
<Button isLoading={isLoading} onClick={mutate}>
|
||||
确认
|
||||
</Button>
|
||||
</ModalFooter>
|
||||
</MyModal>
|
||||
);
|
||||
};
|
||||
|
||||
export default UrlFetchModal;
|
251
projects/app/src/pages/dataset/detail/components/Info.tsx
Normal file
251
projects/app/src/pages/dataset/detail/components/Info.tsx
Normal file
@@ -0,0 +1,251 @@
|
||||
import React, {
|
||||
useCallback,
|
||||
useState,
|
||||
useRef,
|
||||
forwardRef,
|
||||
useImperativeHandle,
|
||||
ForwardedRef
|
||||
} from 'react';
|
||||
import { useRouter } from 'next/router';
|
||||
import { Box, Flex, Button, FormControl, IconButton, Input } from '@chakra-ui/react';
|
||||
import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
|
||||
import { delDatasetById, putDatasetById } from '@/web/core/dataset/api';
|
||||
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
import { useConfirm } from '@/web/common/hooks/useConfirm';
|
||||
import { UseFormReturn } from 'react-hook-form';
|
||||
import { compressImg } from '@/web/common/file/utils';
|
||||
import type { DatasetItemType } from '@/types/core/dataset';
|
||||
import Avatar from '@/components/Avatar';
|
||||
import Tag from '@/components/Tag';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
|
||||
export interface ComponentRef {
|
||||
initInput: (tags: string) => void;
|
||||
}
|
||||
|
||||
const Info = (
|
||||
{ datasetId, form }: { datasetId: string; form: UseFormReturn<DatasetItemType, any> },
|
||||
ref: ForwardedRef<ComponentRef>
|
||||
) => {
|
||||
const { getValues, formState, setValue, register, handleSubmit } = form;
|
||||
const InputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const { toast } = useToast();
|
||||
const router = useRouter();
|
||||
|
||||
const [btnLoading, setBtnLoading] = useState(false);
|
||||
const [refresh, setRefresh] = useState(false);
|
||||
|
||||
const { openConfirm, ConfirmModal } = useConfirm({
|
||||
content: '确认删除该知识库?数据将无法恢复,请确认!'
|
||||
});
|
||||
|
||||
const { File, onOpen: onOpenSelectFile } = useSelectFile({
|
||||
fileType: '.jpg,.png',
|
||||
multiple: false
|
||||
});
|
||||
|
||||
const { datasetDetail, loadDatasetDetail, loadDatasets } = useDatasetStore();
|
||||
|
||||
/* 点击删除 */
|
||||
const onclickDelKb = useCallback(async () => {
|
||||
setBtnLoading(true);
|
||||
try {
|
||||
await delDatasetById(datasetId);
|
||||
toast({
|
||||
title: '删除成功',
|
||||
status: 'success'
|
||||
});
|
||||
router.replace(`/dataset/list`);
|
||||
await loadDatasets();
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: err?.message || '删除失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setBtnLoading(false);
|
||||
}, [setBtnLoading, datasetId, toast, router, loadDatasets]);
|
||||
|
||||
const saveSubmitSuccess = useCallback(
|
||||
async (data: DatasetItemType) => {
|
||||
setBtnLoading(true);
|
||||
try {
|
||||
await putDatasetById({
|
||||
id: datasetId,
|
||||
...data
|
||||
});
|
||||
await loadDatasetDetail(datasetId, true);
|
||||
toast({
|
||||
title: '更新成功',
|
||||
status: 'success'
|
||||
});
|
||||
loadDatasets();
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: err?.message || '更新失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setBtnLoading(false);
|
||||
},
|
||||
[loadDatasetDetail, datasetId, loadDatasets, toast]
|
||||
);
|
||||
const saveSubmitError = useCallback(() => {
|
||||
// deep search message
|
||||
const deepSearch = (obj: any): string => {
|
||||
if (!obj) return '提交表单错误';
|
||||
if (!!obj.message) {
|
||||
return obj.message;
|
||||
}
|
||||
return deepSearch(Object.values(obj)[0]);
|
||||
};
|
||||
toast({
|
||||
title: deepSearch(formState.errors),
|
||||
status: 'error',
|
||||
duration: 4000,
|
||||
isClosable: true
|
||||
});
|
||||
}, [formState.errors, toast]);
|
||||
|
||||
const onSelectFile = useCallback(
|
||||
async (e: File[]) => {
|
||||
const file = e[0];
|
||||
if (!file) return;
|
||||
try {
|
||||
const src = await compressImg({
|
||||
file,
|
||||
maxW: 100,
|
||||
maxH: 100
|
||||
});
|
||||
|
||||
setValue('avatar', src);
|
||||
|
||||
setRefresh((state) => !state);
|
||||
} catch (err: any) {
|
||||
toast({
|
||||
title: typeof err === 'string' ? err : '头像选择异常',
|
||||
status: 'warning'
|
||||
});
|
||||
}
|
||||
},
|
||||
[setRefresh, setValue, toast]
|
||||
);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
initInput: (tags: string) => {
|
||||
if (InputRef.current) {
|
||||
InputRef.current.value = tags;
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
return (
|
||||
<Box py={5} px={[5, 10]}>
|
||||
<Flex mt={5} w={'100%'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
知识库 ID
|
||||
</Box>
|
||||
<Box flex={1}>{datasetDetail._id}</Box>
|
||||
</Flex>
|
||||
<Flex mt={8} w={'100%'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
索引模型
|
||||
</Box>
|
||||
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
|
||||
</Flex>
|
||||
<Flex mt={8} w={'100%'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
MaxTokens
|
||||
</Box>
|
||||
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
|
||||
</Flex>
|
||||
<Flex mt={5} w={'100%'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
知识库头像
|
||||
</Box>
|
||||
<Box flex={[1, '0 0 300px']}>
|
||||
<MyTooltip label={'点击切换头像'}>
|
||||
<Avatar
|
||||
m={'auto'}
|
||||
src={getValues('avatar')}
|
||||
w={['32px', '40px']}
|
||||
h={['32px', '40px']}
|
||||
cursor={'pointer'}
|
||||
onClick={onOpenSelectFile}
|
||||
/>
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
</Flex>
|
||||
<FormControl mt={8} w={'100%'} display={'flex'} alignItems={'center'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
知识库名称
|
||||
</Box>
|
||||
<Input
|
||||
flex={[1, '0 0 300px']}
|
||||
{...register('name', {
|
||||
required: '知识库名称不能为空'
|
||||
})}
|
||||
/>
|
||||
</FormControl>
|
||||
<Flex mt={8} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
|
||||
标签
|
||||
<MyTooltip label={'用空格隔开多个标签,便于搜索'} forceShow>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
<Input
|
||||
flex={[1, '0 0 300px']}
|
||||
ref={InputRef}
|
||||
defaultValue={getValues('tags')}
|
||||
placeholder={'标签,使用空格分割。'}
|
||||
maxLength={30}
|
||||
onChange={(e) => {
|
||||
setValue('tags', e.target.value);
|
||||
setRefresh(!refresh);
|
||||
}}
|
||||
/>
|
||||
<Flex w={'100%'} pl={['90px', '160px']} mt={2}>
|
||||
{getValues('tags')
|
||||
.split(' ')
|
||||
.filter((item) => item)
|
||||
.map((item, i) => (
|
||||
<Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
|
||||
{item}
|
||||
</Tag>
|
||||
))}
|
||||
</Flex>
|
||||
</Flex>
|
||||
<Flex mt={5} w={'100%'} alignItems={'flex-end'}>
|
||||
<Box flex={['0 0 90px', '0 0 160px']} w={0}></Box>
|
||||
<Button
|
||||
isLoading={btnLoading}
|
||||
mr={4}
|
||||
w={'100px'}
|
||||
onClick={handleSubmit(saveSubmitSuccess, saveSubmitError)}
|
||||
>
|
||||
保存
|
||||
</Button>
|
||||
<IconButton
|
||||
isLoading={btnLoading}
|
||||
icon={<DeleteIcon />}
|
||||
aria-label={''}
|
||||
variant={'outline'}
|
||||
size={'sm'}
|
||||
_hover={{
|
||||
color: 'red.600',
|
||||
borderColor: 'red.600'
|
||||
}}
|
||||
onClick={openConfirm(onclickDelKb)}
|
||||
/>
|
||||
</Flex>
|
||||
<File onSelect={onSelectFile} />
|
||||
<ConfirmModal />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default forwardRef(Info);
|
@@ -0,0 +1,299 @@
|
||||
import React, { useMemo } from 'react';
|
||||
import { Box, Flex, Button, Textarea, IconButton, BoxProps } from '@chakra-ui/react';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import {
|
||||
postData2Dataset,
|
||||
putDatasetDataById,
|
||||
delOneDatasetDataById
|
||||
} from '@/web/core/dataset/api';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import MyModal from '@/components/MyModal';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
import { getFileAndOpen } from '@/web/common/file/utils';
|
||||
import { strIsLink } from '@fastgpt/global/common/string/tools';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq';
|
||||
import { useRequest } from '@/web/common/hooks/useRequest';
|
||||
import { countPromptTokens } from '@/global/common/tiktoken';
|
||||
import { useConfirm } from '@/web/common/hooks/useConfirm';
|
||||
|
||||
export type RawSourceType = {
|
||||
sourceName?: string;
|
||||
sourceId?: string;
|
||||
};
|
||||
export type RawSourceTextProps = BoxProps & RawSourceType;
|
||||
export type InputDataType = SetOneDatasetDataProps & RawSourceType;
|
||||
|
||||
const InputDataModal = ({
|
||||
onClose,
|
||||
onSuccess,
|
||||
onDelete,
|
||||
datasetId,
|
||||
defaultValues = {
|
||||
datasetId: '',
|
||||
collectionId: '',
|
||||
sourceId: '',
|
||||
sourceName: ''
|
||||
}
|
||||
}: {
|
||||
onClose: () => void;
|
||||
onSuccess: (data: SetOneDatasetDataProps) => void;
|
||||
onDelete?: () => void;
|
||||
datasetId: string;
|
||||
defaultValues: InputDataType;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const { toast } = useToast();
|
||||
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
|
||||
|
||||
const { register, handleSubmit, reset } = useForm<InputDataType>({
|
||||
defaultValues
|
||||
});
|
||||
|
||||
const { ConfirmModal, openConfirm } = useConfirm({
|
||||
content: t('dataset.data.Delete Tip')
|
||||
});
|
||||
|
||||
const maxToken = datasetDetail.vectorModel?.maxToken || 2000;
|
||||
|
||||
/**
|
||||
* 确认导入新数据
|
||||
*/
|
||||
const { mutate: sureImportData, isLoading: isImporting } = useRequest({
|
||||
mutationFn: async (e: InputDataType) => {
|
||||
if (!e.q) {
|
||||
return toast({
|
||||
title: '匹配的知识点不能为空',
|
||||
status: 'warning'
|
||||
});
|
||||
}
|
||||
if (countPromptTokens(e.q) >= maxToken) {
|
||||
return toast({
|
||||
title: '总长度超长了',
|
||||
status: 'warning'
|
||||
});
|
||||
}
|
||||
|
||||
const data = { ...e };
|
||||
delete data.sourceName;
|
||||
delete data.sourceId;
|
||||
|
||||
data.id = await postData2Dataset(data);
|
||||
|
||||
return data;
|
||||
},
|
||||
successToast: t('dataset.data.Input Success Tip'),
|
||||
onSuccess(e) {
|
||||
reset({
|
||||
...e,
|
||||
q: '',
|
||||
a: ''
|
||||
});
|
||||
|
||||
onSuccess(e);
|
||||
},
|
||||
errorToast: t('common.error.unKnow')
|
||||
});
|
||||
|
||||
const { mutate: onUpdateData, isLoading: isUpdating } = useRequest({
|
||||
mutationFn: async (e: SetOneDatasetDataProps) => {
|
||||
if (!e.id) return e;
|
||||
|
||||
// not exactly same
|
||||
if (e.q !== defaultValues.q || e.a !== defaultValues.a) {
|
||||
await putDatasetDataById({
|
||||
...e,
|
||||
q: e.q === defaultValues.q ? '' : e.q
|
||||
});
|
||||
return e;
|
||||
}
|
||||
|
||||
return e;
|
||||
},
|
||||
successToast: t('dataset.data.Update Success Tip'),
|
||||
errorToast: t('common.error.unKnow'),
|
||||
onSuccess(data) {
|
||||
onSuccess(data);
|
||||
onClose();
|
||||
}
|
||||
});
|
||||
|
||||
const loading = useMemo(() => isImporting || isUpdating, [isImporting, isUpdating]);
|
||||
|
||||
useQuery(['loadDatasetDetail'], () => {
|
||||
if (datasetDetail._id === datasetId) return null;
|
||||
return loadDatasetDetail(datasetId);
|
||||
});
|
||||
|
||||
return (
|
||||
<MyModal
|
||||
isOpen={true}
|
||||
isCentered
|
||||
title={defaultValues.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')}
|
||||
w={'90vw'}
|
||||
maxW={'90vw'}
|
||||
h={'90vh'}
|
||||
>
|
||||
<Flex flexDirection={'column'} h={'100%'}>
|
||||
<Box
|
||||
display={'flex'}
|
||||
flexDirection={['column', 'row']}
|
||||
flex={'1 0 0'}
|
||||
h={['100%', 0]}
|
||||
overflow={'overlay'}
|
||||
px={6}
|
||||
pb={2}
|
||||
>
|
||||
<Box flex={1} mr={[0, 4]} mb={[4, 0]} h={['50%', '100%']}>
|
||||
<Flex>
|
||||
<Box h={'30px'}>{'匹配的知识点'}</Box>
|
||||
<MyTooltip label={'被向量化的部分,通常是问题,也可以是一段陈述描述'}>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Flex>
|
||||
<Textarea
|
||||
placeholder={`匹配的知识点。这部分内容会被搜索,请把控内容的质量,最多 ${maxToken} 字。`}
|
||||
maxLength={maxToken}
|
||||
resize={'none'}
|
||||
h={'calc(100% - 30px)'}
|
||||
{...register(`q`, {
|
||||
required: true
|
||||
})}
|
||||
/>
|
||||
</Box>
|
||||
<Box flex={1} h={['50%', '100%']}>
|
||||
<Flex>
|
||||
<Box h={'30px'}>{'补充内容'}</Box>
|
||||
<MyTooltip
|
||||
label={'匹配的知识点被命中后,这部分内容会随匹配知识点一起注入模型,引导模型回答'}
|
||||
>
|
||||
<QuestionOutlineIcon ml={1} />
|
||||
</MyTooltip>
|
||||
</Flex>
|
||||
<Textarea
|
||||
placeholder={
|
||||
'这部分内容不会被搜索,但会作为"匹配的知识点"的内容补充,通常是问题的答案。'
|
||||
}
|
||||
resize={'none'}
|
||||
h={'calc(100% - 30px)'}
|
||||
{...register('a')}
|
||||
/>
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
<Flex px={6} pt={['34px', 2]} pb={4} alignItems={'center'} position={'relative'}>
|
||||
<RawSourceText
|
||||
sourceName={defaultValues.sourceName}
|
||||
sourceId={defaultValues.sourceId}
|
||||
position={'absolute'}
|
||||
left={'50%'}
|
||||
top={['16px', '50%']}
|
||||
transform={'translate(-50%,-50%)'}
|
||||
/>
|
||||
|
||||
<Box flex={1}>
|
||||
{defaultValues.id && onDelete && (
|
||||
<IconButton
|
||||
variant={'outline'}
|
||||
icon={<MyIcon name={'delete'} w={'16px'} h={'16px'} />}
|
||||
aria-label={''}
|
||||
isLoading={loading}
|
||||
size={'sm'}
|
||||
_hover={{
|
||||
color: 'red.600',
|
||||
borderColor: 'red.600'
|
||||
}}
|
||||
onClick={openConfirm(async () => {
|
||||
if (!onDelete || !defaultValues.id) return;
|
||||
try {
|
||||
await delOneDatasetDataById(defaultValues.id);
|
||||
onDelete();
|
||||
onClose();
|
||||
toast({
|
||||
status: 'success',
|
||||
title: '记录已删除'
|
||||
});
|
||||
} catch (error) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: getErrText(error)
|
||||
});
|
||||
console.log(error);
|
||||
}
|
||||
})}
|
||||
/>
|
||||
)}
|
||||
</Box>
|
||||
<Box>
|
||||
<Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
|
||||
{t('common.Close')}
|
||||
</Button>
|
||||
<Button
|
||||
isLoading={loading}
|
||||
// @ts-ignore
|
||||
onClick={handleSubmit(defaultValues.id ? onUpdateData : sureImportData)}
|
||||
>
|
||||
{defaultValues.id ? '确认变更' : '确认导入'}
|
||||
</Button>
|
||||
</Box>
|
||||
</Flex>
|
||||
</Flex>
|
||||
<ConfirmModal />
|
||||
</MyModal>
|
||||
);
|
||||
};
|
||||
|
||||
export default InputDataModal;
|
||||
|
||||
export function RawSourceText({ sourceId, sourceName = '', ...props }: RawSourceTextProps) {
|
||||
const { t } = useTranslation();
|
||||
const { toast } = useToast();
|
||||
const { setLoading } = useSystemStore();
|
||||
|
||||
const canPreview = useMemo(() => !!sourceId, [sourceId]);
|
||||
|
||||
return (
|
||||
<MyTooltip
|
||||
label={sourceId ? t('file.Click to view file') || '' : ''}
|
||||
shouldWrapChildren={false}
|
||||
>
|
||||
<Box
|
||||
color={'myGray.600'}
|
||||
display={'inline-block'}
|
||||
whiteSpace={'nowrap'}
|
||||
maxW={['200px', '300px']}
|
||||
className={'textEllipsis'}
|
||||
{...(canPreview
|
||||
? {
|
||||
cursor: 'pointer',
|
||||
textDecoration: 'underline',
|
||||
onClick: async () => {
|
||||
if (strIsLink(sourceId)) {
|
||||
return window.open(sourceId, '_blank');
|
||||
}
|
||||
setLoading(true);
|
||||
try {
|
||||
await getFileAndOpen(sourceId as string);
|
||||
} catch (error) {
|
||||
toast({
|
||||
title: getErrText(error, '获取文件地址失败'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
: {})}
|
||||
{...props}
|
||||
>
|
||||
{sourceName || t('common.Unknow Source')}
|
||||
</Box>
|
||||
</MyTooltip>
|
||||
);
|
||||
}
|
296
projects/app/src/pages/dataset/detail/components/Test.tsx
Normal file
296
projects/app/src/pages/dataset/detail/components/Test.tsx
Normal file
@@ -0,0 +1,296 @@
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
import { useSearchTestStore, SearchTestStoreItemType } from '@/web/core/dataset/store/searchTest';
|
||||
import { getDatasetDataItemById, postSearchText } from '@/web/core/dataset/api';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import { useRequest } from '@/web/common/hooks/useRequest';
|
||||
import { formatTimeToChatTime } from '@/utils/tools';
|
||||
import InputDataModal, { type InputDataType } from './InputDataModal';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
|
||||
const Test = ({ datasetId }: { datasetId: string }) => {
|
||||
const theme = useTheme();
|
||||
const { toast } = useToast();
|
||||
const { setLoading } = useSystemStore();
|
||||
const { datasetDetail } = useDatasetStore();
|
||||
const { datasetTestList, pushDatasetTestItem, delDatasetTestItemById, updateDatasetItemById } =
|
||||
useSearchTestStore();
|
||||
const [inputText, setInputText] = useState('');
|
||||
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
|
||||
const [editInputData, setEditInputData] = useState<InputDataType>();
|
||||
|
||||
const kbTestHistory = useMemo(
|
||||
() => datasetTestList.filter((item) => item.datasetId === datasetId),
|
||||
[datasetId, datasetTestList]
|
||||
);
|
||||
|
||||
const { mutate, isLoading } = useRequest({
|
||||
mutationFn: () => postSearchText({ datasetId, text: inputText.trim() }),
|
||||
onSuccess(res) {
|
||||
const testItem = {
|
||||
id: nanoid(),
|
||||
datasetId,
|
||||
text: inputText.trim(),
|
||||
time: new Date(),
|
||||
results: res
|
||||
};
|
||||
pushDatasetTestItem(testItem);
|
||||
setDatasetTestItem(testItem);
|
||||
},
|
||||
onError(err) {
|
||||
toast({
|
||||
title: getErrText(err),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
setDatasetTestItem(undefined);
|
||||
}, [datasetId]);
|
||||
|
||||
return (
|
||||
<Box h={'100%'} display={['block', 'flex']}>
|
||||
<Box
|
||||
h={['auto', '100%']}
|
||||
display={['block', 'flex']}
|
||||
flexDirection={'column'}
|
||||
flex={1}
|
||||
maxW={'500px'}
|
||||
py={4}
|
||||
borderRight={['none', theme.borders.base]}
|
||||
>
|
||||
<Box border={'2px solid'} borderColor={'myBlue.600'} p={3} mx={4} borderRadius={'md'}>
|
||||
<Box fontSize={'sm'} fontWeight={'bold'}>
|
||||
<MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'myBlue.700'} />
|
||||
测试文本
|
||||
</Box>
|
||||
<Textarea
|
||||
rows={6}
|
||||
resize={'none'}
|
||||
variant={'unstyled'}
|
||||
maxLength={datasetDetail.vectorModel.maxToken}
|
||||
placeholder="输入需要测试的文本"
|
||||
value={inputText}
|
||||
onChange={(e) => setInputText(e.target.value)}
|
||||
/>
|
||||
<Flex alignItems={'center'} justifyContent={'flex-end'}>
|
||||
<Box mr={3} color={'myGray.500'}>
|
||||
{inputText.length}
|
||||
</Box>
|
||||
<Button isDisabled={inputText === ''} isLoading={isLoading} onClick={mutate}>
|
||||
测试
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
<Box mt={5} flex={'1 0 0'} px={4} overflow={'overlay'} display={['none', 'block']}>
|
||||
<Flex alignItems={'center'} color={'myGray.600'}>
|
||||
<MyIcon mr={2} name={'history'} w={'16px'} h={'16px'} />
|
||||
<Box fontSize={'2xl'}>测试历史</Box>
|
||||
</Flex>
|
||||
<Box mt={2}>
|
||||
<Flex py={2} fontWeight={'bold'} borderBottom={theme.borders.sm}>
|
||||
<Box flex={1}>测试文本</Box>
|
||||
<Box w={'80px'}>时间</Box>
|
||||
<Box w={'14px'}></Box>
|
||||
</Flex>
|
||||
{kbTestHistory.map((item) => (
|
||||
<Flex
|
||||
key={item.id}
|
||||
p={1}
|
||||
alignItems={'center'}
|
||||
borderBottom={theme.borders.base}
|
||||
_hover={{
|
||||
bg: '#f4f4f4',
|
||||
'& .delete': {
|
||||
display: 'block'
|
||||
}
|
||||
}}
|
||||
cursor={'pointer'}
|
||||
onClick={() => setDatasetTestItem(item)}
|
||||
>
|
||||
<Box flex={1} mr={2}>
|
||||
{item.text}
|
||||
</Box>
|
||||
<Box w={'80px'}>{formatTimeToChatTime(item.time)}</Box>
|
||||
<MyTooltip label={'删除该测试记录'}>
|
||||
<Box w={'14px'} h={'14px'}>
|
||||
<MyIcon
|
||||
className="delete"
|
||||
name={'delete'}
|
||||
w={'14px'}
|
||||
display={'none'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
delDatasetTestItemById(item.id);
|
||||
datasetTestItem?.id === item.id && setDatasetTestItem(undefined);
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
</MyTooltip>
|
||||
</Flex>
|
||||
))}
|
||||
</Box>
|
||||
</Box>
|
||||
</Box>
|
||||
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={1}>
|
||||
{!datasetTestItem?.results || datasetTestItem.results.length === 0 ? (
|
||||
<Flex
|
||||
mt={[10, 0]}
|
||||
h={'100%'}
|
||||
flexDirection={'column'}
|
||||
alignItems={'center'}
|
||||
justifyContent={'center'}
|
||||
>
|
||||
<MyIcon name={'empty'} color={'transparent'} w={'54px'} />
|
||||
<Box mt={3} color={'myGray.600'}>
|
||||
测试结果将在这里展示
|
||||
</Box>
|
||||
</Flex>
|
||||
) : (
|
||||
<>
|
||||
<Flex alignItems={'center'}>
|
||||
<Box fontSize={'3xl'} color={'myGray.600'}>
|
||||
测试结果
|
||||
</Box>
|
||||
<MyTooltip
|
||||
label={
|
||||
'根据知识库内容与测试文本的相似度进行排序,你可以根据测试结果调整对应的文本。\n注意:测试记录中的数据可能已经被修改过,点击某条测试数据后将展示最新的数据。'
|
||||
}
|
||||
forceShow
|
||||
>
|
||||
<QuestionOutlineIcon
|
||||
ml={2}
|
||||
color={'myGray.600'}
|
||||
cursor={'pointer'}
|
||||
fontSize={'lg'}
|
||||
/>
|
||||
</MyTooltip>
|
||||
</Flex>
|
||||
<Grid
|
||||
mt={1}
|
||||
gridTemplateColumns={[
|
||||
'repeat(1,1fr)',
|
||||
'repeat(1,1fr)',
|
||||
'repeat(1,1fr)',
|
||||
'repeat(1,1fr)',
|
||||
'repeat(2,1fr)'
|
||||
]}
|
||||
gridGap={4}
|
||||
>
|
||||
{datasetTestItem?.results.map((item) => (
|
||||
<Box
|
||||
key={item.id}
|
||||
pb={2}
|
||||
borderRadius={'sm'}
|
||||
border={theme.borders.base}
|
||||
_notLast={{ mb: 2 }}
|
||||
cursor={'pointer'}
|
||||
title={'编辑'}
|
||||
onClick={async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const data = await getDatasetDataItemById(item.id);
|
||||
|
||||
if (!data) {
|
||||
throw new Error('该数据已被删除');
|
||||
}
|
||||
|
||||
setEditInputData({
|
||||
id: data.id,
|
||||
datasetId: data.datasetId,
|
||||
collectionId: data.collectionId,
|
||||
q: data.q,
|
||||
a: data.a,
|
||||
sourceName: data.sourceName,
|
||||
sourceId: data.sourceId
|
||||
});
|
||||
} catch (err) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: getErrText(err)
|
||||
});
|
||||
}
|
||||
setLoading(false);
|
||||
}}
|
||||
>
|
||||
<Flex p={3} alignItems={'center'} color={'myGray.500'}>
|
||||
<MyIcon name={'kbTest'} w={'14px'} />
|
||||
<Progress
|
||||
mx={2}
|
||||
flex={1}
|
||||
value={item.score * 100}
|
||||
size="sm"
|
||||
borderRadius={'20px'}
|
||||
colorScheme="gray"
|
||||
/>
|
||||
<Box>{item.score.toFixed(4)}</Box>
|
||||
</Flex>
|
||||
<Box
|
||||
px={2}
|
||||
fontSize={'xs'}
|
||||
color={'myGray.600'}
|
||||
maxH={'200px'}
|
||||
overflow={'overlay'}
|
||||
>
|
||||
<Box>{item.q}</Box>
|
||||
<Box>{item.a}</Box>
|
||||
</Box>
|
||||
</Box>
|
||||
))}
|
||||
</Grid>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
{!!editInputData && (
|
||||
<InputDataModal
|
||||
datasetId={editInputData.datasetId}
|
||||
defaultValues={editInputData}
|
||||
onClose={() => setEditInputData(undefined)}
|
||||
onSuccess={(data) => {
|
||||
if (datasetTestItem && editInputData.id) {
|
||||
const newTestItem: SearchTestStoreItemType = {
|
||||
...datasetTestItem,
|
||||
results: datasetTestItem.results.map((item) =>
|
||||
item.id === editInputData.id
|
||||
? {
|
||||
...item,
|
||||
q: data.q || '',
|
||||
a: data.a || ''
|
||||
}
|
||||
: item
|
||||
)
|
||||
};
|
||||
updateDatasetItemById(newTestItem);
|
||||
setDatasetTestItem(newTestItem);
|
||||
}
|
||||
|
||||
setEditInputData(undefined);
|
||||
}}
|
||||
onDelete={() => {
|
||||
if (datasetTestItem && editInputData.id) {
|
||||
const newTestItem = {
|
||||
...datasetTestItem,
|
||||
results: datasetTestItem.results.filter((item) => item.id !== editInputData.id)
|
||||
};
|
||||
updateDatasetItemById(newTestItem);
|
||||
setDatasetTestItem(newTestItem);
|
||||
}
|
||||
setEditInputData(undefined);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default Test;
|
207
projects/app/src/pages/dataset/detail/index.tsx
Normal file
207
projects/app/src/pages/dataset/detail/index.tsx
Normal file
@@ -0,0 +1,207 @@
|
||||
import React, { useCallback, useEffect, useRef } from 'react';
|
||||
import { useRouter } from 'next/router';
|
||||
import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
|
||||
import { useToast } from '@/web/common/hooks/useToast';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { DatasetItemType } from '@/types/core/dataset';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import { type ComponentRef } from './components/Info';
|
||||
import Tabs from '@/components/Tabs';
|
||||
import dynamic from 'next/dynamic';
|
||||
import MyIcon from '@/components/Icon';
|
||||
import SideTabs from '@/components/SideTabs';
|
||||
import PageContainer from '@/components/PageContainer';
|
||||
import Avatar from '@/components/Avatar';
|
||||
import Info from './components/Info';
|
||||
import { serviceSideProps } from '@/web/common/utils/i18n';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { getTrainingQueueLen, delDatasetEmptyFiles } from '@/web/core/dataset/api';
|
||||
import MyTooltip from '@/components/MyTooltip';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
import { feConfigs } from '@/web/common/system/staticData';
|
||||
import Script from 'next/script';
|
||||
import CollectionCard from './components/CollectionCard';
|
||||
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
|
||||
|
||||
const DataCard = dynamic(() => import('./components/DataCard'), {
|
||||
ssr: false
|
||||
});
|
||||
const Test = dynamic(() => import('./components/Test'), {
|
||||
ssr: false
|
||||
});
|
||||
|
||||
export enum TabEnum {
|
||||
dataCard = 'dataCard',
|
||||
collectionCard = 'collectionCard',
|
||||
test = 'test',
|
||||
info = 'info'
|
||||
}
|
||||
|
||||
const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${TabEnum}` }) => {
|
||||
const InfoRef = useRef<ComponentRef>(null);
|
||||
const theme = useTheme();
|
||||
const { t } = useTranslation();
|
||||
const { toast } = useToast();
|
||||
const router = useRouter();
|
||||
const { isPc } = useSystemStore();
|
||||
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
|
||||
|
||||
const tabList = useRef([
|
||||
{ label: '数据集', id: TabEnum.collectionCard, icon: 'overviewLight' },
|
||||
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
|
||||
{ label: '配置', id: TabEnum.info, icon: 'settingLight' }
|
||||
]);
|
||||
|
||||
const setCurrentTab = useCallback(
|
||||
(tab: `${TabEnum}`) => {
|
||||
router.replace({
|
||||
query: {
|
||||
datasetId,
|
||||
currentTab: tab
|
||||
}
|
||||
});
|
||||
},
|
||||
[datasetId, router]
|
||||
);
|
||||
|
||||
const form = useForm<DatasetItemType>({
|
||||
defaultValues: datasetDetail
|
||||
});
|
||||
|
||||
useQuery([datasetId], () => loadDatasetDetail(datasetId), {
|
||||
onSuccess(res) {
|
||||
form.reset(res);
|
||||
InfoRef.current?.initInput(res.tags);
|
||||
},
|
||||
onError(err: any) {
|
||||
router.replace(`/dataset/list`);
|
||||
toast({
|
||||
title: getErrText(err, '获取知识库异常'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, {
|
||||
refetchInterval: 10000
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
try {
|
||||
delDatasetEmptyFiles(datasetId);
|
||||
} catch (error) {}
|
||||
};
|
||||
}, [datasetId]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Script src="/js/pdf.js" strategy="lazyOnload"></Script>
|
||||
<PageContainer>
|
||||
<Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
|
||||
{isPc ? (
|
||||
<Flex
|
||||
flexDirection={'column'}
|
||||
p={4}
|
||||
h={'100%'}
|
||||
flex={'0 0 200px'}
|
||||
borderRight={theme.borders.base}
|
||||
>
|
||||
<Flex mb={4} alignItems={'center'}>
|
||||
<Avatar src={datasetDetail.avatar} w={'34px'} borderRadius={'lg'} />
|
||||
<Box ml={2} fontWeight={'bold'}>
|
||||
{datasetDetail.name}
|
||||
</Box>
|
||||
</Flex>
|
||||
<SideTabs
|
||||
flex={1}
|
||||
mx={'auto'}
|
||||
mt={2}
|
||||
w={'100%'}
|
||||
list={tabList.current}
|
||||
activeId={currentTab}
|
||||
onChange={(e: any) => {
|
||||
setCurrentTab(e);
|
||||
}}
|
||||
/>
|
||||
<Box textAlign={'center'}>
|
||||
<Flex justifyContent={'center'} alignItems={'center'}>
|
||||
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
|
||||
<Box>{t('dataset.System Data Queue')}</Box>
|
||||
<MyTooltip
|
||||
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
|
||||
placement={'top'}
|
||||
>
|
||||
<QuestionOutlineIcon ml={1} w={'16px'} />
|
||||
</MyTooltip>
|
||||
</Flex>
|
||||
<Box mt={1} fontWeight={'bold'}>
|
||||
{trainingQueueLen}
|
||||
</Box>
|
||||
</Box>
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
cursor={'pointer'}
|
||||
py={2}
|
||||
px={3}
|
||||
borderRadius={'md'}
|
||||
_hover={{ bg: 'myGray.100' }}
|
||||
onClick={() => router.replace('/dataset/list')}
|
||||
>
|
||||
<IconButton
|
||||
mr={3}
|
||||
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
|
||||
bg={'white'}
|
||||
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
|
||||
h={'28px'}
|
||||
size={'sm'}
|
||||
borderRadius={'50%'}
|
||||
aria-label={''}
|
||||
/>
|
||||
全部知识库
|
||||
</Flex>
|
||||
</Flex>
|
||||
) : (
|
||||
<Box mb={3}>
|
||||
<Tabs
|
||||
m={'auto'}
|
||||
w={'260px'}
|
||||
size={isPc ? 'md' : 'sm'}
|
||||
list={tabList.current.map((item) => ({
|
||||
id: item.id,
|
||||
label: item.label
|
||||
}))}
|
||||
activeId={currentTab}
|
||||
onChange={(e: any) => setCurrentTab(e)}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{!!datasetDetail._id && (
|
||||
<Box flex={'1 0 0'} pb={[4, 0]}>
|
||||
{currentTab === TabEnum.collectionCard && <CollectionCard />}
|
||||
{currentTab === TabEnum.dataCard && <DataCard />}
|
||||
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
|
||||
{currentTab === TabEnum.info && (
|
||||
<Info ref={InfoRef} datasetId={datasetId} form={form} />
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
</Flex>
|
||||
</PageContainer>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export async function getServerSideProps(context: any) {
|
||||
const currentTab = context?.query?.currentTab || TabEnum.collectionCard;
|
||||
const datasetId = context?.query?.datasetId;
|
||||
|
||||
return {
|
||||
props: { currentTab, datasetId, ...(await serviceSideProps(context)) }
|
||||
};
|
||||
}
|
||||
|
||||
export default React.memo(Detail);
|
Reference in New Issue
Block a user