This commit is contained in:
Archer
2023-10-22 23:54:04 +08:00
committed by GitHub
parent 3091a90df6
commit a3534407bf
365 changed files with 7266 additions and 6055 deletions

View File

@@ -0,0 +1,601 @@
import React, { useCallback, useState, useRef, useMemo, useEffect } from 'react';
import {
Box,
Flex,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
Image,
MenuButton,
useTheme,
useDisclosure,
ModalFooter,
Button
} from '@chakra-ui/react';
import {
getDatasetCollections,
delDatasetCollectionById,
putDatasetCollectionById,
postDatasetCollection,
getDatasetCollectionPathById
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { debounce } from 'lodash';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { useTranslation } from 'react-i18next';
import MyIcon from '@/components/Icon';
import MyInput from '@/components/MyInput';
import dayjs from 'dayjs';
import { useRequest } from '@/web/common/hooks/useRequest';
import { useLoading } from '@/web/common/hooks/useLoading';
import { useRouter } from 'next/router';
import { usePagination } from '@/web/common/hooks/usePagination';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyMenu from '@/components/MyMenu';
import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
import EmptyTip from '@/components/EmptyTip';
import { AddIcon } from '@chakra-ui/icons';
import { FolderAvatarSrc, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
import { TabEnum } from '..';
import ParentPath from '@/components/common/ParentPaths';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import dynamic from 'next/dynamic';
import { useDrag } from '@/web/common/hooks/useDrag';
import SelectCollections from '@/web/core/dataset/components/SelectCollections';
import { useToast } from '@/web/common/hooks/useToast';
import MyTooltip from '@/components/MyTooltip';
const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
const CollectionCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
const theme = useTheme();
const lastSearch = useRef('');
const router = useRouter();
const { toast } = useToast();
const { parentId = '', datasetId } = router.query as { parentId: string; datasetId: string };
const { t } = useTranslation();
const { Loading } = useLoading();
const { isPc } = useSystemStore();
const [searchText, setSearchText] = useState('');
const { setLoading } = useSystemStore();
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('dataset.Confirm to delete the file')
});
const {
isOpen: isOpenFileImportModal,
onOpen: onOpenFileImportModal,
onClose: onCloseFileImportModal
} = useDisclosure();
const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
useEditTitle({
title: t('dataset.Create Virtual File'),
tip: t('dataset.Virtual File Tip')
});
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('Rename')
});
const { editFolderData, setEditFolderData } = useEditFolder();
const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
const {
data: collections,
Pagination,
total,
getData,
isLoading,
pageNum,
pageSize
} = usePagination<DatasetCollectionsListItemType>({
api: getDatasetCollections,
pageSize: 20,
params: {
datasetId,
parentId,
searchText
},
defaultRequest: false,
onChange() {
if (BoxRef.current) {
BoxRef.current.scrollTop = 0;
}
}
});
const { moveDataId, setMoveDataId, dragStartId, setDragStartId, dragTargetId, setDragTargetId } =
useDrag();
// change search
const debounceRefetch = useCallback(
debounce(() => {
getData(1);
lastSearch.current = searchText;
}, 300),
[]
);
// add file icon
const formatCollections = useMemo(
() =>
collections.map((collection) => {
const icon = getCollectionIcon(collection.type, collection.name);
return {
...collection,
icon,
...(collection.trainingAmount > 0
? {
statusText: t('dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
color: 'myGray.500'
}
: {
statusText: t('dataset.collections.Ready'),
color: 'green.500'
})
};
}),
[collections, t]
);
const hasTrainingData = useMemo(
() => !!formatCollections.find((item) => item.trainingAmount > 0),
[formatCollections]
);
const { mutate: onCreateVirtualFile } = useRequest({
mutationFn: ({ name }: { name: string }) => {
setLoading(true);
return postDatasetCollection({
parentId,
datasetId,
name,
type: DatasetCollectionTypeEnum.virtual
});
},
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('dataset.collections.Create Virtual File Success'),
errorToast: t('common.Create Virtual File Failed')
});
const { mutate: onUpdateCollectionName } = useRequest({
mutationFn: ({ collectionId, name }: { collectionId: string; name: string }) => {
return putDatasetCollectionById({
id: collectionId,
name
});
},
onSuccess() {
getData(pageNum);
},
successToast: t('common.Rename Success'),
errorToast: t('common.Rename Failed')
});
const { mutate: onDelCollection } = useRequest({
mutationFn: (collectionId: string) => {
setLoading(true);
return delDatasetCollectionById({
collectionId
});
},
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
getDatasetCollectionPathById(parentId)
);
useQuery(['loadDatasetDetail', datasetId], () => loadDatasetDetail(datasetId));
useQuery(
['refreshCollection'],
() => {
getData(1);
return null;
},
{
refetchInterval: 6000,
enabled: hasTrainingData
}
);
useEffect(() => {
getData(1);
}, [parentId]);
return (
<Box ref={BoxRef} py={[1, 3]} h={'100%'} overflow={'overlay'}>
<Flex px={[2, 5]} alignItems={['flex-start', 'center']}>
<Box flex={1}>
<ParentPath
paths={paths.map((path, i) => ({
parentId: path.parentId,
parentName: i === paths.length - 1 ? `${path.parentName}(${total})` : path.parentName
}))}
FirstPathDom={
<Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
{t('common.File')}({total})
</Box>
}
onClick={(e) => {
router.replace({
query: {
...router.query,
parentId: e
}
});
}}
/>
</Box>
{isPc && (
<Flex alignItems={'center'} mr={2}>
<MyInput
leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
}
w={['100%', '250px']}
size={['sm', 'md']}
placeholder={t('common.Search') || ''}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
debounceRefetch();
}}
onBlur={() => {
if (searchText === lastSearch.current) return;
getData(1);
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
getData(1);
}
}}
/>
</Flex>
)}
<MyMenu
offset={[-10, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
fontSize={['sm', 'md']}
>
<Flex
alignItems={'center'}
border={theme.borders.base}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
h={['28px', '35px']}
>
<AddIcon mr={2} />
<Box>{t('Create New')}</Box>
</Flex>
</MenuButton>
}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.Create Virtual File')}
</Flex>
),
onClick: () => {
onOpenCreateVirtualFileModal({
defaultVal: '',
onSuccess: (name) => onCreateVirtualFile({ name })
});
}
},
{
child: (
<Flex>
<Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
{t('dataset.File Input')}
</Flex>
),
onClick: onOpenFileImportModal
}
]}
/>
</Flex>
<TableContainer mt={[0, 3]} position={'relative'} minH={'50vh'}>
<Table variant={'simple'} fontSize={'sm'} draggable={false}>
<Thead draggable={false}>
<Tr>
<Th>{t('common.Name')}</Th>
<Th>{t('dataset.collections.Data Amount')}</Th>
<Th>{t('common.Time')}</Th>
<Th>{t('common.Status')}</Th>
<Th />
</Tr>
</Thead>
<Tbody>
{formatCollections.map((collection) => (
<Tr
key={collection._id}
_hover={{ bg: 'myWhite.600' }}
cursor={'pointer'}
data-drag-id={
collection.type === DatasetCollectionTypeEnum.folder ? collection._id : undefined
}
bg={dragTargetId === collection._id ? 'myBlue.200' : ''}
userSelect={'none'}
onDragStart={(e) => {
setDragStartId(collection._id);
}}
onDragOver={(e) => {
e.preventDefault();
const targetId = e.currentTarget.getAttribute('data-drag-id');
if (!targetId) return;
DatasetCollectionTypeEnum.folder && setDragTargetId(targetId);
}}
onDragLeave={(e) => {
e.preventDefault();
setDragTargetId(undefined);
}}
onDrop={async (e) => {
e.preventDefault();
if (!dragTargetId || !dragStartId || dragTargetId === dragStartId) return;
// update parentId
try {
await putDatasetCollectionById({
id: dragStartId,
parentId: dragTargetId
});
getData(pageNum);
} catch (error) {}
setDragTargetId(undefined);
}}
title={
collection.type === DatasetCollectionTypeEnum.folder
? t('dataset.collections.Click to view folder')
: t('dataset.collections.Click to view file')
}
onClick={() => {
if (collection.type === DatasetCollectionTypeEnum.folder) {
router.replace({
query: {
...router.query,
parentId: collection._id
}
});
} else {
router.replace({
query: {
...router.query,
collectionId: collection._id,
currentTab: TabEnum.dataCard
}
});
}
}}
>
<Td maxW={['200px', '300px']} draggable>
<Flex alignItems={'center'}>
<Image src={collection.icon} w={'16px'} mr={2} alt={''} />
<MyTooltip label={t('common.folder.Drag Tip')} shouldWrapChildren={false}>
<Box fontWeight={'bold'} className="textEllipsis">
{collection.name}
</Box>
</MyTooltip>
</Flex>
</Td>
<Td fontSize={'md'}>{collection.dataAmount ?? '-'}</Td>
<Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>
<Flex
alignItems={'center'}
_before={{
content: '""',
w: '10px',
h: '10px',
mr: 2,
borderRadius: 'lg',
bg: collection?.color
}}
>
{collection?.statusText}
</Flex>
</Td>
<Td onClick={(e) => e.stopPropagation()}>
<MyMenu
width={100}
Button={
<MenuButton
w={'22px'}
h={'22px'}
borderRadius={'md'}
_hover={{
color: 'myBlue.600',
'& .icon': {
bg: 'myGray.100'
}
}}
>
<MyIcon
className="icon"
name={'more'}
h={'16px'}
w={'16px'}
px={1}
py={1}
borderRadius={'md'}
cursor={'pointer'}
/>
</MenuButton>
}
menuList={[
{
child: (
<Flex alignItems={'center'}>
<MyIcon name={'moveLight'} w={'14px'} mr={2} />
{t('Move')}
</Flex>
),
onClick: () => setMoveCollectionData({ collectionId: collection._id })
},
{
child: (
<Flex alignItems={'center'}>
<MyIcon name={'edit'} w={'14px'} mr={2} />
{t('Rename')}
</Flex>
),
onClick: () =>
onOpenEditTitleModal({
defaultVal: collection.name,
onSuccess: (newName) => {
onUpdateCollectionName({
collectionId: collection._id,
name: newName
});
}
})
},
{
child: (
<Flex alignItems={'center'}>
<MyIcon
mr={1}
name={'delete'}
w={'14px'}
_hover={{ color: 'red.600' }}
/>
<Box>{t('common.Delete')}</Box>
</Flex>
),
onClick: () =>
openConfirm(
() => {
onDelCollection(collection._id);
},
undefined,
collection.type === DatasetCollectionTypeEnum.folder
? t('dataset.collections.Confirm to delete the folder')
: t('dataset.Confirm to delete the file')
)()
}
]}
/>
</Td>
</Tr>
))}
</Tbody>
</Table>
<Loading loading={isLoading && collections.length === 0} fixed={false} />
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
{total === 0 && <EmptyTip text="数据集空空如也" />}
</TableContainer>
<ConfirmModal />
<EditTitleModal />
<EditCreateVirtualFileModal />
{isOpenFileImportModal && (
<FileImportModal
datasetId={datasetId}
parentId={parentId}
uploadSuccess={() => {
getData(1);
onCloseFileImportModal();
}}
onClose={onCloseFileImportModal}
/>
)}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}
editCallback={async (name) => {
try {
if (editFolderData.id) {
await putDatasetCollectionById({
id: editFolderData.id,
name
});
} else {
await postDatasetCollection({
parentId,
datasetId,
name,
type: DatasetCollectionTypeEnum.folder
});
}
getData(pageNum);
} catch (error) {
return Promise.reject(error);
}
}}
isEdit={!!editFolderData.id}
name={editFolderData.name}
/>
)}
{!!moveCollectionData && (
<SelectCollections
datasetId={datasetId}
type="folder"
defaultSelectedId={[moveCollectionData.collectionId]}
onClose={() => setMoveCollectionData(undefined)}
onSuccess={async ({ parentId }) => {
await putDatasetCollectionById({
id: moveCollectionData.collectionId,
parentId
});
getData(pageNum);
setMoveCollectionData(undefined);
toast({
status: 'success',
title: t('common.folder.Move Success')
});
}}
/>
)}
</Box>
);
};
export default React.memo(CollectionCard);

View File

@@ -0,0 +1,267 @@
import React, { useCallback, useState, useRef, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Grid, Image, Button } from '@chakra-ui/react';
import { usePagination } from '@/web/common/hooks/usePagination';
import {
getDatasetDataList,
delOneDatasetDataById,
getDatasetCollectionById
} from '@/web/core/dataset/api';
import { DeleteIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@/web/common/hooks/useToast';
import { debounce } from 'lodash';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { useTranslation } from 'react-i18next';
import { useRouter } from 'next/router';
import MyIcon from '@/components/Icon';
import MyInput from '@/components/MyInput';
import { useLoading } from '@/web/common/hooks/useLoading';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import InputDataModal, { RawSourceText, type InputDataType } from '../components/InputDataModal';
import type { DatasetDataListItemType } from '@/global/core/dataset/response.d';
import { TabEnum } from '..';
const DataCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const { collectionId = '' } = router.query as { collectionId: string };
const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const { toast } = useToast();
const [isDeleting, setIsDeleting] = useState(false);
const { openConfirm, ConfirmModal } = useConfirm({
content: t('dataset.Confirm to delete the data')
});
const {
data: datasetDataList,
Pagination,
total,
getData,
pageNum,
pageSize
} = usePagination<DatasetDataListItemType>({
api: getDatasetDataList,
pageSize: 24,
params: {
collectionId,
searchText
},
onChange() {
setIsLoading(false);
if (BoxRef.current) {
BoxRef.current.scrollTop = 0;
}
}
});
const [editInputData, setEditInputData] = useState<InputDataType>();
// get first page data
const getFirstData = useCallback(
debounce(() => {
getData(1);
lastSearch.current = searchText;
}, 300),
[]
);
// get file info
const { data: collection } = useQuery(['getDatasetCollectionById', collectionId], () =>
getDatasetCollectionById(collectionId)
);
const fileIcon = useMemo(
() => getCollectionIcon(collection?.type, collection?.name),
[collection?.name, collection?.type]
);
return (
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex alignItems={'center'}>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={['14px', '18px']} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
onClick={() =>
router.replace({
query: {
datasetId: router.query.datasetId,
parentId: router.query.parentId,
currentTab: TabEnum.collectionCard
}
})
}
/>
<Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
<Image src={fileIcon || '/imgs/files/file.svg'} w={'16px'} mr={2} alt={''} />
<RawSourceText
sourceName={collection?.name}
sourceId={collection?.metadata?.fileId || collection?.metadata?.rawLink}
fontSize={['md', 'lg']}
color={'black'}
textDecoration={'none'}
/>
</Flex>
<Box>
<Button
ml={2}
variant={'base'}
size={['sm', 'md']}
onClick={() => {
if (!collection) return;
setEditInputData({
datasetId: collection.datasetId,
collectionId: collection._id,
sourceId: collection.metadata?.fileId || collection.metadata?.rawLink,
sourceName: collection.name
});
}}
>
{t('dataset.Insert Data')}
</Button>
</Box>
</Flex>
<Flex my={3} alignItems={'center'}>
<Box>
<Box as={'span'} fontSize={['md', 'lg']}>
{total}
</Box>
</Box>
<Box flex={1} mr={1} />
<MyInput
leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
}
w={['200px', '300px']}
placeholder="根据匹配知识,预期答案和来源进行搜索"
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
getFirstData();
}}
onBlur={() => {
if (searchText === lastSearch.current) return;
getFirstData();
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
getFirstData();
}
}}
/>
</Flex>
<Grid
minH={'100px'}
gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)']}
gridGap={4}
>
{datasetDataList.map((item) => (
<Card
key={item.id}
cursor={'pointer'}
pt={3}
userSelect={'none'}
boxShadow={'none'}
_hover={{ boxShadow: 'lg', '& .delete': { display: 'flex' } }}
border={'1px solid '}
borderColor={'myGray.200'}
onClick={() => {
if (!collection) return;
setEditInputData({
id: item.id,
datasetId: collection.datasetId,
collectionId: collection._id,
q: item.q,
a: item.a,
sourceId: collection.metadata?.fileId || collection.metadata?.rawLink,
sourceName: collection.name
});
}}
>
<Box
h={'95px'}
overflow={'hidden'}
wordBreak={'break-all'}
px={3}
py={1}
fontSize={'13px'}
>
<Box color={'myGray.1000'} mb={2}>
{item.q}
</Box>
<Box color={'myGray.600'}>{item.a}</Box>
</Box>
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
<Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
ID:{item.id}
</Box>
<IconButton
className="delete"
display={['flex', 'none']}
icon={<DeleteIcon />}
variant={'base'}
colorScheme={'gray'}
aria-label={'delete'}
size={'xs'}
borderRadius={'md'}
_hover={{ color: 'red.600' }}
isLoading={isDeleting}
onClick={(e) => {
e.stopPropagation();
openConfirm(async () => {
try {
setIsDeleting(true);
await delOneDatasetDataById(item.id);
getData(pageNum);
} catch (error) {
toast({
title: getErrText(error),
status: 'error'
});
}
setIsDeleting(false);
})();
}}
/>
</Flex>
</Card>
))}
</Grid>
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
{total === 0 && (
<Flex flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
</Box>
</Flex>
)}
{editInputData !== undefined && collection && (
<InputDataModal
datasetId={collection.datasetId}
defaultValues={editInputData}
onClose={() => setEditInputData(undefined)}
onSuccess={() => getData(pageNum)}
/>
)}
<ConfirmModal />
<Loading fixed={false} />
</Box>
);
};
export default React.memo(DataCard);

View File

@@ -0,0 +1,125 @@
import React, { useState } from 'react';
import {
Box,
Flex,
Button,
NumberInput,
NumberInputField,
NumberInputStepper,
NumberIncrementStepper,
NumberDecrementStepper
} from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
const ChunkImport = () => {
const { datasetDetail } = useDatasetStore();
const vectorModel = datasetDetail.vectorModel;
const unitPrice = vectorModel?.price || 0.2;
const {
chunkLen,
setChunkLen,
successChunks,
totalChunks,
isUnselectedFile,
price,
onclickUpload,
onReSplitChunks,
uploading,
showRePreview,
setReShowRePreview
} = useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: `该任务无法终止,需要一定时间生成索引,请确认导入。如果余额不足,未完成的任务会被暂停,充值后可继续进行。`
});
return (
<Box display={['block', 'flex']} h={['auto', '100%']}>
<SelectorContainer fileExtension={fileExtension}>
{/* chunk size */}
<Flex py={4} alignItems={'center'}>
<Box>
<MyTooltip
label={
'按结束标点符号进行分段。前后段落会有 20% 的内容重叠。\n中文文档建议不要超过1000英文不要超过1500'
}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box
flex={1}
css={{
'& > span': {
display: 'block'
}
}}
>
<MyTooltip label={`范围: 100~${datasetDetail.vectorModel.maxToken}`}>
<NumberInput
ml={4}
defaultValue={chunkLen}
min={100}
max={datasetDetail.vectorModel.maxToken}
step={10}
onChange={(e) => {
setChunkLen(+e);
setReShowRePreview(true);
}}
>
<NumberInputField />
<NumberInputStepper>
<NumberIncrementStepper />
<NumberDecrementStepper />
</NumberInputStepper>
</NumberInput>
</MyTooltip>
</Box>
</Flex>
{/* price */}
<Flex py={4} alignItems={'center'}>
<Box>
<MyTooltip
label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box ml={4}>{price}</Box>
</Flex>
<Flex mt={3}>
{showRePreview && (
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
</Button>
)}
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
</Button>
</Flex>
</SelectorContainer>
{!isUnselectedFile && (
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
<PreviewFileOrChunk />
</Box>
)}
<ConfirmModal />
</Box>
);
};
export default ChunkImport;

View File

@@ -0,0 +1,66 @@
import React from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Input, Textarea, ModalBody, ModalFooter, Button } from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import { useRequest } from '@/web/common/hooks/useRequest';
const CreateFileModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: { filename: string; content: string }) => Promise<void>;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
filename: '',
content: ''
}
});
const { mutate, isLoading } = useRequest({
mutationFn: () => handleSubmit(onSuccess)(),
onSuccess: () => {
onClose();
}
});
return (
<MyModal title={t('file.Create File')} isOpen w={'600px'} top={'15vh'}>
<ModalBody>
<Box mb={1} fontSize={'sm'}>
</Box>
<Input
mb={5}
{...register('filename', {
required: '文件名不能为空'
})}
/>
<Box mb={1} fontSize={'sm'}>
</Box>
<Textarea
{...register('content', {
required: '文件内容不能为空'
})}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
/>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
</Button>
<Button isLoading={isLoading} onClick={mutate}>
</Button>
</ModalFooter>
</MyModal>
);
};
export default CreateFileModal;

View File

@@ -0,0 +1,36 @@
import React from 'react';
import { Box, Flex, Button } from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
const fileExtension = '.csv';
const CsvImport = () => {
const { successChunks, totalChunks, isUnselectedFile, onclickUpload, uploading } =
useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: `该任务无法终止,需要一定时间生成索引,请确认导入。如果余额不足,未完成的任务会被暂停,充值后可继续进行。`
});
return (
<Box display={['block', 'flex']} h={['auto', '100%']}>
<SelectorContainer fileExtension={fileExtension} showUrlFetch={false}>
<Flex mt={3}>
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
</Button>
</Flex>
</SelectorContainer>
{!isUnselectedFile && (
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
<PreviewFileOrChunk />
</Box>
)}
<ConfirmModal />
</Box>
);
};
export default CsvImport;

View File

@@ -0,0 +1,424 @@
import MyIcon from '@/components/Icon';
import { useLoading } from '@/web/common/hooks/useLoading';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useToast } from '@/web/common/hooks/useToast';
import { splitText2Chunks } from '@/global/common/string/tools';
import { simpleText } from '@fastgpt/global/common/string/tools';
import {
uploadFiles,
fileDownload,
readCsvContent,
readTxtContent,
readPdfContent,
readDocContent
} from '@/web/common/file/utils';
import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
import { DragEvent, useCallback, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { customAlphabet } from 'nanoid';
import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import type {
DatasetChunkItemType,
DatasetCollectionSchemaType
} from '@fastgpt/global/core/dataset/type';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { countPromptTokens } from '@/global/common/tiktoken';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const csvTemplate = `index,content\n"被索引的内容","对应的答案。CSV 中请注意内容不能包含双引号,双引号是列分割符号"\n"什么是 laf","laf 是一个云函数开发平台……",""\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
export type FileItemType = {
id: string; // fileId / raw Link
filename: string;
chunks: DatasetChunkItemType[];
text: string; // raw text
icon: string;
tokens: number; // total tokens
type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
metadata: DatasetCollectionSchemaType['metadata'];
};
interface Props extends BoxProps {
fileExtension: string;
onPushFiles: (files: FileItemType[]) => void;
tipText?: string;
chunkLen?: number;
isCsv?: boolean;
showUrlFetch?: boolean;
showCreateFile?: boolean;
}
const FileSelect = ({
fileExtension,
onPushFiles,
tipText,
chunkLen = 500,
isCsv = false,
showUrlFetch = true,
showCreateFile = true,
...props
}: Props) => {
const { datasetDetail } = useDatasetStore();
const { Loading: FileSelectLoading } = useLoading();
const { t } = useTranslation();
const { toast } = useToast();
const { File: FileSelector, onOpen } = useSelectFile({
fileType: fileExtension,
multiple: true
});
const [isDragging, setIsDragging] = useState(false);
const [selectingText, setSelectingText] = useState<string>();
const {
isOpen: isOpenUrlFetch,
onOpen: onOpenUrlFetch,
onClose: onCloseUrlFetch
} = useDisclosure();
const {
isOpen: isOpenCreateFile,
onOpen: onOpenCreateFile,
onClose: onCloseCreateFile
} = useDisclosure();
// select file
const onSelectFile = useCallback(
async (files: File[]) => {
try {
for await (let file of files) {
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
/* text file */
const icon = getFileIcon(file?.name);
// ts
if (!icon) continue;
// upload file
const filesId = await uploadFiles([file], { datasetId: datasetDetail._id }, (percent) => {
if (percent < 100) {
setSelectingText(
t('file.Uploading', { name: file.name.slice(0, 30), percent }) || ''
);
} else {
setSelectingText(t('file.Parse', { name: file.name.slice(0, 30) }) || '');
}
});
const fileId = filesId[0];
/* csv file */
if (extension === 'csv') {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'index' || header[1] !== 'content') {
throw new Error('csv 文件格式有误,请确保 index 和 content 两列');
}
const filterData = data
.filter((item) => item[0])
.map((item) => ({
q: item[0] || '',
a: item[1] || ''
}));
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
tokens: filterData.reduce((sum, item) => sum + countPromptTokens(item.q), 0),
text: '',
chunks: filterData,
type: DatasetCollectionTypeEnum.file,
metadata: {
fileId
}
};
onPushFiles([fileItem]);
continue;
}
// parse and upload files
let text = await (async () => {
switch (extension) {
case 'txt':
case 'md':
return readTxtContent(file);
case 'pdf':
return readPdfContent(file);
case 'doc':
case 'docx':
return readDocContent(file);
}
return '';
})();
if (text) {
text = simpleText(text);
const splitRes = splitText2Chunks({
text,
maxLen: chunkLen
});
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
text,
tokens: splitRes.tokens,
type: DatasetCollectionTypeEnum.file,
metadata: {
fileId
},
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: ''
}))
};
onPushFiles([fileItem]);
}
}
} catch (error: any) {
console.log(error);
toast({
title: getErrText(error, '解析文件失败'),
status: 'error'
});
}
setSelectingText(undefined);
},
[chunkLen, datasetDetail._id, onPushFiles, t, toast]
);
// link fetch
const onUrlFetch = useCallback(
(e: FetchResultItem[]) => {
const result: FileItemType[] = e.map(({ url, content }) => {
const splitRes = splitText2Chunks({
text: content,
maxLen: chunkLen
});
return {
id: nanoid(),
filename: url,
icon: '/imgs/files/link.svg',
text: content,
tokens: splitRes.tokens,
type: DatasetCollectionTypeEnum.link,
metadata: {
rawLink: url
},
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: ''
}))
};
});
onPushFiles(result);
},
[chunkLen, onPushFiles]
);
// manual create file and copy data
const onCreateFile = useCallback(
async ({ filename, content }: { filename: string; content: string }) => {
content = simpleText(content);
// create virtual txt file
const txtBlob = new Blob([content], { type: 'text/plain' });
const txtFile = new File([txtBlob], `${filename}.txt`, {
type: txtBlob.type,
lastModified: new Date().getTime()
});
const fileIds = await uploadFiles([txtFile], { datasetId: datasetDetail._id });
const splitRes = splitText2Chunks({
text: content,
maxLen: chunkLen
});
onPushFiles([
{
id: nanoid(),
filename,
icon: '/imgs/files/txt.svg',
text: content,
tokens: splitRes.tokens,
type: DatasetCollectionTypeEnum.file,
metadata: {
fileId: fileIds[0]
},
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: ''
}))
}
]);
},
[chunkLen, datasetDetail._id, onPushFiles]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(true);
};
const handleDragLeave = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
};
const handleDrop = useCallback(
async (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
const items = e.dataTransfer.items;
const fileList: File[] = [];
if (e.dataTransfer.items.length <= 1) {
const traverseFileTree = async (item: any) => {
return new Promise<void>((resolve, reject) => {
if (item.isFile) {
item.file((file: File) => {
fileList.push(file);
resolve();
});
} else if (item.isDirectory) {
const dirReader = item.createReader();
dirReader.readEntries(async (entries: any[]) => {
for (let i = 0; i < entries.length; i++) {
await traverseFileTree(entries[i]);
}
resolve();
});
}
});
};
for (let i = 0; i < items.length; i++) {
const item = items[i].webkitGetAsEntry();
if (item) {
await traverseFileTree(item);
}
}
} else {
const files = Array.from(e.dataTransfer.files);
let isErr = files.some((item) => item.type === '');
if (isErr) {
return toast({
title: t('file.upload error description'),
status: 'error'
});
}
for (let i = 0; i < files.length; i++) {
fileList.push(files[i]);
}
}
onSelectFile(fileList);
},
[onSelectFile, t, toast]
);
const SelectTextStyles: BoxProps = {
ml: 1,
as: 'span',
cursor: 'pointer',
color: 'myBlue.700',
_hover: {
textDecoration: 'underline'
}
};
return (
<Box
display={'inline-block'}
textAlign={'center'}
bg={'myWhite.400'}
p={5}
borderRadius={'lg'}
border={'1px dashed'}
borderColor={'myGray.300'}
w={'100%'}
position={'relative'}
{...props}
onDragEnter={handleDragEnter}
onDragOver={handleDragEnter}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name={'uploadFile'} w={'16px'} />
{isDragging ? (
t('file.Release the mouse to upload the file')
) : (
<Box>
{t('file.Drag and drop')},
<MyTooltip label={t('file.max 10')}>
<Box {...SelectTextStyles} onClick={onOpen}>
{t('file.select a document')}
</Box>
</MyTooltip>
{showUrlFetch && (
<>
,
<Box {...SelectTextStyles} onClick={onOpenUrlFetch}>
{t('file.Fetch Url')}
</Box>
</>
)}
{showCreateFile && (
<>
,
<Box {...SelectTextStyles} onClick={onOpenCreateFile}>
{t('file.Create file')}
</Box>
</>
)}
</Box>
)}
</Flex>
<Box mt={1}>{t('file.support', { fileExtension: fileExtension })}</Box>
{tipText && (
<Box mt={1} fontSize={'sm'} color={'myGray.600'}>
{t(tipText)}
</Box>
)}
{isCsv && (
<Box
mt={1}
cursor={'pointer'}
textDecoration={'underline'}
color={'myBlue.600'}
fontSize={'12px'}
onClick={() =>
fileDownload({
text: csvTemplate,
type: 'text/csv',
filename: 'template.csv'
})
}
>
{t('file.Click to download CSV template')}
</Box>
)}
{selectingText !== undefined && (
<FileSelectLoading loading text={selectingText} fixed={false} />
)}
<FileSelector onSelect={onSelectFile} />
{isOpenUrlFetch && <UrlFetchModal onClose={onCloseUrlFetch} onSuccess={onUrlFetch} />}
{isOpenCreateFile && <CreateFileModal onClose={onCloseCreateFile} onSuccess={onCreateFile} />}
</Box>
);
};
export default FileSelect;

View File

@@ -0,0 +1,123 @@
import React, { useMemo, useState } from 'react';
import { Box, type BoxProps, Flex, useTheme, ModalCloseButton } from '@chakra-ui/react';
import MyRadio from '@/components/Radio/index';
import dynamic from 'next/dynamic';
import ChunkImport from './Chunk';
import { useTranslation } from 'react-i18next';
const QAImport = dynamic(() => import('./QA'), {});
const CsvImport = dynamic(() => import('./Csv'), {});
import MyModal from '@/components/MyModal';
import Provider from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { qaModelList } from '@/web/common/system/staticData';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
export enum ImportTypeEnum {
index = 'index',
qa = 'qa',
csv = 'csv'
}
const ImportData = ({
datasetId,
parentId,
onClose,
uploadSuccess
}: {
datasetId: string;
parentId: string;
onClose: () => void;
uploadSuccess: () => void;
}) => {
const { t } = useTranslation();
const theme = useTheme();
const { datasetDetail } = useDatasetStore();
const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.index);
const typeMap = useMemo(() => {
const vectorModel = datasetDetail.vectorModel;
const qaModel = qaModelList[0];
const map = {
[ImportTypeEnum.index]: {
defaultChunkLen: vectorModel?.defaultToken || 500,
unitPrice: vectorModel?.price || 0.2,
mode: TrainingModeEnum.index
},
[ImportTypeEnum.qa]: {
defaultChunkLen: qaModel?.maxToken * 0.5 || 8000,
unitPrice: qaModel?.price || 3,
mode: TrainingModeEnum.qa
},
[ImportTypeEnum.csv]: {
defaultChunkLen: vectorModel?.defaultToken || 500,
unitPrice: vectorModel?.price || 0.2,
mode: TrainingModeEnum.index
}
};
return map[importType];
}, [datasetDetail.vectorModel, importType]);
const TitleStyle: BoxProps = {
fontWeight: 'bold',
fontSize: ['md', 'xl']
};
return (
<MyModal
title={<Box {...TitleStyle}>{t('dataset.data.File import')}</Box>}
isOpen
isCentered
maxW={['90vw', '85vw']}
w={['90vw', '85vw']}
h={'90vh'}
>
<ModalCloseButton onClick={onClose} />
<Flex flexDirection={'column'} flex={'1 0 0'}>
<Box pb={[5, 7]} px={[4, 8]} borderBottom={theme.borders.base}>
<MyRadio
gridTemplateColumns={['repeat(1,1fr)', 'repeat(3, 350px)']}
list={[
{
icon: 'indexImport',
title: '直接分段',
desc: '选择文本文件,直接将其按分段进行处理',
value: ImportTypeEnum.index
},
{
icon: 'qaImport',
title: 'QA拆分',
desc: '选择文本文件,让大模型自动生成问答对',
value: ImportTypeEnum.qa
},
{
icon: 'csvImport',
title: 'CSV 导入',
desc: '批量导入问答对,是最精准的数据',
value: ImportTypeEnum.csv
}
]}
value={importType}
onChange={(e) => setImportType(e as `${ImportTypeEnum}`)}
/>
</Box>
<Provider
{...typeMap}
importType={importType}
datasetId={datasetId}
parentId={parentId}
onUploadSuccess={uploadSuccess}
>
<Box flex={'1 0 0'} h={0}>
{importType === ImportTypeEnum.index && <ChunkImport />}
{importType === ImportTypeEnum.qa && <QAImport />}
{importType === ImportTypeEnum.csv && <CsvImport />}
</Box>
</Provider>
</Flex>
</MyModal>
);
};
export default ImportData;

View File

@@ -0,0 +1,473 @@
import React, {
type SetStateAction,
type Dispatch,
useContext,
useCallback,
createContext,
useState,
useMemo,
useEffect
} from 'react';
import FileSelect, { FileItemType } from './FileSelect';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postDatasetCollection } from '@/web/core/dataset/api';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import { splitText2Chunks } from '@/global/common/string/tools';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { Box, Flex, Image, useTheme } from '@chakra-ui/react';
import { CloseIcon } from '@chakra-ui/icons';
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
import MyIcon from '@/components/Icon';
import { chunksUpload } from '@/web/core/dataset/utils';
import { postCreateTrainingBill } from '@/web/common/bill/api';
import { useTranslation } from 'react-i18next';
import { ImportTypeEnum } from './ImportModal';
const filenameStyles = {
className: 'textEllipsis',
maxW: '400px'
};
type useImportStoreType = {
files: FileItemType[];
setFiles: Dispatch<SetStateAction<FileItemType[]>>;
previewFile: FileItemType | undefined;
setPreviewFile: Dispatch<SetStateAction<FileItemType | undefined>>;
successChunks: number;
setSuccessChunks: Dispatch<SetStateAction<number>>;
isUnselectedFile: boolean;
totalChunks: number;
onclickUpload: (e: { files: FileItemType[] }) => void;
onReSplitChunks: () => void;
price: number;
uploading: boolean;
chunkLen: number;
setChunkLen: Dispatch<number>;
showRePreview: boolean;
setReShowRePreview: Dispatch<SetStateAction<boolean>>;
};
const StateContext = createContext<useImportStoreType>({
onclickUpload: function (e: { files: FileItemType[] }): void {
throw new Error('Function not implemented.');
},
uploading: false,
files: [],
previewFile: undefined,
successChunks: 0,
isUnselectedFile: false,
totalChunks: 0,
onReSplitChunks: function (): void {
throw new Error('Function not implemented.');
},
price: 0,
chunkLen: 0,
setChunkLen: function (value: number): void {
throw new Error('Function not implemented.');
},
setFiles: function (value: React.SetStateAction<FileItemType[]>): void {
throw new Error('Function not implemented.');
},
setPreviewFile: function (value: React.SetStateAction<FileItemType | undefined>): void {
throw new Error('Function not implemented.');
},
setSuccessChunks: function (value: React.SetStateAction<number>): void {
throw new Error('Function not implemented.');
},
showRePreview: false,
setReShowRePreview: function (value: React.SetStateAction<boolean>): void {
throw new Error('Function not implemented.');
}
});
export const useImportStore = () => useContext(StateContext);
const Provider = ({
datasetId,
parentId,
unitPrice,
mode,
defaultChunkLen = 500,
importType,
onUploadSuccess,
children
}: {
datasetId: string;
parentId: string;
unitPrice: number;
mode: `${TrainingModeEnum}`;
defaultChunkLen: number;
importType: `${ImportTypeEnum}`;
onUploadSuccess: () => void;
children: React.ReactNode;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const [files, setFiles] = useState<FileItemType[]>([]);
const [successChunks, setSuccessChunks] = useState(0);
const [chunkLen, setChunkLen] = useState(defaultChunkLen);
const [previewFile, setPreviewFile] = useState<FileItemType>();
const [showRePreview, setReShowRePreview] = useState(false);
const isUnselectedFile = useMemo(() => files.length === 0, [files]);
const totalChunks = useMemo(
() => files.reduce((sum, file) => sum + file.chunks.length, 0),
[files]
);
const price = useMemo(() => {
return formatPrice(files.reduce((sum, file) => sum + file.tokens, 0) * unitPrice);
}, [files, unitPrice]);
/* start upload data */
const { mutate: onclickUpload, isLoading: uploading } = useRequest({
mutationFn: async () => {
let totalInsertion = 0;
for await (const file of files) {
const chunks = file.chunks;
// create a file collection and training bill
const [collectionId, billId] = await Promise.all([
postDatasetCollection({
datasetId,
parentId,
name: file.filename,
type: file.type,
metadata: file.metadata
}),
postCreateTrainingBill({
name: t('dataset.collections.Create Training Data', { filename: file.filename })
})
]);
// upload data
const { insertLen } = await chunksUpload({
collectionId,
billId,
chunks,
mode,
onUploading: (insertLen) => {
setSuccessChunks((state) => state + insertLen);
}
});
totalInsertion += insertLen;
}
return totalInsertion;
},
onSuccess(num) {
toast({
title: `共成功导入 ${num} 组数据,请耐心等待训练.`,
status: 'success'
});
onUploadSuccess();
},
errorToast: '导入文件失败'
});
const onReSplitChunks = useCallback(async () => {
try {
setFiles((state) =>
state.map((file) => {
const splitRes = splitText2Chunks({
text: file.text,
maxLen: chunkLen
});
return {
...file,
tokens: splitRes.tokens,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: ''
}))
};
})
);
setReShowRePreview(false);
} catch (error) {
toast({
status: 'warning',
title: getErrText(error, '文本分段异常')
});
}
}, [chunkLen, toast]);
const reset = useCallback(() => {
setFiles([]);
setSuccessChunks(0);
setChunkLen(defaultChunkLen);
setPreviewFile(undefined);
setReShowRePreview(false);
}, [defaultChunkLen]);
useEffect(() => {
reset();
}, [importType, reset]);
const value = {
files,
setFiles,
previewFile,
setPreviewFile,
successChunks,
setSuccessChunks,
isUnselectedFile,
totalChunks,
price,
onReSplitChunks,
onclickUpload,
uploading,
chunkLen,
setChunkLen,
showRePreview,
setReShowRePreview
};
return <StateContext.Provider value={value}>{children}</StateContext.Provider>;
};
export default React.memo(Provider);
export const PreviewFileOrChunk = () => {
const theme = useTheme();
const { setFiles, previewFile, setPreviewFile, setReShowRePreview, totalChunks, files } =
useImportStore();
return (
<Box h={'100%'} w={'100%'}>
{!!previewFile ? (
<Box
position={'relative'}
display={['block', 'flex']}
h={'100%'}
flexDirection={'column'}
pt={[3, 6]}
bg={'myWhite.400'}
>
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'} {...filenameStyles}>
{previewFile.filename}
</Box>
<CloseIcon
position={'absolute'}
right={[4, 8]}
top={4}
cursor={'pointer'}
onClick={() => setPreviewFile(undefined)}
/>
<Box
flex={'1 0 0'}
h={['auto', 0]}
overflow={'overlay'}
px={[4, 8]}
my={4}
contentEditable
dangerouslySetInnerHTML={{ __html: previewFile.text }}
fontSize={'sm'}
whiteSpace={'pre-wrap'}
wordBreak={'break-all'}
onBlur={(e) => {
// @ts-ignore
const val = e.target.innerText;
setReShowRePreview(true);
setFiles((state) =>
state.map((file) =>
file.id === previewFile.id
? {
...file,
text: val
}
: file
)
);
}}
/>
</Box>
) : (
<Box pt={[3, 6]}>
<Flex px={[4, 8]} alignItems={'center'}>
<Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
({totalChunks})
</Box>
{totalChunks > 50 && (
<Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
</Box>
)}
</Flex>
<Box px={[4, 8]} overflow={'overlay'}>
{files.map((file) =>
file.chunks.slice(0, 50).map((chunk, i) => (
<Box
key={i}
py={4}
bg={'myWhite.500'}
my={2}
borderRadius={'md'}
fontSize={'sm'}
_hover={{ ...hoverDeleteStyles }}
>
<Flex mb={1} px={4} userSelect={'none'}>
<Box
flexShrink={0}
px={3}
py={'1px'}
border={theme.borders.base}
borderRadius={'md'}
>
# {i + 1}
</Box>
<Box ml={2} fontSize={'sm'} color={'myhGray.500'} {...filenameStyles}>
{file.filename}
</Box>
<Box flex={1} />
<DeleteIcon
onClick={() => {
setFiles((state) =>
state.map((stateFile) =>
stateFile.id === file.id
? {
...file,
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
}
: stateFile
)
);
}}
/>
</Flex>
<Box
px={4}
fontSize={'sm'}
whiteSpace={'pre-wrap'}
wordBreak={'break-all'}
contentEditable={!chunk.a}
dangerouslySetInnerHTML={{
__html: chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q
}}
onBlur={(e) => {
// @ts-ignore
const val = e.target.innerText;
/* delete file */
if (val === '') {
setFiles((state) =>
state.map((stateFile) =>
stateFile.id === file.id
? {
...file,
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
}
: stateFile
)
);
} else {
// update chunk
setFiles((stateFiles) =>
stateFiles.map((stateFile) =>
file.id === stateFile.id
? {
...stateFile,
chunks: stateFile.chunks.map((chunk, index) => ({
...chunk,
index: i === index ? val : chunk.q
}))
}
: stateFile
)
);
}
}}
/>
</Box>
))
)}
</Box>
</Box>
)}
</Box>
);
};
export const SelectorContainer = ({
fileExtension,
showUrlFetch,
showCreateFile,
children
}: {
fileExtension: string;
showUrlFetch?: boolean;
showCreateFile?: boolean;
children: React.ReactNode;
}) => {
const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen } = useImportStore();
return (
<Box
h={'100%'}
overflowY={'auto'}
flex={['auto', '1 0 400px']}
{...(isUnselectedFile
? {}
: {
maxW: ['auto', '500px']
})}
p={[4, 8]}
>
<FileSelect
fileExtension={fileExtension}
onPushFiles={(files) => {
setFiles((state) => files.concat(state));
}}
chunkLen={chunkLen}
showUrlFetch={showUrlFetch}
showCreateFile={showCreateFile}
py={isUnselectedFile ? '100px' : 5}
/>
{!isUnselectedFile && (
<Box py={4} px={2} maxH={'400px'} overflowY={'auto'}>
{files.map((item) => (
<Flex
key={item.id}
w={'100%'}
_notLast={{ mb: 5 }}
px={5}
py={2}
boxShadow={'1px 1px 5px rgba(0,0,0,0.15)'}
borderRadius={'md'}
cursor={'pointer'}
position={'relative'}
alignItems={'center'}
_hover={{
bg: 'myBlue.100',
'& .delete': {
display: 'block'
}
}}
onClick={() => setPreviewFile(item)}
>
<Image src={item.icon} w={'16px'} alt={''} />
<Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
{item.filename}
</Box>
<MyIcon
position={'absolute'}
right={3}
className="delete"
name={'delete'}
w={'16px'}
_hover={{ color: 'red.600' }}
display={['block', 'none']}
onClick={(e) => {
e.stopPropagation();
setFiles((state) => state.filter((file) => file.id !== item.id));
}}
/>
</Flex>
))}
</Box>
)}
{!isUnselectedFile && <>{children}</>}
</Box>
);
};

View File

@@ -0,0 +1,100 @@
import React, { useState, useMemo } from 'react';
import { Box, Flex, Button, Input } from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons';
import { Prompt_AgentQA } from '@/global/core/prompt/agent';
import { replaceVariable } from '@/global/common/string/tools';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
const QAImport = () => {
const { datasetDetail } = useDatasetStore();
const vectorModel = datasetDetail.vectorModel;
const unitPrice = vectorModel?.price || 0.2;
const {
successChunks,
totalChunks,
isUnselectedFile,
price,
onclickUpload,
onReSplitChunks,
uploading,
showRePreview
} = useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: `该任务无法终止!导入后会自动调用大模型生成问答对,会有一些细节丢失,请确认!如果余额不足,未完成的任务会被暂停。`
});
const [prompt, setPrompt] = useState('');
const previewQAPrompt = useMemo(() => {
return replaceVariable(Prompt_AgentQA.prompt, {
theme: prompt || Prompt_AgentQA.defaultTheme
});
}, [prompt]);
return (
<Box display={['block', 'flex']} h={['auto', '100%']}>
<SelectorContainer fileExtension={fileExtension}>
{/* prompt */}
<Box py={5}>
<Box mb={2}>
QA {' '}
<MyTooltip label={previewQAPrompt} forceShow>
<InfoOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Flex alignItems={'center'} fontSize={'sm'}>
<Box mr={2}></Box>
<Input
fontSize={'sm'}
flex={1}
placeholder={Prompt_AgentQA.defaultTheme}
bg={'myWhite.500'}
defaultValue={prompt}
onChange={(e) => setPrompt(e.target.value || '')}
/>
</Flex>
</Box>
{/* price */}
<Flex py={5} alignItems={'center'}>
<Box>
<MyTooltip
label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box ml={4}>{price}</Box>
</Flex>
<Flex mt={3}>
{showRePreview && (
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
</Button>
)}
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
</Button>
</Flex>
</SelectorContainer>
{!isUnselectedFile && (
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
<PreviewFileOrChunk />
</Box>
)}
<ConfirmModal />
</Box>
);
};
export default QAImport;

View File

@@ -0,0 +1,67 @@
import React, { useRef } from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postFetchUrls } from '@/web/common/plugin/api';
const UrlFetchModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: FetchResultItem[]) => void;
}) => {
const { t } = useTranslation();
const Dom = useRef<HTMLTextAreaElement>(null);
const { mutate, isLoading } = useRequest({
mutationFn: async () => {
const val = Dom.current?.value || '';
const urls = val.split('\n').filter((e) => e);
const res = await postFetchUrls(urls);
onSuccess(res);
onClose();
},
errorToast: '获取链接失败'
});
return (
<MyModal
title={
<>
<Box>{t('file.Fetch Url')}</Box>
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
</Box>
</>
}
top={'15vh'}
isOpen
onClose={onClose}
w={'600px'}
>
<ModalBody>
<Textarea
ref={Dom}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={'最多10个链接每行一个。'}
/>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
</Button>
<Button isLoading={isLoading} onClick={mutate}>
</Button>
</ModalFooter>
</MyModal>
);
};
export default UrlFetchModal;

View File

@@ -0,0 +1,251 @@
import React, {
useCallback,
useState,
useRef,
forwardRef,
useImperativeHandle,
ForwardedRef
} from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, Button, FormControl, IconButton, Input } from '@chakra-ui/react';
import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { delDatasetById, putDatasetById } from '@/web/core/dataset/api';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useToast } from '@/web/common/hooks/useToast';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { UseFormReturn } from 'react-hook-form';
import { compressImg } from '@/web/common/file/utils';
import type { DatasetItemType } from '@/types/core/dataset';
import Avatar from '@/components/Avatar';
import Tag from '@/components/Tag';
import MyTooltip from '@/components/MyTooltip';
export interface ComponentRef {
initInput: (tags: string) => void;
}
const Info = (
{ datasetId, form }: { datasetId: string; form: UseFormReturn<DatasetItemType, any> },
ref: ForwardedRef<ComponentRef>
) => {
const { getValues, formState, setValue, register, handleSubmit } = form;
const InputRef = useRef<HTMLInputElement>(null);
const { toast } = useToast();
const router = useRouter();
const [btnLoading, setBtnLoading] = useState(false);
const [refresh, setRefresh] = useState(false);
const { openConfirm, ConfirmModal } = useConfirm({
content: '确认删除该知识库?数据将无法恢复,请确认!'
});
const { File, onOpen: onOpenSelectFile } = useSelectFile({
fileType: '.jpg,.png',
multiple: false
});
const { datasetDetail, loadDatasetDetail, loadDatasets } = useDatasetStore();
/* 点击删除 */
const onclickDelKb = useCallback(async () => {
setBtnLoading(true);
try {
await delDatasetById(datasetId);
toast({
title: '删除成功',
status: 'success'
});
router.replace(`/dataset/list`);
await loadDatasets();
} catch (err: any) {
toast({
title: err?.message || '删除失败',
status: 'error'
});
}
setBtnLoading(false);
}, [setBtnLoading, datasetId, toast, router, loadDatasets]);
const saveSubmitSuccess = useCallback(
async (data: DatasetItemType) => {
setBtnLoading(true);
try {
await putDatasetById({
id: datasetId,
...data
});
await loadDatasetDetail(datasetId, true);
toast({
title: '更新成功',
status: 'success'
});
loadDatasets();
} catch (err: any) {
toast({
title: err?.message || '更新失败',
status: 'error'
});
}
setBtnLoading(false);
},
[loadDatasetDetail, datasetId, loadDatasets, toast]
);
const saveSubmitError = useCallback(() => {
// deep search message
const deepSearch = (obj: any): string => {
if (!obj) return '提交表单错误';
if (!!obj.message) {
return obj.message;
}
return deepSearch(Object.values(obj)[0]);
};
toast({
title: deepSearch(formState.errors),
status: 'error',
duration: 4000,
isClosable: true
});
}, [formState.errors, toast]);
const onSelectFile = useCallback(
async (e: File[]) => {
const file = e[0];
if (!file) return;
try {
const src = await compressImg({
file,
maxW: 100,
maxH: 100
});
setValue('avatar', src);
setRefresh((state) => !state);
} catch (err: any) {
toast({
title: typeof err === 'string' ? err : '头像选择异常',
status: 'warning'
});
}
},
[setRefresh, setValue, toast]
);
useImperativeHandle(ref, () => ({
initInput: (tags: string) => {
if (InputRef.current) {
InputRef.current.value = tags;
}
}
}));
return (
<Box py={5} px={[5, 10]}>
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
ID
</Box>
<Box flex={1}>{datasetDetail._id}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
MaxTokens
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
</Flex>
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
</Box>
<Box flex={[1, '0 0 300px']}>
<MyTooltip label={'点击切换头像'}>
<Avatar
m={'auto'}
src={getValues('avatar')}
w={['32px', '40px']}
h={['32px', '40px']}
cursor={'pointer'}
onClick={onOpenSelectFile}
/>
</MyTooltip>
</Box>
</Flex>
<FormControl mt={8} w={'100%'} display={'flex'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
</Box>
<Input
flex={[1, '0 0 300px']}
{...register('name', {
required: '知识库名称不能为空'
})}
/>
</FormControl>
<Flex mt={8} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<MyTooltip label={'用空格隔开多个标签,便于搜索'} forceShow>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Input
flex={[1, '0 0 300px']}
ref={InputRef}
defaultValue={getValues('tags')}
placeholder={'标签,使用空格分割。'}
maxLength={30}
onChange={(e) => {
setValue('tags', e.target.value);
setRefresh(!refresh);
}}
/>
<Flex w={'100%'} pl={['90px', '160px']} mt={2}>
{getValues('tags')
.split(' ')
.filter((item) => item)
.map((item, i) => (
<Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
{item}
</Tag>
))}
</Flex>
</Flex>
<Flex mt={5} w={'100%'} alignItems={'flex-end'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}></Box>
<Button
isLoading={btnLoading}
mr={4}
w={'100px'}
onClick={handleSubmit(saveSubmitSuccess, saveSubmitError)}
>
</Button>
<IconButton
isLoading={btnLoading}
icon={<DeleteIcon />}
aria-label={''}
variant={'outline'}
size={'sm'}
_hover={{
color: 'red.600',
borderColor: 'red.600'
}}
onClick={openConfirm(onclickDelKb)}
/>
</Flex>
<File onSelect={onSelectFile} />
<ConfirmModal />
</Box>
);
};
export default forwardRef(Info);

View File

@@ -0,0 +1,299 @@
import React, { useMemo } from 'react';
import { Box, Flex, Button, Textarea, IconButton, BoxProps } from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import {
postData2Dataset,
putDatasetDataById,
delOneDatasetDataById
} from '@/web/core/dataset/api';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import MyIcon from '@/components/Icon';
import MyModal from '@/components/MyModal';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query';
import { useTranslation } from 'react-i18next';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { getFileAndOpen } from '@/web/common/file/utils';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq';
import { useRequest } from '@/web/common/hooks/useRequest';
import { countPromptTokens } from '@/global/common/tiktoken';
import { useConfirm } from '@/web/common/hooks/useConfirm';
export type RawSourceType = {
sourceName?: string;
sourceId?: string;
};
export type RawSourceTextProps = BoxProps & RawSourceType;
export type InputDataType = SetOneDatasetDataProps & RawSourceType;
const InputDataModal = ({
onClose,
onSuccess,
onDelete,
datasetId,
defaultValues = {
datasetId: '',
collectionId: '',
sourceId: '',
sourceName: ''
}
}: {
onClose: () => void;
onSuccess: (data: SetOneDatasetDataProps) => void;
onDelete?: () => void;
datasetId: string;
defaultValues: InputDataType;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
const { register, handleSubmit, reset } = useForm<InputDataType>({
defaultValues
});
const { ConfirmModal, openConfirm } = useConfirm({
content: t('dataset.data.Delete Tip')
});
const maxToken = datasetDetail.vectorModel?.maxToken || 2000;
/**
* 确认导入新数据
*/
const { mutate: sureImportData, isLoading: isImporting } = useRequest({
mutationFn: async (e: InputDataType) => {
if (!e.q) {
return toast({
title: '匹配的知识点不能为空',
status: 'warning'
});
}
if (countPromptTokens(e.q) >= maxToken) {
return toast({
title: '总长度超长了',
status: 'warning'
});
}
const data = { ...e };
delete data.sourceName;
delete data.sourceId;
data.id = await postData2Dataset(data);
return data;
},
successToast: t('dataset.data.Input Success Tip'),
onSuccess(e) {
reset({
...e,
q: '',
a: ''
});
onSuccess(e);
},
errorToast: t('common.error.unKnow')
});
const { mutate: onUpdateData, isLoading: isUpdating } = useRequest({
mutationFn: async (e: SetOneDatasetDataProps) => {
if (!e.id) return e;
// not exactly same
if (e.q !== defaultValues.q || e.a !== defaultValues.a) {
await putDatasetDataById({
...e,
q: e.q === defaultValues.q ? '' : e.q
});
return e;
}
return e;
},
successToast: t('dataset.data.Update Success Tip'),
errorToast: t('common.error.unKnow'),
onSuccess(data) {
onSuccess(data);
onClose();
}
});
const loading = useMemo(() => isImporting || isUpdating, [isImporting, isUpdating]);
useQuery(['loadDatasetDetail'], () => {
if (datasetDetail._id === datasetId) return null;
return loadDatasetDetail(datasetId);
});
return (
<MyModal
isOpen={true}
isCentered
title={defaultValues.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')}
w={'90vw'}
maxW={'90vw'}
h={'90vh'}
>
<Flex flexDirection={'column'} h={'100%'}>
<Box
display={'flex'}
flexDirection={['column', 'row']}
flex={'1 0 0'}
h={['100%', 0]}
overflow={'overlay'}
px={6}
pb={2}
>
<Box flex={1} mr={[0, 4]} mb={[4, 0]} h={['50%', '100%']}>
<Flex>
<Box h={'30px'}>{'匹配的知识点'}</Box>
<MyTooltip label={'被向量化的部分,通常是问题,也可以是一段陈述描述'}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
<Textarea
placeholder={`匹配的知识点。这部分内容会被搜索,请把控内容的质量,最多 ${maxToken} 字。`}
maxLength={maxToken}
resize={'none'}
h={'calc(100% - 30px)'}
{...register(`q`, {
required: true
})}
/>
</Box>
<Box flex={1} h={['50%', '100%']}>
<Flex>
<Box h={'30px'}>{'补充内容'}</Box>
<MyTooltip
label={'匹配的知识点被命中后,这部分内容会随匹配知识点一起注入模型,引导模型回答'}
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
<Textarea
placeholder={
'这部分内容不会被搜索,但会作为"匹配的知识点"的内容补充,通常是问题的答案。'
}
resize={'none'}
h={'calc(100% - 30px)'}
{...register('a')}
/>
</Box>
</Box>
<Flex px={6} pt={['34px', 2]} pb={4} alignItems={'center'} position={'relative'}>
<RawSourceText
sourceName={defaultValues.sourceName}
sourceId={defaultValues.sourceId}
position={'absolute'}
left={'50%'}
top={['16px', '50%']}
transform={'translate(-50%,-50%)'}
/>
<Box flex={1}>
{defaultValues.id && onDelete && (
<IconButton
variant={'outline'}
icon={<MyIcon name={'delete'} w={'16px'} h={'16px'} />}
aria-label={''}
isLoading={loading}
size={'sm'}
_hover={{
color: 'red.600',
borderColor: 'red.600'
}}
onClick={openConfirm(async () => {
if (!onDelete || !defaultValues.id) return;
try {
await delOneDatasetDataById(defaultValues.id);
onDelete();
onClose();
toast({
status: 'success',
title: '记录已删除'
});
} catch (error) {
toast({
status: 'warning',
title: getErrText(error)
});
console.log(error);
}
})}
/>
)}
</Box>
<Box>
<Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
{t('common.Close')}
</Button>
<Button
isLoading={loading}
// @ts-ignore
onClick={handleSubmit(defaultValues.id ? onUpdateData : sureImportData)}
>
{defaultValues.id ? '确认变更' : '确认导入'}
</Button>
</Box>
</Flex>
</Flex>
<ConfirmModal />
</MyModal>
);
};
export default InputDataModal;
export function RawSourceText({ sourceId, sourceName = '', ...props }: RawSourceTextProps) {
const { t } = useTranslation();
const { toast } = useToast();
const { setLoading } = useSystemStore();
const canPreview = useMemo(() => !!sourceId, [sourceId]);
return (
<MyTooltip
label={sourceId ? t('file.Click to view file') || '' : ''}
shouldWrapChildren={false}
>
<Box
color={'myGray.600'}
display={'inline-block'}
whiteSpace={'nowrap'}
maxW={['200px', '300px']}
className={'textEllipsis'}
{...(canPreview
? {
cursor: 'pointer',
textDecoration: 'underline',
onClick: async () => {
if (strIsLink(sourceId)) {
return window.open(sourceId, '_blank');
}
setLoading(true);
try {
await getFileAndOpen(sourceId as string);
} catch (error) {
toast({
title: getErrText(error, '获取文件地址失败'),
status: 'error'
});
}
setLoading(false);
}
}
: {})}
{...props}
>
{sourceName || t('common.Unknow Source')}
</Box>
</MyTooltip>
);
}

View File

@@ -0,0 +1,296 @@
import React, { useEffect, useMemo, useState } from 'react';
import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useSearchTestStore, SearchTestStoreItemType } from '@/web/core/dataset/store/searchTest';
import { getDatasetDataItemById, postSearchText } from '@/web/core/dataset/api';
import MyIcon from '@/components/Icon';
import { useRequest } from '@/web/common/hooks/useRequest';
import { formatTimeToChatTime } from '@/utils/tools';
import InputDataModal, { type InputDataType } from './InputDataModal';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useToast } from '@/web/common/hooks/useToast';
import { customAlphabet } from 'nanoid';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const Test = ({ datasetId }: { datasetId: string }) => {
const theme = useTheme();
const { toast } = useToast();
const { setLoading } = useSystemStore();
const { datasetDetail } = useDatasetStore();
const { datasetTestList, pushDatasetTestItem, delDatasetTestItemById, updateDatasetItemById } =
useSearchTestStore();
const [inputText, setInputText] = useState('');
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const [editInputData, setEditInputData] = useState<InputDataType>();
const kbTestHistory = useMemo(
() => datasetTestList.filter((item) => item.datasetId === datasetId),
[datasetId, datasetTestList]
);
const { mutate, isLoading } = useRequest({
mutationFn: () => postSearchText({ datasetId, text: inputText.trim() }),
onSuccess(res) {
const testItem = {
id: nanoid(),
datasetId,
text: inputText.trim(),
time: new Date(),
results: res
};
pushDatasetTestItem(testItem);
setDatasetTestItem(testItem);
},
onError(err) {
toast({
title: getErrText(err),
status: 'error'
});
}
});
useEffect(() => {
setDatasetTestItem(undefined);
}, [datasetId]);
return (
<Box h={'100%'} display={['block', 'flex']}>
<Box
h={['auto', '100%']}
display={['block', 'flex']}
flexDirection={'column'}
flex={1}
maxW={'500px'}
py={4}
borderRight={['none', theme.borders.base]}
>
<Box border={'2px solid'} borderColor={'myBlue.600'} p={3} mx={4} borderRadius={'md'}>
<Box fontSize={'sm'} fontWeight={'bold'}>
<MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'myBlue.700'} />
</Box>
<Textarea
rows={6}
resize={'none'}
variant={'unstyled'}
maxLength={datasetDetail.vectorModel.maxToken}
placeholder="输入需要测试的文本"
value={inputText}
onChange={(e) => setInputText(e.target.value)}
/>
<Flex alignItems={'center'} justifyContent={'flex-end'}>
<Box mr={3} color={'myGray.500'}>
{inputText.length}
</Box>
<Button isDisabled={inputText === ''} isLoading={isLoading} onClick={mutate}>
</Button>
</Flex>
</Box>
<Box mt={5} flex={'1 0 0'} px={4} overflow={'overlay'} display={['none', 'block']}>
<Flex alignItems={'center'} color={'myGray.600'}>
<MyIcon mr={2} name={'history'} w={'16px'} h={'16px'} />
<Box fontSize={'2xl'}></Box>
</Flex>
<Box mt={2}>
<Flex py={2} fontWeight={'bold'} borderBottom={theme.borders.sm}>
<Box flex={1}></Box>
<Box w={'80px'}></Box>
<Box w={'14px'}></Box>
</Flex>
{kbTestHistory.map((item) => (
<Flex
key={item.id}
p={1}
alignItems={'center'}
borderBottom={theme.borders.base}
_hover={{
bg: '#f4f4f4',
'& .delete': {
display: 'block'
}
}}
cursor={'pointer'}
onClick={() => setDatasetTestItem(item)}
>
<Box flex={1} mr={2}>
{item.text}
</Box>
<Box w={'80px'}>{formatTimeToChatTime(item.time)}</Box>
<MyTooltip label={'删除该测试记录'}>
<Box w={'14px'} h={'14px'}>
<MyIcon
className="delete"
name={'delete'}
w={'14px'}
display={'none'}
_hover={{ color: 'red.600' }}
onClick={(e) => {
e.stopPropagation();
delDatasetTestItemById(item.id);
datasetTestItem?.id === item.id && setDatasetTestItem(undefined);
}}
/>
</Box>
</MyTooltip>
</Flex>
))}
</Box>
</Box>
</Box>
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={1}>
{!datasetTestItem?.results || datasetTestItem.results.length === 0 ? (
<Flex
mt={[10, 0]}
h={'100%'}
flexDirection={'column'}
alignItems={'center'}
justifyContent={'center'}
>
<MyIcon name={'empty'} color={'transparent'} w={'54px'} />
<Box mt={3} color={'myGray.600'}>
</Box>
</Flex>
) : (
<>
<Flex alignItems={'center'}>
<Box fontSize={'3xl'} color={'myGray.600'}>
</Box>
<MyTooltip
label={
'根据知识库内容与测试文本的相似度进行排序,你可以根据测试结果调整对应的文本。\n注意测试记录中的数据可能已经被修改过点击某条测试数据后将展示最新的数据。'
}
forceShow
>
<QuestionOutlineIcon
ml={2}
color={'myGray.600'}
cursor={'pointer'}
fontSize={'lg'}
/>
</MyTooltip>
</Flex>
<Grid
mt={1}
gridTemplateColumns={[
'repeat(1,1fr)',
'repeat(1,1fr)',
'repeat(1,1fr)',
'repeat(1,1fr)',
'repeat(2,1fr)'
]}
gridGap={4}
>
{datasetTestItem?.results.map((item) => (
<Box
key={item.id}
pb={2}
borderRadius={'sm'}
border={theme.borders.base}
_notLast={{ mb: 2 }}
cursor={'pointer'}
title={'编辑'}
onClick={async () => {
try {
setLoading(true);
const data = await getDatasetDataItemById(item.id);
if (!data) {
throw new Error('该数据已被删除');
}
setEditInputData({
id: data.id,
datasetId: data.datasetId,
collectionId: data.collectionId,
q: data.q,
a: data.a,
sourceName: data.sourceName,
sourceId: data.sourceId
});
} catch (err) {
toast({
status: 'warning',
title: getErrText(err)
});
}
setLoading(false);
}}
>
<Flex p={3} alignItems={'center'} color={'myGray.500'}>
<MyIcon name={'kbTest'} w={'14px'} />
<Progress
mx={2}
flex={1}
value={item.score * 100}
size="sm"
borderRadius={'20px'}
colorScheme="gray"
/>
<Box>{item.score.toFixed(4)}</Box>
</Flex>
<Box
px={2}
fontSize={'xs'}
color={'myGray.600'}
maxH={'200px'}
overflow={'overlay'}
>
<Box>{item.q}</Box>
<Box>{item.a}</Box>
</Box>
</Box>
))}
</Grid>
</>
)}
</Box>
{!!editInputData && (
<InputDataModal
datasetId={editInputData.datasetId}
defaultValues={editInputData}
onClose={() => setEditInputData(undefined)}
onSuccess={(data) => {
if (datasetTestItem && editInputData.id) {
const newTestItem: SearchTestStoreItemType = {
...datasetTestItem,
results: datasetTestItem.results.map((item) =>
item.id === editInputData.id
? {
...item,
q: data.q || '',
a: data.a || ''
}
: item
)
};
updateDatasetItemById(newTestItem);
setDatasetTestItem(newTestItem);
}
setEditInputData(undefined);
}}
onDelete={() => {
if (datasetTestItem && editInputData.id) {
const newTestItem = {
...datasetTestItem,
results: datasetTestItem.results.filter((item) => item.id !== editInputData.id)
};
updateDatasetItemById(newTestItem);
setDatasetTestItem(newTestItem);
}
setEditInputData(undefined);
}}
/>
)}
</Box>
);
};
export default Test;

View File

@@ -0,0 +1,207 @@
import React, { useCallback, useEffect, useRef } from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
import { useToast } from '@/web/common/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useQuery } from '@tanstack/react-query';
import { DatasetItemType } from '@/types/core/dataset';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { type ComponentRef } from './components/Info';
import Tabs from '@/components/Tabs';
import dynamic from 'next/dynamic';
import MyIcon from '@/components/Icon';
import SideTabs from '@/components/SideTabs';
import PageContainer from '@/components/PageContainer';
import Avatar from '@/components/Avatar';
import Info from './components/Info';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useTranslation } from 'react-i18next';
import { getTrainingQueueLen, delDatasetEmptyFiles } from '@/web/core/dataset/api';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { feConfigs } from '@/web/common/system/staticData';
import Script from 'next/script';
import CollectionCard from './components/CollectionCard';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
});
const Test = dynamic(() => import('./components/Test'), {
ssr: false
});
export enum TabEnum {
dataCard = 'dataCard',
collectionCard = 'collectionCard',
test = 'test',
info = 'info'
}
const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${TabEnum}` }) => {
const InfoRef = useRef<ComponentRef>(null);
const theme = useTheme();
const { t } = useTranslation();
const { toast } = useToast();
const router = useRouter();
const { isPc } = useSystemStore();
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
const tabList = useRef([
{ label: '数据集', id: TabEnum.collectionCard, icon: 'overviewLight' },
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
{ label: '配置', id: TabEnum.info, icon: 'settingLight' }
]);
const setCurrentTab = useCallback(
(tab: `${TabEnum}`) => {
router.replace({
query: {
datasetId,
currentTab: tab
}
});
},
[datasetId, router]
);
const form = useForm<DatasetItemType>({
defaultValues: datasetDetail
});
useQuery([datasetId], () => loadDatasetDetail(datasetId), {
onSuccess(res) {
form.reset(res);
InfoRef.current?.initInput(res.tags);
},
onError(err: any) {
router.replace(`/dataset/list`);
toast({
title: getErrText(err, '获取知识库异常'),
status: 'error'
});
}
});
const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, {
refetchInterval: 10000
});
useEffect(() => {
return () => {
try {
delDatasetEmptyFiles(datasetId);
} catch (error) {}
};
}, [datasetId]);
return (
<>
<Script src="/js/pdf.js" strategy="lazyOnload"></Script>
<PageContainer>
<Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={datasetDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{datasetDetail.name}
</Box>
</Flex>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/>
<Box textAlign={'center'}>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
<Box>{t('dataset.System Data Queue')}</Box>
<MyTooltip
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
placement={'top'}
>
<QuestionOutlineIcon ml={1} w={'16px'} />
</MyTooltip>
</Flex>
<Box mt={1} fontWeight={'bold'}>
{trainingQueueLen}
</Box>
</Box>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={2}
px={3}
borderRadius={'md'}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/dataset/list')}
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
/>
</Flex>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
w={'260px'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={(e: any) => setCurrentTab(e)}
/>
</Box>
)}
{!!datasetDetail._id && (
<Box flex={'1 0 0'} pb={[4, 0]}>
{currentTab === TabEnum.collectionCard && <CollectionCard />}
{currentTab === TabEnum.dataCard && <DataCard />}
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
{currentTab === TabEnum.info && (
<Info ref={InfoRef} datasetId={datasetId} form={form} />
)}
</Box>
)}
</Flex>
</PageContainer>
</>
);
};
export async function getServerSideProps(context: any) {
const currentTab = context?.query?.currentTab || TabEnum.collectionCard;
const datasetId = context?.query?.datasetId;
return {
props: { currentTab, datasetId, ...(await serviceSideProps(context)) }
};
}
export default React.memo(Detail);