This commit is contained in:
Archer
2023-12-27 11:07:39 +08:00
committed by GitHub
parent 86286efb54
commit 759a2330e6
182 changed files with 3099 additions and 81685 deletions

View File

@@ -53,7 +53,7 @@ const EditFolderModal = ({
/>
</ModalBody>
<ModalFooter>
<Button mr={3} variant={'base'} onClick={onClose}>
<Button mr={3} variant={'whiteBase'} onClick={onClose}>
{t('Cancel')}
</Button>
<Button isLoading={isLoading} onClick={onSave}>

View File

@@ -311,7 +311,7 @@ const CollectionCard = () => {
target="_blank"
mr={2}
textDecoration={'underline'}
color={'blue.600'}
color={'primary.600'}
>
{datasetDetail.websiteConfig.url}
</Link>
@@ -371,7 +371,7 @@ const CollectionCard = () => {
Button={
<MenuButton
_hover={{
color: 'blue.500'
color: 'primary.500'
}}
fontSize={['sm', 'md']}
>
@@ -381,7 +381,7 @@ const CollectionCard = () => {
py={2}
borderRadius={'md'}
cursor={'pointer'}
bg={'blue.500'}
bg={'primary.500'}
overflow={'hidden'}
color={'white'}
h={['28px', '35px']}
@@ -489,7 +489,7 @@ const CollectionCard = () => {
data-drag-id={
collection.type === DatasetCollectionTypeEnum.folder ? collection._id : undefined
}
bg={dragTargetId === collection._id ? 'blue.100' : ''}
bg={dragTargetId === collection._id ? 'primary.100' : ''}
userSelect={'none'}
onDragStart={(e) => {
setDragStartId(collection._id);
@@ -579,7 +579,7 @@ const CollectionCard = () => {
h={'22px'}
borderRadius={'md'}
_hover={{
color: 'blue.500',
color: 'primary.500',
'& .icon': {
bg: 'myGray.100'
}

View File

@@ -163,10 +163,9 @@ const DataCard = () => {
<Flex alignItems={'center'}>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={['14px', '18px']} color={'blue.500'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
size={'sm'}
icon={<MyIcon name={'backFill'} w={['14px', '18px']} color={'primary.500'} />}
variant={'whitePrimary'}
size={'smSquare'}
borderRadius={'50%'}
aria-label={''}
onClick={() =>
@@ -200,7 +199,7 @@ const DataCard = () => {
<Box>
<Button
mx={2}
variant={'base'}
variant={'whitePrimary'}
size={['sm', 'md']}
onClick={() => {
if (!collection) return;
@@ -217,7 +216,7 @@ const DataCard = () => {
{isPc && (
<MyTooltip label={t('core.dataset.collection.metadata.Read Metadata')}>
<IconButton
variant={'base'}
variant={'whiteBase'}
size={['sm', 'md']}
icon={<MyIcon name={'menu'} w={'18px'} />}
aria-label={''}
@@ -338,12 +337,9 @@ const DataCard = () => {
<IconButton
display={'flex'}
icon={<DeleteIcon />}
variant={'base'}
colorScheme={'gray'}
variant={'whiteDanger'}
size={'xsSquare'}
aria-label={'delete'}
size={'xs'}
borderRadius={'md'}
_hover={{ color: 'red.600' }}
onClick={(e) => {
e.stopPropagation();
openConfirm(async () => {
@@ -385,7 +381,7 @@ const DataCard = () => {
))}
{collection?.sourceId && (
<Button
variant={'base'}
variant={'whitePrimary'}
onClick={() => collection.sourceId && getFileAndOpen(collection.sourceId)}
>
{t('core.dataset.collection.metadata.read source')}
@@ -394,7 +390,7 @@ const DataCard = () => {
</DrawerBody>
<DrawerFooter>
<Button variant={'base'} onClick={onClose}>
<Button variant={'whitePrimary'} onClick={onClose}>
{t('common.Close')}
</Button>
</DrawerFooter>

View File

@@ -7,7 +7,8 @@ import {
NumberInputField,
NumberInputStepper,
NumberIncrementStepper,
NumberDecrementStepper
NumberDecrementStepper,
Input
} from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
@@ -18,7 +19,7 @@ import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
import { useTranslation } from 'next-i18next';
const fileExtension = '.txt, .docx, .pdf, .md';
const fileExtension = '.txt, .docx, .pdf, .md, .html';
const ChunkImport = () => {
const { t } = useTranslation();
@@ -29,6 +30,7 @@ const ChunkImport = () => {
const {
chunkLen,
setChunkLen,
setCustomSplitChar,
successChunks,
totalChunks,
isUnselectedFile,
@@ -48,15 +50,15 @@ const ChunkImport = () => {
<Box display={['block', 'flex']} h={['auto', '100%']}>
<SelectorContainer fileExtension={fileExtension}>
{/* chunk size */}
<Flex py={4} alignItems={'center'}>
<Box mt={4} alignItems={'center'}>
<Box>
{t('core.dataset.import.Ideal chunk length')}
<MyTooltip label={t('core.dataset.import.Ideal chunk length Tips')} forceShow>
<QuestionOutlineIcon ml={1} />
<QuestionOutlineIcon />
</MyTooltip>
</Box>
<Box
flex={1}
mt={1}
css={{
'& > span': {
display: 'block'
@@ -69,7 +71,6 @@ const ChunkImport = () => {
})}
>
<NumberInput
ml={4}
defaultValue={chunkLen}
min={100}
max={datasetDetail.vectorModel.maxToken}
@@ -87,9 +88,28 @@ const ChunkImport = () => {
</NumberInput>
</MyTooltip>
</Box>
</Flex>
</Box>
{/* custom split char */}
<Box mt={4} alignItems={'center'}>
<Box>
{t('core.dataset.import.Custom split char')}
<MyTooltip label={t('core.dataset.import.Custom split char Tips')} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Box>
<Box mt={1}>
<Input
defaultValue={''}
placeholder="\n;======;==SPLIT=="
onChange={(e) => {
setCustomSplitChar(e.target.value);
setReShowRePreview(true);
}}
/>
</Box>
</Box>
{/* price */}
<Flex py={4} alignItems={'center'}>
<Flex mt={4} alignItems={'center'}>
<Box>
{t('core.dataset.import.Estimated Price')}
<MyTooltip
@@ -105,12 +125,23 @@ const ChunkImport = () => {
</Flex>
<Flex mt={3}>
{showRePreview && (
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
<Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
{t('core.dataset.import.Re Preview')}
</Button>
)}
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
<Button
isDisabled={uploading}
onClick={() => {
onReSplitChunks();
openConfirm(onclickUpload)();
}}
>
{uploading ? (
<Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
) : (
t('common.Confirm Import')
)}
</Button>
</Flex>
</SelectorContainer>

View File

@@ -58,7 +58,7 @@ const CreateFileModal = ({
/>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
<Button variant={'whiteBase'} mr={4} onClick={onClose}>
{t('common.Close')}
</Button>
<Button isLoading={isLoading} onClick={mutate}>

View File

@@ -7,11 +7,11 @@ import { simpleText } from '@fastgpt/global/common/string/tools';
import {
fileDownload,
readCsvContent,
readTxtContent,
readPdfContent,
readDocContent
} from '@/web/common/file/utils';
import { uploadFiles } from '@/web/common/file/controller';
import { readFileRawText, readMdFile, readHtmlFile } from '@fastgpt/web/common/file/read';
import { getUploadMdImgController, uploadFiles } from '@/web/common/file/controller';
import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
import React, { DragEvent, useCallback, useState } from 'react';
import { useTranslation } from 'next-i18next';
@@ -48,6 +48,7 @@ export interface Props extends BoxProps {
onPushFiles: (files: FileItemType[]) => void;
tipText?: string;
chunkLen?: number;
customSplitChar?: string;
overlapRatio?: number;
fileTemplate?: {
type: string;
@@ -64,6 +65,7 @@ const FileSelect = ({
onPushFiles,
tipText,
chunkLen = 500,
customSplitChar,
overlapRatio,
fileTemplate,
showUrlFetch = true,
@@ -133,7 +135,7 @@ const FileSelect = ({
});
const fileId = filesId[0];
/* csv file */
/* QA csv file */
if (extension === 'csv') {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'index' || header[1] !== 'content') {
@@ -168,8 +170,19 @@ const FileSelect = ({
let text = await (async () => {
switch (extension) {
case 'txt':
return readFileRawText(file);
case 'md':
return readTxtContent(file);
return readMdFile({
file,
uploadImgController: (base64Img) =>
getUploadMdImgController({ base64Img, metadata: { fileId } })
});
case 'html':
return readHtmlFile({
file,
uploadImgController: (base64Img) =>
getUploadMdImgController({ base64Img, metadata: { fileId } })
});
case 'pdf':
return readPdfContent(file);
case 'docx':
@@ -185,7 +198,8 @@ const FileSelect = ({
const { chunks, tokens } = splitText2Chunks({
text,
chunkLen,
overlapRatio
overlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
const fileItem: FileItemType = {
@@ -213,7 +227,7 @@ const FileSelect = ({
}
setSelectingText(undefined);
},
[chunkLen, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
[chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
);
// link fetch
const onUrlFetch = useCallback(
@@ -222,7 +236,8 @@ const FileSelect = ({
const { chunks, tokens } = splitText2Chunks({
text: content,
chunkLen,
overlapRatio
overlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
return {
id: nanoid(),
@@ -240,7 +255,7 @@ const FileSelect = ({
});
onPushFiles(result);
},
[chunkLen, onPushFiles, overlapRatio]
[chunkLen, customSplitChar, onPushFiles, overlapRatio]
);
// manual create file and copy data
const onCreateFile = useCallback(
@@ -262,7 +277,8 @@ const FileSelect = ({
const { chunks, tokens } = splitText2Chunks({
text: content,
chunkLen,
overlapRatio
overlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
onPushFiles([
@@ -281,7 +297,7 @@ const FileSelect = ({
}
]);
},
[chunkLen, datasetDetail._id, onPushFiles, overlapRatio]
[chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
@@ -352,7 +368,7 @@ const FileSelect = ({
ml: 1,
as: 'span',
cursor: 'pointer',
color: 'blue.600',
color: 'primary.600',
_hover: {
textDecoration: 'underline'
}
@@ -417,7 +433,7 @@ const FileSelect = ({
mt={1}
cursor={'pointer'}
textDecoration={'underline'}
color={'blue.500'}
color={'primary.500'}
fontSize={'12px'}
onClick={() =>
fileDownload({

View File

@@ -43,18 +43,20 @@ type useImportStoreType = {
setSuccessChunks: Dispatch<SetStateAction<number>>;
isUnselectedFile: boolean;
totalChunks: number;
onclickUpload: (e: { prompt?: string }) => void;
onclickUpload: (e?: { prompt?: string }) => void;
onReSplitChunks: () => void;
price: number;
uploading: boolean;
chunkLen: number;
chunkOverlapRatio: number;
setChunkLen: Dispatch<number>;
customSplitChar?: string;
setCustomSplitChar: Dispatch<string>;
showRePreview: boolean;
setReShowRePreview: Dispatch<SetStateAction<boolean>>;
};
const StateContext = createContext<useImportStoreType>({
onclickUpload: function (e: { prompt?: string }): void {
onclickUpload: function (e?: { prompt?: string }): void {
throw new Error('Function not implemented.');
},
uploading: false,
@@ -72,6 +74,10 @@ const StateContext = createContext<useImportStoreType>({
price: 0,
chunkLen: 0,
chunkOverlapRatio: 0,
customSplitChar: undefined,
setCustomSplitChar: function (value: string): void {
throw new Error('Function not implemented.');
},
setChunkLen: function (value: number): void {
throw new Error('Function not implemented.');
},
@@ -123,6 +129,7 @@ const Provider = ({
const [files, setFiles] = useState<FileItemType[]>([]);
const [successChunks, setSuccessChunks] = useState(0);
const [chunkLen, setChunkLen] = useState(defaultChunkLen);
const [customSplitChar, setCustomSplitChar] = useState<string>();
const [previewFile, setPreviewFile] = useState<FileItemType>();
const [showRePreview, setReShowRePreview] = useState(false);
@@ -198,7 +205,8 @@ const Provider = ({
const { chunks, tokens } = splitText2Chunks({
text: file.rawText,
chunkLen,
overlapRatio: chunkOverlapRatio
overlapRatio: chunkOverlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
return {
@@ -218,7 +226,7 @@ const Provider = ({
title: getErrText(error, t('core.dataset.import.Set Chunk Error'))
});
}
}, [chunkLen, chunkOverlapRatio, t, toast]);
}, [chunkLen, chunkOverlapRatio, customSplitChar, t, toast]);
const reset = useCallback(() => {
setFiles([]);
@@ -246,6 +254,8 @@ const Provider = ({
onclickUpload,
uploading,
chunkLen,
customSplitChar,
setCustomSplitChar,
chunkOverlapRatio,
setChunkLen,
showRePreview,
@@ -405,7 +415,7 @@ export const SelectorContainer = ({
{...(isUnselectedFile
? {}
: {
maxW: ['auto', '500px']
maxW: ['auto', '450px']
})}
p={[4, 8]}
>
@@ -437,7 +447,7 @@ export const SelectorContainer = ({
position={'relative'}
alignItems={'center'}
_hover={{
bg: 'blue.50',
bg: 'primary.50',
'& .delete': {
display: 'block'
}

View File

@@ -9,7 +9,7 @@ import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provide
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useTranslation } from 'next-i18next';
const fileExtension = '.txt, .docx, .pdf, .md';
const fileExtension = '.txt, .docx, .pdf, .md .html';
const QAImport = () => {
const { t } = useTranslation();
@@ -71,11 +71,18 @@ const QAImport = () => {
</Flex>
<Flex mt={3}>
{showRePreview && (
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
<Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
{t('core.dataset.import.Re Preview')}
</Button>
)}
<Button isDisabled={uploading} onClick={openConfirm(() => onclickUpload({ prompt }))}>
<Button
isDisabled={uploading}
onClick={() => {
onReSplitChunks();
openConfirm(() => onclickUpload({ prompt }))();
}}
>
{uploading ? (
<Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
) : (

View File

@@ -80,7 +80,7 @@ const UrlFetchModal = ({
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
<Button variant={'whiteBase'} mr={4} onClick={onClose}>
{t('common.Close')}
</Button>
<Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>

View File

@@ -79,7 +79,7 @@ const WebsiteConfigModal = ({
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'base'} onClick={onClose}>
<Button variant={'whiteBase'} onClick={onClose}>
{t('common.Close')}
</Button>
<Button

View File

@@ -187,12 +187,8 @@ const Info = ({ datasetId }: { datasetId: string }) => {
isLoading={btnLoading}
icon={<DeleteIcon />}
aria-label={''}
variant={'outline'}
size={'sm'}
_hover={{
color: 'red.600',
borderColor: 'red.600'
}}
variant={'whiteDanger'}
size={'smSquare'}
onClick={openConfirm(onclickDelete)}
/>
)}

View File

@@ -313,7 +313,7 @@ const InputDataModal = ({
borderColor={'transparent'}
px={0}
_focus={{
borderColor: 'blue.400',
borderColor: 'primary.400',
px: 3
}}
placeholder={t('dataset.data.Index Placeholder')}
@@ -332,7 +332,7 @@ const InputDataModal = ({
border={theme.borders.base}
cursor={'pointer'}
_hover={{
bg: 'blue.50'
bg: 'primary.50'
}}
minH={'100px'}
onClick={() =>
@@ -351,7 +351,7 @@ const InputDataModal = ({
)}
</Box>
<Flex justifyContent={'flex-end'} px={5} mt={4}>
<Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
<Button variant={'whitePrimary'} mr={3} isLoading={loading} onClick={onClose}>
{t('common.Close')}
</Button>
<MyTooltip label={collection.canWrite ? '' : t('dataset.data.Can not edit')}>

View File

@@ -102,14 +102,14 @@ const Test = ({ datasetId }: { datasetId: string }) => {
py={4}
borderRight={['none', theme.borders.base]}
>
<Box border={'2px solid'} borderColor={'blue.500'} p={3} mx={4} borderRadius={'md'}>
<Box border={'2px solid'} borderColor={'primary.500'} p={3} mx={4} borderRadius={'md'}>
<Flex alignItems={'center'}>
<Box fontSize={'sm'} fontWeight={'bold'} flex={1}>
<MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'blue.600'} />
<MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'primary.600'} />
{t('core.dataset.test.Test Text')}
</Box>
<Button
variant={'base'}
variant={'whitePrimary'}
leftIcon={<MyIcon name={searchModeData.icon as any} w={'14px'} />}
size={'sm'}
onClick={onOpenSelectMode}

View File

@@ -181,11 +181,10 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'blue.500'} />}
icon={<MyIcon name={'backFill'} w={'18px'} color={'primary.500'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
size={'smSquare'}
borderRadius={'50%'}
aria-label={''}
/>

View File

@@ -184,7 +184,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={3} onClick={onClose}>
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
{t('common.Close')}
</Button>
<Button isLoading={creating} onClick={handleSubmit((data) => onclickCreate(data))}>

View File

@@ -79,7 +79,7 @@ const MoveModal = ({
: {
cursor: 'pointer',
_hover: {
color: 'blue.500'
color: 'primary.500'
},
onClick: () => {
setParentId(item.parentId);

View File

@@ -7,7 +7,8 @@ import {
useDisclosure,
Card,
MenuButton,
Image
Image,
Button
} from '@chakra-ui/react';
import { useRouter } from 'next/router';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
@@ -131,8 +132,8 @@ const Kb = () => {
);
return (
<PageContainer isLoading={isFetching}>
<Flex pt={3} px={5} alignItems={'center'}>
<PageContainer isLoading={isFetching} insertProps={{ px: [5, '48px'] }}>
<Flex pt={[4, '30px']} alignItems={'center'} justifyContent={'space-between'}>
{/* url path */}
<ParentPaths
paths={paths.map((path, i) => ({
@@ -161,23 +162,14 @@ const Kb = () => {
offset={[-30, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'blue.500'
}}
>
<Flex
alignItems={'center'}
border={theme.borders.base}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
>
<AddIcon mr={2} />
<Box>{t('Create New')}</Box>
</Flex>
</MenuButton>
<Button variant={'primaryOutline'} px={0}>
<MenuButton h={'100%'}>
<Flex alignItems={'center'} px={'20px'}>
<AddIcon mr={2} />
<Box>{t('Create New')}</Box>
</Flex>
</MenuButton>
</Button>
}
menuList={[
{
@@ -203,25 +195,26 @@ const Kb = () => {
)}
</Flex>
<Grid
p={5}
gridTemplateColumns={['1fr', 'repeat(3,1fr)', 'repeat(4,1fr)', 'repeat(5,1fr)']}
py={5}
gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
gridGap={5}
userSelect={'none'}
>
{formatDatasets.map((dataset) => (
<Card
<Box
display={'flex'}
flexDirection={'column'}
key={dataset._id}
py={3}
px={5}
cursor={'pointer'}
borderWidth={1.5}
borderColor={dragTargetId === dataset._id ? 'primary.600' : 'borderColor.low'}
bg={'white'}
borderRadius={'lg'}
minH={'130px'}
border={theme.borders.md}
boxShadow={'none'}
position={'relative'}
data-drag-id={dataset.type === DatasetTypeEnum.folder ? dataset._id : undefined}
borderColor={dragTargetId === dataset._id ? 'blue.500' : ''}
draggable
onDragStart={(e) => {
setDragStartId(dataset._id);
@@ -250,8 +243,8 @@ const Kb = () => {
setDragTargetId(undefined);
}}
_hover={{
boxShadow: '1px 1px 10px rgba(0,0,0,0.2)',
borderColor: 'transparent',
borderColor: 'primary.300',
boxShadow: '1.5',
'& .delete': {
display: 'block'
}
@@ -287,7 +280,7 @@ const Kb = () => {
h={'22px'}
borderRadius={'md'}
_hover={{
color: 'blue.500',
color: 'primary.500',
'& .icon': {
bg: 'myGray.100'
}
@@ -419,7 +412,7 @@ const Kb = () => {
<MyIcon mr={1} name={dataset.icon as any} w={'12px'} />
<Box color={'myGray.500'}>{t(dataset.label)}</Box>
</Flex>
</Card>
</Box>
))}
</Grid>
{myDatasets.length === 0 && (