ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-02 20:58:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

External dataset (#1519)

Browse Source 
* perf: local file create collection

* rename middleware

* perf: remove code

* feat: next14

* feat: external file dataset

* collection tags field

* external file dataset doc

* fix: ts
This commit is contained in:
Archer
2024-05-17 16:44:15 +08:00
committed by GitHub
parent 2d1ec9b3ad
commit 67c52992d7
102 changed files with 1839 additions and 1282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx
View File

@@ -84,7 +84,7 @@ const Header = ({}: {}) => {
...props
}: {
name: string;
type: `${DatasetCollectionTypeEnum}`;
type: DatasetCollectionTypeEnum;
callback?: (id: string) => void;
trainingType?: TrainingModeEnum;
rawLink?: string;

45
projects/app/src/pages/dataset/detail/components/DataCard.tsx
View File

@@ -38,16 +38,14 @@ import { TabEnum } from '..';
import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import {
DatasetCollectionTypeMap,
TrainingModeEnum,
TrainingTypeMap
} from '@fastgpt/global/core/dataset/constants';
import { DatasetCollectionTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { getFileAndOpen } from '@/web/core/dataset/utils';
import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
import MyTooltip from '@/components/MyTooltip';
import { usePagination } from '@fastgpt/web/hooks/usePagination';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import { useI18n } from '@/web/context/I18n';
const DataCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
@@ -62,6 +60,7 @@ const DataCard = () => {
};
const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
const { t } = useTranslation();
const { datasetT } = useI18n();
const [searchText, setSearchText] = useState('');
const { toast } = useToast();
const { openConfirm, ConfirmModal } = useConfirm({
@@ -69,6 +68,7 @@ const DataCard = () => {
type: 'delete'
});
const { isOpen, onOpen, onClose } = useDisclosure();
const readSource = getCollectionSourceAndOpen(collectionId);
const {
data: datasetDataList,
@@ -169,7 +169,17 @@ const DataCard = () => {
value: webSelector
}
]
: [])
: []),
{
...(collection.tags
? [
{
label: datasetT('Collection tags'),
value: collection.tags?.join(', ') || '-'
}
]
: [])
}
];
}, [collection, t]);
@@ -196,13 +206,15 @@ const DataCard = () => {
/>
<Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
<Box lineHeight={1.2}>
<RawSourceBox
sourceName={collection?.name}
sourceId={collection?.fileId || collection?.rawLink}
fontSize={['md', 'lg']}
color={'black'}
textDecoration={'none'}
/>
{collection?._id && (
<RawSourceBox
collectionId={collection._id}
{...getCollectionSourceData(collection)}
fontSize={['md', 'lg']}
color={'black'}
textDecoration={'none'}
/>
)}
<Box fontSize={'sm'} color={'myGray.500'}>
{t('core.dataset.collection.id')}:{' '}
<Box as={'span'} userSelect={'all'}>
@@ -412,10 +424,7 @@ const DataCard = () => {
</Flex>
))}
{collection?.sourceId && (
<Button
variant={'whitePrimary'}
onClick={() => collection.sourceId && getFileAndOpen(collection.sourceId)}
>
<Button variant={'whitePrimary'} onClick={readSource}>
{t('core.dataset.collection.metadata.read source')}
</Button>
)}

9
projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx
View File

@@ -15,12 +15,12 @@ import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useRouter } from 'next/router';
import { TabEnum } from '../../../index';
import {
postCreateDatasetCsvTableCollection,
postCreateDatasetExternalFileCollection,
postCreateDatasetFileCollection,
postCreateDatasetLinkCollection,
postCreateDatasetTextCollection
@@ -95,6 +95,13 @@ const Upload = () => {
...commonParams,
fileId: item.dbFileId
});
} else if (importSource === ImportDataSourceEnum.externalFile && item.externalFileUrl) {
await postCreateDatasetExternalFileCollection({
...commonParams,
externalFileUrl: item.externalFileUrl,
externalFileId: item.externalFileId,
filename: item.sourceName
});
}
setSources((state) =>

6
projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx
View File

@@ -44,7 +44,8 @@ const PreviewChunks = ({
if (importSource === ImportDataSourceEnum.csvTable) {
return getPreviewChunks({
type: importType2ReadType(importSource),
sourceId: previewSource.dbFileId || previewSource.link || previewSource.sourceUrl || '',
sourceId:
previewSource.dbFileId || previewSource.link || previewSource.externalFileUrl || '',
chunkSize,
overlapRatio: chunkOverlapRatio,
customSplitChar: processParamsForm.getValues('customSplitChar'),
@@ -55,7 +56,8 @@ const PreviewChunks = ({
return getPreviewChunks({
type: importType2ReadType(importSource),
sourceId: previewSource.dbFileId || previewSource.link || previewSource.sourceUrl || '',
sourceId:
previewSource.dbFileId || previewSource.link || previewSource.externalFileUrl || '',
chunkSize,
overlapRatio: chunkOverlapRatio,
customSplitChar: processParamsForm.getValues('customSplitChar'),

5
projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx
View File

@@ -22,7 +22,7 @@ const PreviewRawText = ({
const { importSource, processParamsForm } = useContextSelector(DatasetImportContext, (v) => v);
const { data, isLoading } = useQuery(
['previewSource', previewSource.dbFileId, previewSource.link, previewSource.sourceUrl],
['previewSource', previewSource.dbFileId, previewSource.link, previewSource.externalFileUrl],
() => {
if (importSource === ImportDataSourceEnum.fileCustom && previewSource.rawText) {
return {
@@ -39,7 +39,8 @@ const PreviewRawText = ({
return getPreviewFileContent({
type: importType2ReadType(importSource),
sourceId: previewSource.dbFileId || previewSource.link || previewSource.sourceUrl || '',
sourceId:
previewSource.dbFileId || previewSource.link || previewSource.externalFileUrl || '',
isQAImport: false,
selector: processParamsForm.getValues('webSelector')
});

38
projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx
View File

@@ -50,16 +50,16 @@ const CustomLinkInput = () => {
const { register, reset, handleSubmit, control } = useForm<{
list: {
sourceName: string;
sourceUrl: string;
externalId: string;
externalFileUrl: string;
externalFileId: string;
}[];
}>({
defaultValues: {
list: [
{
sourceName: '',
sourceUrl: '',
externalId: ''
externalFileUrl: '',
externalFileId: ''
}
]
}
@@ -80,8 +80,8 @@ const CustomLinkInput = () => {
reset({
list: sources.map((item) => ({
sourceName: item.sourceName,
sourceUrl: item.sourceUrl || '',
externalId: item.externalId || ''
externalFileUrl: item.externalFileUrl || '',
externalFileId: item.externalFileId || ''
}))
});
}
@@ -104,7 +104,7 @@ const CustomLinkInput = () => {
<Tr key={item.id}>
<Td>
<Input
{...register(`list.${index}.sourceUrl`, {
{...register(`list.${index}.externalFileUrl`, {
required: index !== list.length - 1,
onBlur(e) {
const val = (e.target.value || '') as string;
@@ -112,15 +112,15 @@ const CustomLinkInput = () => {
const sourceName = val.split('/').pop() || '';
update(index, {
...list[index],
sourceUrl: val,
externalFileUrl: val,
sourceName: decodeURIComponent(sourceName)
});
}
if (val && index === list.length - 1) {
append({
sourceName: '',
sourceUrl: '',
externalId: ''
externalFileUrl: '',
externalFileId: ''
});
}
}
@@ -128,7 +128,7 @@ const CustomLinkInput = () => {
/>
</Td>
<Td>
<Input {...register(`list.${index}.externalId`)} />
<Input {...register(`list.${index}.externalFileId`)} />
</Td>
<Td>
<Input {...register(`list.${index}.sourceName`)} />
@@ -154,26 +154,26 @@ const CustomLinkInput = () => {
onClick={() => {
append({
sourceName: '',
sourceUrl: '',
externalId: ''
externalFileUrl: '',
externalFileId: ''
});
}}
>
{commonT('Add new')}
</Button>
<Button
isDisabled={list.filter((item) => !!item.sourceUrl).length === 0}
isDisabled={list.filter((item) => !!item.externalFileUrl).length === 0}
onClick={handleSubmit((data) => {
setSources(
data.list
.filter((item) => !!item.sourceUrl)
.filter((item) => !!item.externalFileUrl)
.map((item) => ({
id: getNanoid(32),
createStatus: 'waiting',
sourceName: item.sourceName || item.sourceUrl,
icon: getFileIcon(item.sourceUrl),
externalId: item.externalId,
sourceUrl: item.sourceUrl
sourceName: item.sourceName || item.externalFileUrl,
icon: getFileIcon(item.externalFileUrl),
externalFileId: item.externalFileId,
externalFileUrl: item.externalFileUrl
}))
);

11
projects/app/src/pages/dataset/detail/components/Info.tsx
View File

@@ -1,10 +1,9 @@
import React, { useState, useMemo } from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, Button, IconButton, Input, Textarea } from '@chakra-ui/react';
import { Box, Flex, Button, IconButton, Input, Textarea, HStack } from '@chakra-ui/react';
import { DeleteIcon } from '@chakra-ui/icons';
import { delDatasetById } from '@/web/core/dataset/api';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useForm } from 'react-hook-form';
import { compressImgFileAndUpload } from '@/web/common/file/controller';
@@ -24,6 +23,7 @@ import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import MyDivider from '@fastgpt/web/components/common/MyDivider/index';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
const Info = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
@@ -191,9 +191,10 @@ const Info = ({ datasetId }: { datasetId: string }) => {
{datasetDetail.type === DatasetTypeEnum.externalFile && (
<>
<Flex w={'100%'} alignItems={'center'}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{datasetT('External read url')}
</Box>
<HStack fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
<Box>{datasetT('External read url')}</Box>
<QuestionTip label={datasetT('External read url tip')} />
</HStack>
<Input
flex={[1, '0 0 320px']}
placeholder="https://test.com/read?fileId={{fileId}}"

1
projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
View File

@@ -237,6 +237,7 @@ const InputDataModal = ({
w={'210px'}
className="textEllipsis3"
whiteSpace={'pre-wrap'}
collectionId={collection._id}
sourceName={collection.sourceName}
sourceId={collection.sourceId}
mb={6}

12
projects/app/src/pages/dataset/list/component/CreateModal.tsx
View File

@@ -116,13 +116,13 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
value: DatasetTypeEnum.websiteDataset,
icon: 'core/dataset/websiteDataset',
desc: datasetT('Website Dataset Desc')
},
{
title: datasetT('External File'),
value: DatasetTypeEnum.externalFile,
icon: 'core/dataset/externalDataset',
desc: datasetT('External file Dataset Desc')
}
// {
// title: datasetT('External File'),
// value: DatasetTypeEnum.externalFile,
// icon: 'core/dataset/websiteDataset',
// desc: datasetT('External file Dataset Desc')
// }
]
: [])
]}