fix: mongodb file oversize (#4594)

This commit is contained in:
Archer
2025-04-18 15:01:18 +08:00
committed by GitHub
parent a18d34e40a
commit d9a4a5f3e7
4 changed files with 82 additions and 52 deletions

View File

@@ -16,4 +16,5 @@ weight: 793
## 🐛 修复 ## 🐛 修复
1. 文件上传分块大小限制,避免超出 MongoDB 限制。

View File

@@ -65,9 +65,10 @@ export async function uploadFile({
const bucket = getGridBucket(bucketName); const bucket = getGridBucket(bucketName);
const fileSize = stats.size; const fileSize = stats.size;
// 单块大小:尽可能大,但不超过 14MB不小于512KB
const chunkSizeBytes = (() => { const chunkSizeBytes = (() => {
// 计算理想块大小:文件大小 ÷ 目标块数(10) // 计算理想块大小:文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
const idealChunkSize = Math.ceil(fileSize / 10); const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
// 确保块大小至少为512KB // 确保块大小至少为512KB
const minChunkSize = 512 * 1024; // 512KB const minChunkSize = 512 * 1024; // 512KB

View File

@@ -16,6 +16,7 @@ import { ImportSourceItemType } from '@/web/core/dataset/type';
import { useI18n } from '@/web/context/I18n'; import { useI18n } from '@/web/context/I18n';
import { useContextSelector } from 'use-context-selector'; import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { getErrText } from '@fastgpt/global/common/error/utils';
export type SelectFileItemType = { export type SelectFileItemType = {
fileId: string; fileId: string;
@@ -71,39 +72,53 @@ const FileSelector = ({
{ {
await Promise.all( await Promise.all(
files.map(async ({ fileId, file }) => { files.map(async ({ fileId, file }) => {
const { fileId: uploadFileId } = await uploadFile2DB({ try {
file, const { fileId: uploadFileId } = await uploadFile2DB({
bucketName: BucketNameEnum.dataset, file,
data: { bucketName: BucketNameEnum.dataset,
datasetId data: {
}, datasetId
percentListen: (e) => { },
setSelectFiles((state) => percentListen: (e) => {
state.map((item) => setSelectFiles((state) =>
item.id === fileId state.map((item) =>
? { item.id === fileId
...item, ? {
uploadedFileRate: item.uploadedFileRate ...item,
? Math.max(e, item.uploadedFileRate) uploadedFileRate: item.uploadedFileRate
: e ? Math.max(e, item.uploadedFileRate)
} : e
: item }
) : item
); )
} );
}); }
setSelectFiles((state) => });
state.map((item) => setSelectFiles((state) =>
item.id === fileId state.map((item) =>
? { item.id === fileId
...item, ? {
dbFileId: uploadFileId, ...item,
isUploading: false, dbFileId: uploadFileId,
uploadedFileRate: 100 isUploading: false,
} uploadedFileRate: 100
: item }
) : item
); )
);
} catch (error) {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
isUploading: false,
errorMsg: getErrText(error)
}
: item
)
);
}
}) })
); );
} }

View File

@@ -9,12 +9,16 @@ import {
Td, Td,
Tbody, Tbody,
Progress, Progress,
IconButton IconButton,
Box
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import { ImportSourceItemType } from '@/web/core/dataset/type.d'; import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import { useI18n } from '@/web/context/I18n'; import { useI18n } from '@/web/context/I18n';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
export const RenderUploadFiles = ({ export const RenderUploadFiles = ({
files, files,
@@ -56,22 +60,31 @@ export const RenderUploadFiles = ({
</Flex> </Flex>
</Td> </Td>
<Td> <Td>
<Flex alignItems={'center'} fontSize={'xs'}> {item.errorMsg ? (
<Progress <MyTooltip label={item.errorMsg}>
value={item.uploadedFileRate} <MyTag colorSchema={'red'}>
h={'6px'} <Box mr={1}>{t('common:common.Error')}</Box>
w={'100%'} <MyIcon name={'help'} w={'0.9rem'} color={'red.500'} />
maxW={'210px'} </MyTag>
size="sm" </MyTooltip>
borderRadius={'20px'} ) : (
colorScheme={(item.uploadedFileRate || 0) >= 100 ? 'green' : 'blue'} <Flex alignItems={'center'} fontSize={'xs'}>
bg="myGray.200" <Progress
hasStripe value={item.uploadedFileRate}
isAnimated h={'6px'}
mr={2} w={'100%'}
/> maxW={'210px'}
{`${item.uploadedFileRate}%`} size="sm"
</Flex> borderRadius={'20px'}
colorScheme={(item.uploadedFileRate || 0) >= 100 ? 'green' : 'blue'}
bg="myGray.200"
hasStripe
isAnimated
mr={2}
/>
{`${item.uploadedFileRate}%`}
</Flex>
)}
</Td> </Td>
<Td>{item.sourceSize}</Td> <Td>{item.sourceSize}</Td>
<Td> <Td>