mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-27 16:33:49 +00:00

* Feat: Images dataset collection (#4941) * New pic (#4858) * 更新数据集相关类型,添加图像文件ID和预览URL支持;优化数据集导入功能,新增图像数据集处理组件;修复部分国际化文本;更新文件上传逻辑以支持新功能。 * 与原先代码的差别 * 新增 V4.9.10 更新说明,支持 PG 设置`systemEnv.hnswMaxScanTuples`参数,优化 LLM stream 调用超时,修复全文检索多知识库排序问题。同时更新数据集索引,移除 datasetId 字段以简化查询。 * 更换成fileId_image逻辑,并增加训练队列匹配的逻辑 * 新增图片集合判断逻辑,优化预览URL生成流程,确保仅在数据集为图片集合时生成预览URL,并添加相关日志输出以便调试。 * Refactor Docker Compose configuration to comment out exposed ports for production environments, update image versions for pgvector, fastgpt, and mcp_server, and enhance Redis service with a health check. Additionally, standardize dataset collection labels in constants and improve internationalization strings across multiple languages. * Enhance TrainingStates component by adding internationalization support for the imageParse training mode and update defaultCounts to include imageParse mode in trainingDetail API. * Enhance dataset import context by adding additional steps for image dataset import process and improve internationalization strings for modal buttons in the useEditTitle hook. * Update DatasetImportContext to conditionally render MyStep component based on data source type, improving the import process for non-image datasets. * Refactor image dataset handling by improving internationalization strings, enhancing error messages, and streamlining the preview URL generation process. * 图片上传到新建的 dataset_collection_images 表,逻辑跟随更改 * 修改了除了controller的其他部分问题 * 把图片数据集的逻辑整合到controller里面 * 补充i18n * 补充i18n * resolve评论:主要是上传逻辑的更改和组件复用 * 图片名称的图标显示 * 修改编译报错的命名问题 * 删除不需要的collectionid部分 * 多余文件的处理和改动一个删除按钮 * 除了loading和统一的imageId,其他都resolve掉的 * 处理图标报错 * 复用了MyPhotoView并采用全部替换的方式将imageFileId变成imageId * 去除不必要文件修改 * 报错和字段修改 * 增加上传成功后删除临时文件的逻辑以及回退一些修改 * 删除path字段,将图片保存到gridfs内,并修改增删等操作的代码 * 修正编译错误 --------- Co-authored-by: archer <545436317@qq.com> * perf: image dataset * feat: insert image * perf: image icon * fix: training state --------- Co-authored-by: Zhuangzai fa <143257420+ctrlz526@users.noreply.github.com> * fix: ts (#4948) * Thirddatasetmd (#4942) * add thirddataset.md * fix thirddataset.md * fix * delete wrong png --------- Co-authored-by: dreamer6680 <146868355@qq.com> * perf: api dataset code * perf: log * add secondary.tsx (#4946) * add secondary.tsx * fix --------- Co-authored-by: dreamer6680 <146868355@qq.com> * perf: multiple menu * perf: i18n * feat: parse queue (#4960) * feat: parse queue * feat: sync parse queue * fix thirddataset.md (#4962) * fix thirddataset-4.png (#4963) * feat: Dataset template import (#4934) * 模版导入部分除了文档还没写 * 修复模版导入的 build 错误 * Document production * compress pictures * Change some constants to variables --------- Co-authored-by: Archer <545436317@qq.com> * perf: template import * doc * llm pargraph * bocha tool * fix: del collection --------- Co-authored-by: Zhuangzai fa <143257420+ctrlz526@users.noreply.github.com> Co-authored-by: dreamer6680 <1468683855@qq.com> Co-authored-by: dreamer6680 <146868355@qq.com>
152 lines
4.1 KiB
TypeScript
152 lines
4.1 KiB
TypeScript
import type { NextApiRequest, NextApiResponse } from 'next';
|
|
import multer from 'multer';
|
|
import path from 'path';
|
|
import type { BucketNameEnum } from '@fastgpt/global/common/file/constants';
|
|
import { bucketNameMap } from '@fastgpt/global/common/file/constants';
|
|
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
|
|
|
export type FileType = {
|
|
fieldname: string;
|
|
originalname: string;
|
|
encoding: string;
|
|
mimetype: string;
|
|
filename: string;
|
|
path: string;
|
|
size: number;
|
|
};
|
|
|
|
/*
|
|
maxSize: File max size (MB)
|
|
*/
|
|
export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
|
|
maxSize *= 1024 * 1024;
|
|
|
|
class UploadModel {
|
|
uploaderSingle = multer({
|
|
limits: {
|
|
fieldSize: maxSize
|
|
},
|
|
preservePath: true,
|
|
storage: multer.diskStorage({
|
|
// destination: (_req, _file, cb) => {
|
|
// cb(null, tmpFileDirPath);
|
|
// },
|
|
filename: (req, file, cb) => {
|
|
if (!file?.originalname) {
|
|
cb(new Error('File not found'), '');
|
|
} else {
|
|
const { ext } = path.parse(decodeURIComponent(file.originalname));
|
|
cb(null, `${getNanoid()}${ext}`);
|
|
}
|
|
}
|
|
})
|
|
}).single('file');
|
|
async getUploadFile<T = any>(
|
|
req: NextApiRequest,
|
|
res: NextApiResponse,
|
|
originBucketName?: `${BucketNameEnum}`
|
|
) {
|
|
return new Promise<{
|
|
file: FileType;
|
|
metadata: Record<string, any>;
|
|
data: T;
|
|
bucketName?: `${BucketNameEnum}`;
|
|
}>((resolve, reject) => {
|
|
// @ts-ignore
|
|
this.uploaderSingle(req, res, (error) => {
|
|
if (error) {
|
|
return reject(error);
|
|
}
|
|
|
|
// check bucket name
|
|
const bucketName = (req.body?.bucketName || originBucketName) as `${BucketNameEnum}`;
|
|
if (bucketName && !bucketNameMap[bucketName]) {
|
|
return reject('BucketName is invalid');
|
|
}
|
|
|
|
// @ts-ignore
|
|
const file = req.file as FileType;
|
|
|
|
resolve({
|
|
file: {
|
|
...file,
|
|
originalname: decodeURIComponent(file.originalname)
|
|
},
|
|
bucketName,
|
|
metadata: (() => {
|
|
if (!req.body?.metadata) return {};
|
|
try {
|
|
return JSON.parse(req.body.metadata);
|
|
} catch (error) {
|
|
return {};
|
|
}
|
|
})(),
|
|
data: (() => {
|
|
if (!req.body?.data) return {};
|
|
try {
|
|
return JSON.parse(req.body.data);
|
|
} catch (error) {
|
|
return {};
|
|
}
|
|
})()
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
uploaderMultiple = multer({
|
|
limits: {
|
|
fieldSize: maxSize
|
|
},
|
|
preservePath: true,
|
|
storage: multer.diskStorage({
|
|
// destination: (_req, _file, cb) => {
|
|
// cb(null, tmpFileDirPath);
|
|
// },
|
|
filename: (req, file, cb) => {
|
|
if (!file?.originalname) {
|
|
cb(new Error('File not found'), '');
|
|
} else {
|
|
const { ext } = path.parse(decodeURIComponent(file.originalname));
|
|
cb(null, `${getNanoid()}${ext}`);
|
|
}
|
|
}
|
|
})
|
|
}).array('file', global.feConfigs?.uploadFileMaxSize);
|
|
async getUploadFiles<T = any>(req: NextApiRequest, res: NextApiResponse) {
|
|
return new Promise<{
|
|
files: FileType[];
|
|
data: T;
|
|
}>((resolve, reject) => {
|
|
// @ts-ignore
|
|
this.uploaderMultiple(req, res, (error) => {
|
|
if (error) {
|
|
console.log(error);
|
|
return reject(error);
|
|
}
|
|
|
|
// @ts-ignore
|
|
const files = req.files as FileType[];
|
|
|
|
resolve({
|
|
files: files.map((file) => ({
|
|
...file,
|
|
originalname: decodeURIComponent(file.originalname)
|
|
})),
|
|
data: (() => {
|
|
if (!req.body?.data) return {};
|
|
try {
|
|
return JSON.parse(req.body.data);
|
|
} catch (error) {
|
|
return {};
|
|
}
|
|
})()
|
|
});
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
return new UploadModel();
|
|
};
|