From 44e772f0fd1cb824e1c8db947acf48c27c0fc62c Mon Sep 17 00:00:00 2001 From: archer <545436317@qq.com> Date: Mon, 4 Sep 2023 10:51:57 +0800 Subject: [PATCH] feat: file relate kb --- client/src/pages/api/plugins/file/read.ts | 3 +-- client/src/pages/api/plugins/file/upload.ts | 27 +++++++++++++------ client/src/pages/api/plugins/kb/delete.ts | 21 +++++++-------- .../detail/components/Import/FileSelect.tsx | 4 ++- client/src/pages/kb/list/index.tsx | 2 +- client/src/service/lib/gridfs.ts | 12 ++++++++- client/src/utils/file.ts | 7 ++++- 7 files changed, 50 insertions(+), 26 deletions(-) diff --git a/client/src/pages/api/plugins/file/read.ts b/client/src/pages/api/plugins/file/read.ts index 4da034948..232f94829 100644 --- a/client/src/pages/api/plugins/file/read.ts +++ b/client/src/pages/api/plugins/file/read.ts @@ -26,8 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const encoding = jschardet.detect(buffer)?.encoding; - res.setHeader('encoding', encoding); - res.setHeader('Content-Type', file.contentType); + res.setHeader('Content-Type', `${file.contentType}; charset=${encoding}`); res.setHeader('Cache-Control', 'public, max-age=3600'); res.setHeader('Content-Disposition', `inline; filename="${encodeURIComponent(file.filename)}"`); diff --git a/client/src/pages/api/plugins/file/upload.ts b/client/src/pages/api/plugins/file/upload.ts index efeccf310..b130a2dd7 100644 --- a/client/src/pages/api/plugins/file/upload.ts +++ b/client/src/pages/api/plugins/file/upload.ts @@ -38,7 +38,7 @@ class UploadModel { }).any(); async doUpload(req: NextApiRequest, res: NextApiResponse) { - return new Promise<{ files: FileType[] }>((resolve, reject) => { + return new Promise<{ files: FileType[]; metadata: Record }>((resolve, reject) => { // @ts-ignore this.uploader(req, res, (error) => { if (error) { @@ -46,11 +46,22 @@ class UploadModel { } resolve({ - // @ts-ignore - files: req.files?.map((file) => ({ - ...file, - originalname: decodeURIComponent(file.originalname) - })) + files: + // @ts-ignore + req.files?.map((file) => ({ + ...file, + originalname: decodeURIComponent(file.originalname) + })) || [], + metadata: (() => { + if (!req.body?.metadata) return {}; + try { + return JSON.parse(req.body.metadata); + } catch (error) { + console.log(error); + + return {}; + } + })() }); }); }); @@ -64,7 +75,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await connectToDatabase(); const { userId } = await authUser({ req, authToken: true }); - const { files = [] } = await upload.doUpload(req, res); + const { files, metadata } = await upload.doUpload(req, res); const gridFs = new GridFSStorage('dataset', userId); @@ -74,8 +85,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< path: file.path, filename: file.originalname, metadata: { + ...metadata, contentType: file.mimetype, - encoding: file.encoding, userId } }) diff --git a/client/src/pages/api/plugins/kb/delete.ts b/client/src/pages/api/plugins/kb/delete.ts index defbef9d8..3b2d27d32 100644 --- a/client/src/pages/api/plugins/kb/delete.ts +++ b/client/src/pages/api/plugins/kb/delete.ts @@ -5,6 +5,7 @@ import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; import { Types } from 'mongoose'; import { PgTrainingTableName } from '@/constants/plugin'; +import { GridFSStorage } from '@/service/lib/gridfs'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -21,24 +22,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await connectToDatabase(); - // delete all pg data - await PgClient.delete(PgTrainingTableName, { - where: [['user_id', userId], 'AND', ['kb_id', id]] - }); - // delete training data await TrainingData.deleteMany({ userId, kbId: id }); - // delete related app - await App.updateMany( - { - userId - }, - { $pull: { 'chat.relatedKbs': new Types.ObjectId(id) } } - ); + // delete all pg data + await PgClient.delete(PgTrainingTableName, { + where: [['user_id', userId], 'AND', ['kb_id', id]] + }); + + // delete related files + const gridFs = new GridFSStorage('dataset', userId); + await gridFs.deleteFilesByKbId(id); // delete kb data await KB.findOneAndDelete({ diff --git a/client/src/pages/kb/detail/components/Import/FileSelect.tsx b/client/src/pages/kb/detail/components/Import/FileSelect.tsx index 85d3789b9..a1e92c168 100644 --- a/client/src/pages/kb/detail/components/Import/FileSelect.tsx +++ b/client/src/pages/kb/detail/components/Import/FileSelect.tsx @@ -19,6 +19,7 @@ import dynamic from 'next/dynamic'; import MyTooltip from '@/components/MyTooltip'; import { FetchResultItem, DatasetItemType } from '@/types/plugin'; import { getErrText } from '@/utils/tools'; +import { useUserStore } from '@/store/user'; const UrlFetchModal = dynamic(() => import('./UrlFetchModal')); const CreateFileModal = dynamic(() => import('./CreateFileModal')); @@ -54,6 +55,7 @@ const FileSelect = ({ showCreateFile = true, ...props }: Props) => { + const { kbDetail } = useUserStore(); const { Loading: FileSelectLoading } = useLoading(); const { t } = useTranslation(); @@ -109,7 +111,7 @@ const FileSelect = ({ } return ''; })(), - uploadFiles([file], (percent) => { + uploadFiles([file], { kbId: kbDetail._id }, (percent) => { if (percent < 100) { setSelectingText( t('file.Uploading', { name: file.name.slice(0, 20), percent }) || '' diff --git a/client/src/pages/kb/list/index.tsx b/client/src/pages/kb/list/index.tsx index 110a2447b..d145f8b5f 100644 --- a/client/src/pages/kb/list/index.tsx +++ b/client/src/pages/kb/list/index.tsx @@ -31,7 +31,7 @@ const Kb = () => { const { toast } = useToast(); const { openConfirm, ConfirmModal } = useConfirm({ title: '删除提示', - content: '确认删除该知识库？' + content: '确认删除该知识库？知识库相关的文件、记录将永久删除，无法恢复！' }); const { myKbList, loadKbList, setKbList } = useUserStore(); diff --git a/client/src/service/lib/gridfs.ts b/client/src/service/lib/gridfs.ts index f6fe10e18..3804091f7 100644 --- a/client/src/service/lib/gridfs.ts +++ b/client/src/service/lib/gridfs.ts @@ -79,7 +79,7 @@ export class GridFSStorage { return Promise.reject(`file not found`); } - if (String(file.metadata?.userId) !== this.uid) { + if (file.metadata?.userId !== this.uid) { return Promise.reject(ERROR_ENUM.unAuthFile); } @@ -100,6 +100,16 @@ export class GridFSStorage { return true; } + async deleteFilesByKbId(kbId: string) { + if (!kbId) return; + const bucket = this.GridFSBucket(); + const files = await bucket + .find({ ['metadata.kbId']: kbId, ['metadata.userId']: this.uid }, { projection: { _id: 1 } }) + .toArray(); + + return Promise.all(files.map((file) => this.delete(String(file._id)))); + } + async download(id: string) { await this.findAndAuthFile(id); diff --git a/client/src/utils/file.ts b/client/src/utils/file.ts index 502013835..008d50607 100644 --- a/client/src/utils/file.ts +++ b/client/src/utils/file.ts @@ -7,8 +7,13 @@ import { uploadImg, postUploadFiles } from '@/api/system'; /** * upload file to mongo gridfs */ -export const uploadFiles = (files: File[], percentListen?: (percent: number) => void) => { +export const uploadFiles = ( + files: File[], + metadata: Record = {}, + percentListen?: (percent: number) => void +) => { const form = new FormData(); + form.append('metadata', JSON.stringify(metadata)); files.forEach((file) => { form.append('file', file, encodeURIComponent(file.name)); });