From a19afca148bfbc2105eeb7f6a4d511bb5522843b Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Wed, 13 Sep 2023 17:00:17 +0800 Subject: [PATCH] v4.4.1 (#294) * move file * perf: dataset file manage * v441 description * fix: qa csv update file * feat: rename file * frontend show system-version --- client/package.json | 4 +- client/public/locales/en/common.json | 4 + client/public/locales/zh/common.json | 4 + client/src/api/core/dataset/file.d.ts | 8 + client/src/api/core/dataset/file.ts | 15 ++ client/src/api/plugins/kb.ts | 28 +-- client/src/api/request/kb.d.ts | 5 - client/src/api/support/file.ts | 18 ++ client/src/api/system.ts | 16 -- client/src/components/Layout/navbar.tsx | 15 +- client/src/components/MyInput/index.tsx | 2 +- client/src/constants/plugin.ts | 2 +- .../{useEditInfo.tsx => useEditTitle.tsx} | 2 +- client/src/pages/account/components/Info.tsx | 55 +++-- client/src/pages/api/admin/initv43.ts | 6 +- client/src/pages/api/admin/initv44.ts | 4 +- client/src/pages/api/admin/initv441.ts | 35 ++++ .../dataset/file/delById.ts} | 6 +- .../api/core/dataset/file/delEmptyFiles.ts | 31 +++ .../dataset/file/detail.ts} | 0 .../{plugins/kb => core/dataset}/file/list.ts | 32 +-- .../src/pages/api/core/dataset/file/update.ts | 66 ++++++ .../src/pages/api/openapi/kb/delDataById.ts | 4 +- client/src/pages/api/openapi/kb/pushData.ts | 4 +- client/src/pages/api/openapi/kb/searchTest.ts | 4 +- client/src/pages/api/openapi/kb/updateData.ts | 4 +- .../api/plugins/kb/data/exportModelData.ts | 18 +- .../pages/api/plugins/kb/data/getDataById.ts | 4 +- .../pages/api/plugins/kb/data/getDataList.ts | 6 +- .../pages/api/plugins/kb/data/insertData.ts | 4 +- client/src/pages/api/plugins/kb/delete.ts | 6 +- .../api/plugins/kb/file/deleteEmptyFiles.ts | 59 ------ .../api/{plugins => support}/file/delete.ts | 0 .../api/{plugins => support}/file/read.ts | 0 .../api/{plugins => support}/file/readUrl.ts | 2 +- .../api/{plugins => support}/file/upload.ts | 0 client/src/pages/api/system/getInitData.ts | 4 +- .../chat/components/ChatHistorySlider.tsx | 4 +- .../pages/kb/detail/components/DataCard.tsx | 69 ++----- .../pages/kb/detail/components/FileCard.tsx | 188 +++++++++++++----- .../kb/detail/components/Import/Chunk.tsx | 11 + .../pages/kb/detail/components/Import/Csv.tsx | 11 + .../detail/components/Import/FileSelect.tsx | 2 +- .../pages/kb/detail/components/Import/QA.tsx | 11 + .../kb/detail/components/InputDataModal.tsx | 2 +- client/src/pages/kb/detail/index.tsx | 5 +- client/src/pages/kb/list/index.tsx | 45 ++++- client/src/service/lib/gridfs.ts | 3 + .../src/service/moduleDispatch/kb/search.ts | 4 +- client/src/service/pg.ts | 12 +- client/src/store/static.ts | 2 + client/src/utils/file.ts | 2 +- .../docs/installation/upgrading/44 copy.md | 23 +++ 53 files changed, 570 insertions(+), 301 deletions(-) create mode 100644 client/src/api/core/dataset/file.d.ts create mode 100644 client/src/api/core/dataset/file.ts create mode 100644 client/src/api/support/file.ts rename client/src/hooks/{useEditInfo.tsx => useEditTitle.tsx} (98%) create mode 100644 client/src/pages/api/admin/initv441.ts rename client/src/pages/api/{plugins/kb/file/delFileByFileId.ts => core/dataset/file/delById.ts} (89%) create mode 100644 client/src/pages/api/core/dataset/file/delEmptyFiles.ts rename client/src/pages/api/{plugins/kb/file/getFileInfo.ts => core/dataset/file/detail.ts} (100%) rename client/src/pages/api/{plugins/kb => core/dataset}/file/list.ts (79%) create mode 100644 client/src/pages/api/core/dataset/file/update.ts delete mode 100644 client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts rename client/src/pages/api/{plugins => support}/file/delete.ts (100%) rename client/src/pages/api/{plugins => support}/file/read.ts (100%) rename client/src/pages/api/{plugins => support}/file/readUrl.ts (97%) rename client/src/pages/api/{plugins => support}/file/upload.ts (100%) create mode 100644 docSite/content/docs/installation/upgrading/44 copy.md diff --git a/client/package.json b/client/package.json index 2eafc6251..608546db3 100644 --- a/client/package.json +++ b/client/package.json @@ -1,7 +1,7 @@ { "name": "fastgpt", - "version": "3.7", - "private": true, + "version": "4.2.1", + "private": false, "scripts": { "dev": "next dev", "build": "next build", diff --git a/client/public/locales/en/common.json b/client/public/locales/en/common.json index bce86cd63..5bdfd3797 100644 --- a/client/public/locales/en/common.json +++ b/client/public/locales/en/common.json @@ -4,6 +4,7 @@ "Confirm": "Yes", "Create New": "Create", "Dataset": "Dataset", + "Export": "Export", "Folder": "Folder", "Move": "Move", "Name": "Name", @@ -215,6 +216,9 @@ "Response Detail": "Detail", "Response Detail tips": "Whether detailed data such as references and full context need to be returned" }, + "system": { + "Help Document": "Document" + }, "user": { "Account": "Account", "Amount of earnings": "Earnings", diff --git a/client/public/locales/zh/common.json b/client/public/locales/zh/common.json index 733003569..e4a84628c 100644 --- a/client/public/locales/zh/common.json +++ b/client/public/locales/zh/common.json @@ -4,6 +4,7 @@ "Confirm": "确认", "Create New": "新建", "Dataset": "知识库", + "Export": "导出", "Folder": "文件夹", "Move": "移动", "Name": "名称", @@ -215,6 +216,9 @@ "Response Detail": "返回详情", "Response Detail tips": "是否需要返回引用、完整上下文等详细数据" }, + "system": { + "Help Document": "帮助文档" + }, "user": { "Account": "账号", "Amount of earnings": "收益（￥）", diff --git a/client/src/api/core/dataset/file.d.ts b/client/src/api/core/dataset/file.d.ts new file mode 100644 index 000000000..4cd2283d8 --- /dev/null +++ b/client/src/api/core/dataset/file.d.ts @@ -0,0 +1,8 @@ +import { RequestPaging } from '../../../types/index'; + +export type GetFileListProps = RequestPaging & { + kbId: string; + searchText: string; +}; + +export type UpdateFileProps = { id: string; name?: string; datasetUsed?: boolean }; diff --git a/client/src/api/core/dataset/file.ts b/client/src/api/core/dataset/file.ts new file mode 100644 index 000000000..6198ebf20 --- /dev/null +++ b/client/src/api/core/dataset/file.ts @@ -0,0 +1,15 @@ +import { GET, POST, PUT, DELETE } from '@/api/request'; +import type { FileInfo, KbFileItemType } from '@/types/plugin'; + +import type { GetFileListProps, UpdateFileProps } from './file.d'; + +export const getDatasetFiles = (data: GetFileListProps) => + POST(`/core/dataset/file/list`, data); +export const delDatasetFileById = (params: { fileId: string; kbId: string }) => + DELETE(`/core/dataset/file/delById`, params); +export const getFileInfoById = (fileId: string) => + GET(`/core/dataset/file/detail`, { fileId }); +export const delDatasetEmptyFiles = (kbId: string) => + DELETE(`/core/dataset/file/delEmptyFiles`, { kbId }); + +export const updateDatasetFile = (data: UpdateFileProps) => PUT(`/core/dataset/file/update`, data); diff --git a/client/src/api/plugins/kb.ts b/client/src/api/plugins/kb.ts index c1c44f7f9..e5059fb70 100644 --- a/client/src/api/plugins/kb.ts +++ b/client/src/api/plugins/kb.ts @@ -1,12 +1,5 @@ import { GET, POST, PUT, DELETE } from '../request'; -import type { - DatasetItemType, - FileInfo, - KbFileItemType, - KbItemType, - KbListItemType, - KbPathItemType -} from '@/types/plugin'; +import type { DatasetItemType, KbItemType, KbListItemType, KbPathItemType } from '@/types/plugin'; import { TrainingModeEnum } from '@/constants/plugin'; import { Props as PushDataProps, @@ -17,12 +10,7 @@ import { Response as SearchTestResponse } from '@/pages/api/openapi/kb/searchTest'; import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData'; -import type { - KbUpdateParams, - CreateKbParams, - GetKbDataListProps, - GetFileListProps -} from '../request/kb'; +import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb'; import { QuoteItemType } from '@/types/chat'; import { KbTypeEnum } from '@/constants/kb'; @@ -42,16 +30,6 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`); -/* kb file */ -export const getKbFiles = (data: GetFileListProps) => - POST(`/plugins/kb/file/list`, data); -export const deleteKbFileById = (params: { fileId: string; kbId: string }) => - DELETE(`/plugins/kb/file/delFileByFileId`, params); -export const getFileInfoById = (fileId: string) => - GET(`/plugins/kb/file/getFileInfo`, { fileId }); -export const delEmptyFiles = (kbId: string) => - DELETE(`/plugins/kb/file/deleteEmptyFiles`, { kbId }); - /* kb data */ export const getKbDataList = (data: GetKbDataListProps) => POST(`/plugins/kb/data/getDataList`, data); @@ -59,7 +37,7 @@ export const getKbDataList = (data: GetKbDataListProps) => /** * 获取导出数据（不分页） */ -export const getExportDataList = (data: { kbId: string; fileId: string }) => +export const getExportDataList = (data: { kbId: string }) => GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, { timeout: 600000 }); diff --git a/client/src/api/request/kb.d.ts b/client/src/api/request/kb.d.ts index 3d93407c6..b382a0e49 100644 --- a/client/src/api/request/kb.d.ts +++ b/client/src/api/request/kb.d.ts @@ -17,11 +17,6 @@ export type CreateKbParams = { type: `${KbTypeEnum}`; }; -export type GetFileListProps = RequestPaging & { - kbId: string; - searchText: string; -}; - export type GetKbDataListProps = RequestPaging & { kbId: string; searchText: string; diff --git a/client/src/api/support/file.ts b/client/src/api/support/file.ts new file mode 100644 index 000000000..6028162b5 --- /dev/null +++ b/client/src/api/support/file.ts @@ -0,0 +1,18 @@ +import { GET, POST } from '../request'; + +import { AxiosProgressEvent } from 'axios'; + +export const uploadImg = (base64Img: string) => POST('/system/uploadImage', { base64Img }); + +export const postUploadFiles = ( + data: FormData, + onUploadProgress: (progressEvent: AxiosProgressEvent) => void +) => + POST('/support/file/upload', data, { + onUploadProgress, + headers: { + 'Content-Type': 'multipart/form-data; charset=utf-8' + } + }); + +export const getFileViewUrl = (fileId: string) => GET('/support/file/readUrl', { fileId }); diff --git a/client/src/api/system.ts b/client/src/api/system.ts index 3982628ea..0c2fb87c8 100644 --- a/client/src/api/system.ts +++ b/client/src/api/system.ts @@ -1,20 +1,4 @@ import { GET, POST, PUT } from './request'; import type { InitDateResponse } from '@/pages/api/system/getInitData'; -import { AxiosProgressEvent } from 'axios'; export const getInitData = () => GET('/system/getInitData'); - -export const uploadImg = (base64Img: string) => POST('/system/uploadImage', { base64Img }); - -export const postUploadFiles = ( - data: FormData, - onUploadProgress: (progressEvent: AxiosProgressEvent) => void -) => - POST('/plugins/file/upload', data, { - onUploadProgress, - headers: { - 'Content-Type': 'multipart/form-data; charset=utf-8' - } - }); - -export const getFileViewUrl = (fileId: string) => GET('/plugins/file/readUrl', { fileId }); diff --git a/client/src/components/Layout/navbar.tsx b/client/src/components/Layout/navbar.tsx index 6a0f5d435..8480963bb 100644 --- a/client/src/components/Layout/navbar.tsx +++ b/client/src/components/Layout/navbar.tsx @@ -167,20 +167,7 @@ const Navbar = ({ unread }: { unread: number }) => { )} - {feConfigs?.show_doc && ( - - { - window.open(`https://doc.fastgpt.run/docs/intro`); - }} - > - - - - )} + {feConfigs?.show_git && ( diff --git a/client/src/components/MyInput/index.tsx b/client/src/components/MyInput/index.tsx index 30e9d6b52..5b52fb4ce 100644 --- a/client/src/components/MyInput/index.tsx +++ b/client/src/components/MyInput/index.tsx @@ -8,7 +8,7 @@ interface Props extends InputProps { const MyInput = ({ leftIcon, ...props }: Props) => { return ( - + {leftIcon && ( { {t('user.Change')} + {feConfigs?.show_userDetail && ( + + + {t('user.Balance')}:  + + {userInfo?.balance.toFixed(3)} 元 + + + + + )} + {feConfigs?.show_doc && ( + <> + { + window.open(`https://doc.fastgpt.run/docs/intro`); + }} + > + + + {t('system.Help Document')} + + + + V{systemVersion} + + + + )} {feConfigs?.show_userDetail && ( <> - - - {t('user.Balance')}:  - - {userInfo?.balance.toFixed(3)} 元 - - - - - diff --git a/client/src/pages/api/admin/initv43.ts b/client/src/pages/api/admin/initv43.ts index 9ff322e30..2f700dfc1 100644 --- a/client/src/pages/api/admin/initv43.ts +++ b/client/src/pages/api/admin/initv43.ts @@ -3,7 +3,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@/service/response'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -12,7 +12,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const { rowCount } = await PgClient.query(`SELECT 1 FROM information_schema.columns WHERE table_schema = 'public' - AND table_name = '${PgTrainingTableName}' + AND table_name = '${PgDatasetTableName}' AND column_name = 'file_id'`); if (rowCount > 0) { @@ -23,7 +23,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) jsonRes(res, { data: await PgClient.query( - `ALTER TABLE ${PgTrainingTableName} ADD COLUMN file_id VARCHAR(100)` + `ALTER TABLE ${PgDatasetTableName} ADD COLUMN file_id VARCHAR(100)` ) }); } catch (error) { diff --git a/client/src/pages/api/admin/initv44.ts b/client/src/pages/api/admin/initv44.ts index 7c0a78596..e42005564 100644 --- a/client/src/pages/api/admin/initv44.ts +++ b/client/src/pages/api/admin/initv44.ts @@ -5,7 +5,7 @@ import { authUser } from '@/service/utils/auth'; import { connectToDatabase, KB } from '@/service/mongo'; import { KbTypeEnum } from '@/constants/kb'; import { PgClient } from '@/service/pg'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) } ); - const response = await PgClient.update(PgTrainingTableName, { + const response = await PgClient.update(PgDatasetTableName, { where: [['file_id', 'undefined']], values: [{ key: 'file_id', value: '' }] }); diff --git a/client/src/pages/api/admin/initv441.ts b/client/src/pages/api/admin/initv441.ts new file mode 100644 index 000000000..67d74f111 --- /dev/null +++ b/client/src/pages/api/admin/initv441.ts @@ -0,0 +1,35 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { authUser } from '@/service/utils/auth'; +import { connectToDatabase } from '@/service/mongo'; +import mongoose from 'mongoose'; +import { PgClient } from '@/service/pg'; +import { PgDatasetTableName } from '@/constants/plugin'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + await authUser({ req, authRoot: true }); + + const data = await mongoose.connection.db + .collection('dataset.files') + .updateMany({}, { $set: { 'metadata.datasetUsed': true } }); + + // update pg data + const pg = await PgClient.query(`UPDATE ${PgDatasetTableName} + SET file_id = '' + WHERE (file_id = 'undefined' OR LENGTH(file_id) < 20) AND file_id != '';`); + + jsonRes(res, { + data: { + data, + pg + } + }); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } +} diff --git a/client/src/pages/api/plugins/kb/file/delFileByFileId.ts b/client/src/pages/api/core/dataset/file/delById.ts similarity index 89% rename from client/src/pages/api/plugins/kb/file/delFileByFileId.ts rename to client/src/pages/api/core/dataset/file/delById.ts index 0e6b7767c..a4292dc9f 100644 --- a/client/src/pages/api/plugins/kb/file/delFileByFileId.ts +++ b/client/src/pages/api/core/dataset/file/delById.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { GridFSStorage } from '@/service/lib/gridfs'; import { PgClient } from '@/service/pg'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; import { Types } from 'mongoose'; import { OtherFileId } from '@/constants/kb'; @@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const { userId } = await authUser({ req, authToken: true }); if (fileId === OtherFileId) { - await PgClient.delete(PgTrainingTableName, { + await PgClient.delete(PgDatasetTableName, { where: [ ['user_id', userId], 'AND', @@ -37,7 +37,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await gridFs.findAndAuthFile(fileId); // delete all pg data - await PgClient.delete(PgTrainingTableName, { + await PgClient.delete(PgDatasetTableName, { where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]] }); diff --git a/client/src/pages/api/core/dataset/file/delEmptyFiles.ts b/client/src/pages/api/core/dataset/file/delEmptyFiles.ts new file mode 100644 index 000000000..2da6767c0 --- /dev/null +++ b/client/src/pages/api/core/dataset/file/delEmptyFiles.ts @@ -0,0 +1,31 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authUser } from '@/service/utils/auth'; +import { GridFSStorage } from '@/service/lib/gridfs'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + const { kbId } = req.query as { kbId: string }; + // 凭证校验 + const { userId } = await authUser({ req, authToken: true }); + + const gridFs = new GridFSStorage('dataset', userId); + const collection = gridFs.Collection(); + + const files = await collection.deleteMany({ + uploadDate: { $lte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }, + ['metadata.kbId']: kbId, + ['metadata.userId']: userId, + ['metadata.datasetUsed']: { $ne: true } + }); + + jsonRes(res, { + data: files + }); + } catch (err) { + jsonRes(res); + } +} diff --git a/client/src/pages/api/plugins/kb/file/getFileInfo.ts b/client/src/pages/api/core/dataset/file/detail.ts similarity index 100% rename from client/src/pages/api/plugins/kb/file/getFileInfo.ts rename to client/src/pages/api/core/dataset/file/detail.ts diff --git a/client/src/pages/api/plugins/kb/file/list.ts b/client/src/pages/api/core/dataset/file/list.ts similarity index 79% rename from client/src/pages/api/plugins/kb/file/list.ts rename to client/src/pages/api/core/dataset/file/list.ts index 722704a20..72019c468 100644 --- a/client/src/pages/api/plugins/kb/file/list.ts +++ b/client/src/pages/api/core/dataset/file/list.ts @@ -4,9 +4,8 @@ import { connectToDatabase, TrainingData } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { GridFSStorage } from '@/service/lib/gridfs'; import { PgClient } from '@/service/pg'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; import { FileStatusEnum, OtherFileId } from '@/constants/kb'; -import mongoose from 'mongoose'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -16,28 +15,37 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< pageNum = 1, pageSize = 10, kbId, - searchText + searchText = '' } = req.body as { pageNum: number; pageSize: number; kbId: string; searchText: string }; - searchText = searchText.replace(/'/g, ''); + searchText = searchText?.replace(/'/g, ''); // 凭证校验 const { userId } = await authUser({ req, authToken: true }); + // find files const gridFs = new GridFSStorage('dataset', userId); - const bucket = gridFs.GridFSBucket(); + const collection = gridFs.Collection(); const mongoWhere = { ['metadata.kbId']: kbId, + ['metadata.userId']: userId, + ['metadata.datasetUsed']: true, ...(searchText && { filename: { $regex: searchText } }) }; const [files, total] = await Promise.all([ - bucket - .find(mongoWhere) - .sort({ _id: -1 }) + collection + .find(mongoWhere, { + projection: { + _id: 1, + filename: 1, + uploadDate: 1, + length: 1 + } + }) .skip((pageNum - 1) * pageSize) .limit(pageSize) .toArray(), - mongoose.connection.db.collection('dataset.files').countDocuments(mongoWhere) + collection.countDocuments(mongoWhere) ]); async function GetOtherData() { @@ -49,7 +57,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< status: (await TrainingData.findOne({ userId, kbId, file_id: '' })) ? FileStatusEnum.embedding : FileStatusEnum.ready, - chunkLength: await PgClient.count(PgTrainingTableName, { + chunkLength: await PgClient.count(PgDatasetTableName, { fields: ['id'], where: [ ['user_id', userId], @@ -72,7 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< status: (await TrainingData.findOne({ userId, kbId, file_id: file._id })) ? FileStatusEnum.embedding : FileStatusEnum.ready, - chunkLength: await PgClient.count(PgTrainingTableName, { + chunkLength: await PgClient.count(PgDatasetTableName, { fields: ['id'], where: [ ['user_id', userId], @@ -90,7 +98,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< data: { pageNum, pageSize, - data: data.flat().filter((item) => item.chunkLength > 0), + data: data.flat(), total } }); diff --git a/client/src/pages/api/core/dataset/file/update.ts b/client/src/pages/api/core/dataset/file/update.ts new file mode 100644 index 000000000..bcef49057 --- /dev/null +++ b/client/src/pages/api/core/dataset/file/update.ts @@ -0,0 +1,66 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authUser } from '@/service/utils/auth'; +import { GridFSStorage } from '@/service/lib/gridfs'; +import { UpdateFileProps } from '@/api/core/dataset/file.d'; +import { Types } from 'mongoose'; +import { PgClient } from '@/service/pg'; +import { PgDatasetTableName } from '@/constants/plugin'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + const { id, name, datasetUsed } = req.body as UpdateFileProps; + const { userId } = await authUser({ req, authToken: true }); + + const gridFs = new GridFSStorage('dataset', userId); + const collection = gridFs.Collection(); + + await collection.findOneAndUpdate( + { + _id: new Types.ObjectId(id) + }, + { + $set: { + ...(name && { filename: name }), + ...(datasetUsed && { ['metadata.datasetUsed']: datasetUsed }) + } + } + ); + + // data source + updateDatasetSource({ + fileId: id, + userId, + name + }); + + jsonRes(res, {}); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} +async function updateDatasetSource(data: { fileId: string; userId: string; name?: string }) { + const { fileId, userId, name } = data; + if (!fileId || !name || !userId) return; + try { + await PgClient.update(PgDatasetTableName, { + where: [['user_id', userId], 'AND', ['file_id', fileId]], + values: [ + { + key: 'source', + value: name + } + ] + }); + } catch (error) { + setTimeout(() => { + updateDatasetSource(data); + }, 2000); + } +} diff --git a/client/src/pages/api/openapi/kb/delDataById.ts b/client/src/pages/api/openapi/kb/delDataById.ts index a0e7ac9ef..e16995e47 100644 --- a/client/src/pages/api/openapi/kb/delDataById.ts +++ b/client/src/pages/api/openapi/kb/delDataById.ts @@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; import { withNextCors } from '@/service/utils/tools'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -18,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // 凭证校验 const { userId } = await authUser({ req }); - await PgClient.delete(PgTrainingTableName, { + await PgClient.delete(PgDatasetTableName, { where: [['user_id', userId], 'AND', ['id', dataId]] }); diff --git a/client/src/pages/api/openapi/kb/pushData.ts b/client/src/pages/api/openapi/kb/pushData.ts index 1b7d7b460..2718c6de6 100644 --- a/client/src/pages/api/openapi/kb/pushData.ts +++ b/client/src/pages/api/openapi/kb/pushData.ts @@ -4,7 +4,7 @@ import { connectToDatabase, TrainingData, KB } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { authKb } from '@/service/utils/auth'; import { withNextCors } from '@/service/utils/tools'; -import { PgTrainingTableName, TrainingModeEnum } from '@/constants/plugin'; +import { PgDatasetTableName, TrainingModeEnum } from '@/constants/plugin'; import { startQueue } from '@/service/utils/tools'; import { PgClient } from '@/service/pg'; import { modelToolMap } from '@/utils/plugin'; @@ -136,7 +136,7 @@ export async function pushDataToKb({ try { const { rows } = await PgClient.query(` SELECT COUNT(*) > 0 AS exists - FROM ${PgTrainingTableName} + FROM ${PgDatasetTableName} WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}' `); const exists = rows[0]?.exists || false; diff --git a/client/src/pages/api/openapi/kb/searchTest.ts b/client/src/pages/api/openapi/kb/searchTest.ts index a19330a85..25974b082 100644 --- a/client/src/pages/api/openapi/kb/searchTest.ts +++ b/client/src/pages/api/openapi/kb/searchTest.ts @@ -5,7 +5,7 @@ import { PgClient } from '@/service/pg'; import { withNextCors } from '@/service/utils/tools'; import { getVector } from '../plugin/vector'; import type { KbTestItemType } from '@/types/plugin'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; import { KB } from '@/service/mongo'; export type Props = { @@ -43,7 +43,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10}; select id, q, a, source, file_id, (vector <#> '[${ vectors[0] - }]') * -1 AS score from ${PgTrainingTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${ + }]') * -1 AS score from ${PgDatasetTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${ vectors[0] }]' limit 12; COMMIT;` diff --git a/client/src/pages/api/openapi/kb/updateData.ts b/client/src/pages/api/openapi/kb/updateData.ts index eca98af87..430b487fe 100644 --- a/client/src/pages/api/openapi/kb/updateData.ts +++ b/client/src/pages/api/openapi/kb/updateData.ts @@ -5,7 +5,7 @@ import { PgClient } from '@/service/pg'; import { withNextCors } from '@/service/utils/tools'; import { KB, connectToDatabase } from '@/service/mongo'; import { getVector } from '../plugin/vector'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; export type Props = { dataId: string; @@ -47,7 +47,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex })(); // 更新 pg 内容.仅修改a，不需要更新向量。 - await PgClient.update(PgTrainingTableName, { + await PgClient.update(PgDatasetTableName, { where: [['id', dataId], 'AND', ['user_id', userId]], values: [ { key: 'a', value: a.replace(/'/g, '"') }, diff --git a/client/src/pages/api/plugins/kb/data/exportModelData.ts b/client/src/pages/api/plugins/kb/data/exportModelData.ts index fc1e042e8..a6e4f8b4e 100644 --- a/client/src/pages/api/plugins/kb/data/exportModelData.ts +++ b/client/src/pages/api/plugins/kb/data/exportModelData.ts @@ -3,14 +3,13 @@ import { jsonRes } from '@/service/response'; import { connectToDatabase, User } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; -import { PgTrainingTableName } from '@/constants/plugin'; -import { OtherFileId } from '@/constants/kb'; +import { PgDatasetTableName } from '@/constants/plugin'; +import { findAllChildrenIds } from '../delete'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - let { kbId, fileId } = req.query as { + let { kbId } = req.query as { kbId: string; - fileId: string; }; if (!kbId) { @@ -22,6 +21,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // 凭证校验 const { userId } = await authUser({ req, authToken: true }); + const exportIds = [kbId, ...(await findAllChildrenIds(kbId))]; + console.log(exportIds); + const thirtyMinutesAgo = new Date( Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000 ); @@ -43,10 +45,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`); } - const where: any = [['kb_id', kbId], 'AND', ['user_id', userId]]; + const where: any = [ + ['user_id', userId], + 'AND', + `kb_id IN (${exportIds.map((id) => `'${id}'`).join(',')})` + ]; // 从 pg 中获取所有数据 const pgData = await PgClient.select<{ q: string; a: string; source: string }>( - PgTrainingTableName, + PgDatasetTableName, { where, fields: ['q', 'a', 'source'], diff --git a/client/src/pages/api/plugins/kb/data/getDataById.ts b/client/src/pages/api/plugins/kb/data/getDataById.ts index 834e886b9..38d4ca940 100644 --- a/client/src/pages/api/plugins/kb/data/getDataById.ts +++ b/client/src/pages/api/plugins/kb/data/getDataById.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; import type { KbDataItemType } from '@/types/plugin'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; export type Response = { id: string; @@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const where: any = [['user_id', userId], 'AND', ['id', dataId]]; - const searchRes = await PgClient.select(PgTrainingTableName, { + const searchRes = await PgClient.select(PgDatasetTableName, { fields: ['kb_id', 'id', 'q', 'a', 'source', 'file_id'], where, limit: 1 diff --git a/client/src/pages/api/plugins/kb/data/getDataList.ts b/client/src/pages/api/plugins/kb/data/getDataList.ts index 6c8fecf3f..74a261c86 100644 --- a/client/src/pages/api/plugins/kb/data/getDataList.ts +++ b/client/src/pages/api/plugins/kb/data/getDataList.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; import type { KbDataItemType } from '@/types/plugin'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; import { OtherFileId } from '@/constants/kb'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -50,14 +50,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ]; const [searchRes, total] = await Promise.all([ - PgClient.select(PgTrainingTableName, { + PgClient.select(PgDatasetTableName, { fields: ['id', 'q', 'a', 'source', 'file_id'], where, order: [{ field: 'id', mode: 'DESC' }], limit: pageSize, offset: pageSize * (pageNum - 1) }), - PgClient.count(PgTrainingTableName, { + PgClient.count(PgDatasetTableName, { fields: ['id'], where }) diff --git a/client/src/pages/api/plugins/kb/data/insertData.ts b/client/src/pages/api/plugins/kb/data/insertData.ts index fc47d9040..5f581aaf7 100644 --- a/client/src/pages/api/plugins/kb/data/insertData.ts +++ b/client/src/pages/api/plugins/kb/data/insertData.ts @@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response'; import { connectToDatabase, KB } from '@/service/mongo'; import { authKb, authUser } from '@/service/utils/auth'; import { withNextCors } from '@/service/utils/tools'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; import { insertKbItem, PgClient } from '@/service/pg'; import { modelToolMap } from '@/utils/plugin'; import { getVectorModel } from '@/service/utils/data'; @@ -45,7 +45,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex const { rows: existsRows } = await PgClient.query(` SELECT COUNT(*) > 0 AS exists - FROM ${PgTrainingTableName} + FROM ${PgDatasetTableName} WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}' `); const exists = existsRows[0]?.exists || false; diff --git a/client/src/pages/api/plugins/kb/delete.ts b/client/src/pages/api/plugins/kb/delete.ts index ad68a1ad4..4ffff7857 100644 --- a/client/src/pages/api/plugins/kb/delete.ts +++ b/client/src/pages/api/plugins/kb/delete.ts @@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response'; import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; import { GridFSStorage } from '@/service/lib/gridfs'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // delete all pg data - await PgClient.delete(PgTrainingTableName, { + await PgClient.delete(PgDatasetTableName, { where: [ ['user_id', userId], 'AND', @@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } } -async function findAllChildrenIds(id: string) { +export async function findAllChildrenIds(id: string) { // find children const children = await KB.find({ parentId: id }); diff --git a/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts b/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts deleted file mode 100644 index abac5af8f..000000000 --- a/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@/service/response'; -import { connectToDatabase } from '@/service/mongo'; -import { authUser } from '@/service/utils/auth'; -import { GridFSStorage } from '@/service/lib/gridfs'; -import { PgClient } from '@/service/pg'; -import { PgTrainingTableName } from '@/constants/plugin'; -import { Types } from 'mongoose'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await connectToDatabase(); - - const { kbId } = req.query as { kbId: string }; - // 凭证校验 - const { userId } = await authUser({ req, authToken: true }); - - const gridFs = new GridFSStorage('dataset', userId); - const bucket = gridFs.GridFSBucket(); - - const files = await bucket - // 1 hours expired - .find({ - uploadDate: { $lte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }, - ['metadata.kbId']: kbId, - ['metadata.userId']: userId - }) - .sort({ _id: -1 }) - .toArray(); - - const data = await Promise.all( - files.map(async (file) => { - return { - id: file._id, - chunkLength: await PgClient.count(PgTrainingTableName, { - fields: ['id'], - where: [ - ['user_id', userId], - 'AND', - ['kb_id', kbId], - 'AND', - ['file_id', String(file._id)] - ] - }) - }; - }) - ); - - await Promise.all( - data - .filter((item) => item.chunkLength === 0) - .map((file) => bucket.delete(new Types.ObjectId(file.id))) - ); - - jsonRes(res); - } catch (err) { - jsonRes(res); - } -} diff --git a/client/src/pages/api/plugins/file/delete.ts b/client/src/pages/api/support/file/delete.ts similarity index 100% rename from client/src/pages/api/plugins/file/delete.ts rename to client/src/pages/api/support/file/delete.ts diff --git a/client/src/pages/api/plugins/file/read.ts b/client/src/pages/api/support/file/read.ts similarity index 100% rename from client/src/pages/api/plugins/file/read.ts rename to client/src/pages/api/support/file/read.ts diff --git a/client/src/pages/api/plugins/file/readUrl.ts b/client/src/pages/api/support/file/readUrl.ts similarity index 97% rename from client/src/pages/api/plugins/file/readUrl.ts rename to client/src/pages/api/support/file/readUrl.ts index 6b9cb51d9..21f81cb4a 100644 --- a/client/src/pages/api/plugins/file/readUrl.ts +++ b/client/src/pages/api/support/file/readUrl.ts @@ -23,7 +23,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); jsonRes(res, { - data: `/api/plugins/file/read?token=${token}` + data: `/api/support/file/read?token=${token}` }); } catch (error) { jsonRes(res, { diff --git a/client/src/pages/api/plugins/file/upload.ts b/client/src/pages/api/support/file/upload.ts similarity index 100% rename from client/src/pages/api/plugins/file/upload.ts rename to client/src/pages/api/support/file/upload.ts diff --git a/client/src/pages/api/system/getInitData.ts b/client/src/pages/api/system/getInitData.ts index 4a69eafa4..d75c53f98 100644 --- a/client/src/pages/api/system/getInitData.ts +++ b/client/src/pages/api/system/getInitData.ts @@ -13,6 +13,7 @@ export type InitDateResponse = { qaModel: QAModelItemType; vectorModels: VectorModelItemType[]; feConfigs: FeConfigsType; + systemVersion: string; }; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -24,7 +25,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) feConfigs: global.feConfigs, chatModels: global.chatModels, qaModel: global.qaModel, - vectorModels: global.vectorModels + vectorModels: global.vectorModels, + systemVersion: process.env.npm_package_version || '0.0.0' } }); } diff --git a/client/src/pages/chat/components/ChatHistorySlider.tsx b/client/src/pages/chat/components/ChatHistorySlider.tsx index 40fea4f74..de6219829 100644 --- a/client/src/pages/chat/components/ChatHistorySlider.tsx +++ b/client/src/pages/chat/components/ChatHistorySlider.tsx @@ -11,7 +11,7 @@ import { IconButton } from '@chakra-ui/react'; import { useGlobalStore } from '@/store/global'; -import { useEditInfo } from '@/hooks/useEditInfo'; +import { useEditTitle } from '@/hooks/useEditTitle'; import { useRouter } from 'next/router'; import Avatar from '@/components/Avatar'; import MyTooltip from '@/components/MyTooltip'; @@ -70,7 +70,7 @@ const ChatHistorySlider = ({ const isShare = useMemo(() => !appId || !userInfo, [appId, userInfo]); // custom title edit - const { onOpenModal, EditModal: EditTitleModal } = useEditInfo({ + const { onOpenModal, EditModal: EditTitleModal } = useEditTitle({ title: '自定义历史记录标题', placeholder: '如果设置为空，会自动跟随聊天记录。' }); diff --git a/client/src/pages/kb/detail/components/DataCard.tsx b/client/src/pages/kb/detail/components/DataCard.tsx index 3e23a1f1d..916686816 100644 --- a/client/src/pages/kb/detail/components/DataCard.tsx +++ b/client/src/pages/kb/detail/components/DataCard.tsx @@ -1,19 +1,12 @@ import React, { useCallback, useState, useRef, useMemo } from 'react'; -import { Box, Card, IconButton, Flex, Button, Grid, Image } from '@chakra-ui/react'; +import { Box, Card, IconButton, Flex, Grid, Image } from '@chakra-ui/react'; import type { KbDataItemType } from '@/types/plugin'; import { usePagination } from '@/hooks/usePagination'; -import { - getKbDataList, - getExportDataList, - delOneKbDataByDataId, - getTrainingData, - getFileInfoById -} from '@/api/plugins/kb'; +import { getKbDataList, delOneKbDataByDataId, getTrainingData } from '@/api/plugins/kb'; +import { getFileInfoById } from '@/api/core/dataset/file'; import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons'; -import { fileDownload } from '@/utils/file'; -import { useMutation, useQuery } from '@tanstack/react-query'; +import { useQuery } from '@tanstack/react-query'; import { useToast } from '@/hooks/useToast'; -import Papa from 'papaparse'; import InputModal, { FormData as InputDataType } from './InputDataModal'; import { debounce } from 'lodash'; import { getErrText } from '@/utils/tools'; @@ -24,8 +17,6 @@ import MyIcon from '@/components/Icon'; import MyTooltip from '@/components/MyTooltip'; import MyInput from '@/components/MyInput'; import { fileImgs } from '@/constants/common'; -import { useRequest } from '@/hooks/useRequest'; -import { feConfigs } from '@/store/static'; const DataCard = ({ kbId }: { kbId: string }) => { const BoxRef = useRef(null); @@ -82,31 +73,27 @@ const DataCard = ({ kbId }: { kbId: string }) => { [fileInfo?.filename] ); - // get al data and export csv - const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({ - mutationFn: () => getExportDataList({ kbId, fileId }), - onSuccess(res) { - const text = Papa.unparse({ - fields: ['question', 'answer', 'source'], - data: res - }); - - const filenameSplit = fileInfo?.filename?.split('.') || []; - const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.'); - - fileDownload({ - text, - type: 'text/csv', - filename - }); - }, - successToast: `导出成功，下次导出需要 ${feConfigs?.limit?.exportLimitMinutes} 分钟后`, - errorToast: '导出异常' - }); - return ( + } + bg={'white'} + boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'} + h={'28px'} + size={'sm'} + borderRadius={'50%'} + aria-label={''} + onClick={() => + router.replace({ + query: { + kbId, + currentTab: 'dataset' + } + }) + } + /> { {''} {t(fileInfo?.filename || 'Filename')} - { const BoxRef = useRef(null); @@ -45,13 +52,13 @@ const FileCard = ({ kbId }: { kbId: string }) => { data: files, Pagination, total, - isLoading, getData, + isLoading, pageNum, pageSize } = usePagination({ - api: getKbFiles, - pageSize: 40, + api: getDatasetFiles, + pageSize: 20, params: { kbId, searchText @@ -63,6 +70,7 @@ const FileCard = ({ kbId }: { kbId: string }) => { } }); + // change search const debounceRefetch = useCallback( debounce(() => { getData(1); @@ -71,6 +79,7 @@ const FileCard = ({ kbId }: { kbId: string }) => { [] ); + // add file icon const formatFiles = useMemo( () => files.map((file) => ({ @@ -79,15 +88,11 @@ const FileCard = ({ kbId }: { kbId: string }) => { })), [files] ); - const totalDataLength = useMemo( - () => files.reduce((sum, item) => sum + item.chunkLength, 0), - [files] - ); const { mutate: onDeleteFile } = useRequest({ mutationFn: (fileId: string) => { setLoading(true); - return deleteKbFileById({ + return delDatasetFileById({ fileId, kbId }); @@ -101,6 +106,24 @@ const FileCard = ({ kbId }: { kbId: string }) => { successToast: t('common.Delete Success'), errorToast: t('common.Delete Failed') }); + const { mutate: onUpdateFilename } = useRequest({ + mutationFn: (data: { id: string; name: string }) => { + setLoading(true); + return updateDatasetFile(data); + }, + onSuccess() { + getData(pageNum); + }, + onSettled() { + setLoading(false); + }, + successToast: t('common.Delete Success'), + errorToast: t('common.Delete Failed') + }); + + const { onOpenModal, EditModal: EditTitleModal } = useEditTitle({ + title: t('Rename') + }); const statusMap = { [FileStatusEnum.embedding]: { @@ -121,17 +144,21 @@ const FileCard = ({ kbId }: { kbId: string }) => { } }); - useQuery(['refetchTrainingData'], refetchTrainingData, { - refetchInterval: 8000, - enabled: qaListLen > 0 || vectorListLen > 0 - }); + useQuery( + ['refetchTrainingData', kbId], + () => Promise.all([refetchTrainingData(), getData(pageNum)]), + { + refetchInterval: 8000, + enabled: qaListLen > 0 || vectorListLen > 0 + } + ); return ( - - + + - - {t('kb.Files', { total: files.length })} + + {t('kb.Files', { total })} {(qaListLen > 0 || vectorListLen > 0) && ( @@ -149,7 +176,8 @@ const FileCard = ({ kbId }: { kbId: string }) => { leftIcon={ } - w={['100%', '200px']} + w={['100%', '250px']} + size={['sm', 'md']} placeholder={t('common.Search') || ''} value={searchText} onChange={(e) => { @@ -169,14 +197,12 @@ const FileCard = ({ kbId }: { kbId: string }) => { /> - + - + @@ -213,45 +239,103 @@ const FileCard = ({ kbId }: { kbId: string }) => { - ))}
{t('kb.Filename')} - {t('kb.Chunk Length')}({totalDataLength}) - {t('kb.Chunk Length')} {t('kb.Upload Time')} {t('kb.File Size')} {t('common.Status')} {dayjs(file.uploadTime).format('YYYY/MM/DD HH:mm')} {formatFileSize(file.size)} - {statusMap[file.status].text} + + + {statusMap[file.status].text} + e.stopPropagation()}> - - openConfirm(() => { - onDeleteFile(file.id); - })() + + + } + menuList={[ + ...(file.id !== OtherFileId + ? [ + { + child: ( + + + {t('Rename')} + + ), + onClick: () => + onOpenModal({ + defaultVal: file.filename, + onSuccess: (newName) => { + onUpdateFilename({ + id: file.id, + name: newName + }); + } + }) + } + ] + : []), + { + child: ( + + + {t('common.Delete')} + + ), + onClick: openConfirm(() => { + onDeleteFile(file.id); + }) + } + ]} />
+ + {total > pageSize && ( + + + + )} - {total > pageSize && ( - - - - )} - - + ); }; diff --git a/client/src/pages/kb/detail/components/Import/Chunk.tsx b/client/src/pages/kb/detail/components/Import/Chunk.tsx index aa1cf37e5..8baeca52c 100644 --- a/client/src/pages/kb/detail/components/Import/Chunk.tsx +++ b/client/src/pages/kb/detail/components/Import/Chunk.tsx @@ -27,6 +27,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { TrainingModeEnum } from '@/constants/plugin'; import FileSelect, { type FileItemType } from './FileSelect'; import { useDatasetStore } from '@/store/dataset'; +import { updateDatasetFile } from '@/api/core/dataset/file'; const fileExtension = '.txt, .doc, .docx, .pdf, .md'; @@ -64,6 +65,16 @@ const ChunkImport = ({ kbId }: { kbId: string }) => { mutationFn: async () => { const chunks = files.map((file) => file.chunks).flat(); + // mark the file is used + await Promise.all( + files.map((file) => + updateDatasetFile({ + id: file.id, + datasetUsed: true + }) + ) + ); + // subsection import let success = 0; const step = 300; diff --git a/client/src/pages/kb/detail/components/Import/Csv.tsx b/client/src/pages/kb/detail/components/Import/Csv.tsx index 54ffc83f6..c0aef2bfe 100644 --- a/client/src/pages/kb/detail/components/Import/Csv.tsx +++ b/client/src/pages/kb/detail/components/Import/Csv.tsx @@ -11,6 +11,7 @@ import { TrainingModeEnum } from '@/constants/plugin'; import FileSelect, { type FileItemType } from './FileSelect'; import { useRouter } from 'next/router'; import { useDatasetStore } from '@/store/dataset'; +import { updateDatasetFile } from '@/api/core/dataset/file'; const fileExtension = '.csv'; @@ -37,6 +38,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => { const { mutate: onclickUpload, isLoading: uploading } = useMutation({ mutationFn: async () => { + // mark the file is used + await Promise.all( + files.map((file) => + updateDatasetFile({ + id: file.id, + datasetUsed: true + }) + ) + ); + const chunks = files .map((file) => file.chunks) .flat() diff --git a/client/src/pages/kb/detail/components/Import/FileSelect.tsx b/client/src/pages/kb/detail/components/Import/FileSelect.tsx index ea95b8582..fff4590b1 100644 --- a/client/src/pages/kb/detail/components/Import/FileSelect.tsx +++ b/client/src/pages/kb/detail/components/Import/FileSelect.tsx @@ -152,7 +152,7 @@ const FileSelect = ({ throw new Error('csv 文件格式有误,请确保 question 和 answer 两列'); } const fileItem: FileItemType = { - id: nanoid(), + id: filesId[0], filename: file.name, icon, tokens: 0, diff --git a/client/src/pages/kb/detail/components/Import/QA.tsx b/client/src/pages/kb/detail/components/Import/QA.tsx index 920fae6da..dc1b09dca 100644 --- a/client/src/pages/kb/detail/components/Import/QA.tsx +++ b/client/src/pages/kb/detail/components/Import/QA.tsx @@ -16,6 +16,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { TrainingModeEnum } from '@/constants/plugin'; import FileSelect, { type FileItemType } from './FileSelect'; import { useRouter } from 'next/router'; +import { updateDatasetFile } from '@/api/core/dataset/file'; const fileExtension = '.txt, .doc, .docx, .pdf, .md'; @@ -55,6 +56,16 @@ const QAImport = ({ kbId }: { kbId: string }) => { mutationFn: async () => { const chunks = files.map((file) => file.chunks).flat(); + // mark the file is used + await Promise.all( + files.map((file) => + updateDatasetFile({ + id: file.id, + datasetUsed: true + }) + ) + ); + // subsection import let success = 0; const step = 200; diff --git a/client/src/pages/kb/detail/components/InputDataModal.tsx b/client/src/pages/kb/detail/components/InputDataModal.tsx index f900c64ca..45dd0d66c 100644 --- a/client/src/pages/kb/detail/components/InputDataModal.tsx +++ b/client/src/pages/kb/detail/components/InputDataModal.tsx @@ -2,7 +2,7 @@ import React, { useState, useCallback } from 'react'; import { Box, Flex, Button, Textarea, IconButton, BoxProps } from '@chakra-ui/react'; import { useForm } from 'react-hook-form'; import { insertData2Kb, putKbDataById, delOneKbDataByDataId } from '@/api/plugins/kb'; -import { getFileViewUrl } from '@/api/system'; +import { getFileViewUrl } from '@/api/support/file'; import { useToast } from '@/hooks/useToast'; import { getErrText } from '@/utils/tools'; import MyIcon from '@/components/Icon'; diff --git a/client/src/pages/kb/detail/index.tsx b/client/src/pages/kb/detail/index.tsx index 2b12124ac..ac0dfc6d1 100644 --- a/client/src/pages/kb/detail/index.tsx +++ b/client/src/pages/kb/detail/index.tsx @@ -17,7 +17,8 @@ import Avatar from '@/components/Avatar'; import Info from './components/Info'; import { serviceSideProps } from '@/utils/i18n'; import { useTranslation } from 'react-i18next'; -import { delEmptyFiles, getTrainingQueueLen } from '@/api/plugins/kb'; +import { getTrainingQueueLen } from '@/api/plugins/kb'; +import { delDatasetEmptyFiles } from '@/api/core/dataset/file'; import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { feConfigs } from '@/store/static'; @@ -96,7 +97,7 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } useEffect(() => { return () => { try { - delEmptyFiles(kbId); + delDatasetEmptyFiles(kbId); } catch (error) {} }; }, [kbId]); diff --git a/client/src/pages/kb/list/index.tsx b/client/src/pages/kb/list/index.tsx index a6e623e10..49f63ad2e 100644 --- a/client/src/pages/kb/list/index.tsx +++ b/client/src/pages/kb/list/index.tsx @@ -15,7 +15,7 @@ import PageContainer from '@/components/PageContainer'; import { useConfirm } from '@/hooks/useConfirm'; import { AddIcon } from '@chakra-ui/icons'; import { useQuery } from '@tanstack/react-query'; -import { delKbById, getKbPaths, putKbById } from '@/api/plugins/kb'; +import { delKbById, getExportDataList, getKbPaths, putKbById } from '@/api/plugins/kb'; import { useTranslation } from 'react-i18next'; import Avatar from '@/components/Avatar'; import MyIcon from '@/components/Icon'; @@ -26,7 +26,10 @@ import Tag from '@/components/Tag'; import MyMenu from '@/components/MyMenu'; import { useRequest } from '@/hooks/useRequest'; import { useGlobalStore } from '@/store/global'; -import { useEditInfo } from '@/hooks/useEditInfo'; +import { useEditTitle } from '@/hooks/useEditTitle'; +import Papa from 'papaparse'; +import { fileDownload } from '@/utils/file'; +import { feConfigs } from '@/store/static'; const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false }); const EditFolderModal = dynamic(() => import('./component/EditFolderModal'), { ssr: false }); @@ -49,7 +52,7 @@ const Kb = () => { content: '' }); const { myKbList, loadKbList, setKbList, updateDataset } = useDatasetStore(); - const { onOpenModal: onOpenTitleModal, EditModal: EditTitleModal } = useEditInfo({ + const { onOpenModal: onOpenTitleModal, EditModal: EditTitleModal } = useEditTitle({ title: t('Rename') }); const [moveDataId, setMoveDataId] = useState(); @@ -83,7 +86,32 @@ const Kb = () => { errorToast: t('kb.Delete Dataset Error') }); - const { data, refetch } = useQuery(['loadKbList', parentId], () => { + // export dataset to csv + const { mutate: onclickExport } = useRequest({ + mutationFn: (kbId: string) => { + setLoading(true); + return getExportDataList({ kbId }); + }, + onSuccess(res) { + const text = Papa.unparse({ + fields: ['question', 'answer', 'source'], + data: res + }); + + fileDownload({ + text, + type: 'text/csv', + filename: 'dataset.csv' + }); + }, + onSettled() { + setLoading(false); + }, + successToast: `导出成功，下次导出需要 ${feConfigs?.limit?.exportLimitMinutes} 分钟后`, + errorToast: '导出异常' + }); + + const { data, refetch } = useQuery(['loadDataset', parentId], () => { return Promise.all([loadKbList(parentId), getKbPaths(parentId)]); }); @@ -318,6 +346,15 @@ const Kb = () => { ), onClick: () => setMoveDataId(kb._id) }, + { + child: ( + + + {t('Export')} + + ), + onClick: () => onclickExport(kb._id) + }, { child: ( diff --git a/client/src/service/lib/gridfs.ts b/client/src/service/lib/gridfs.ts index 76ed806ed..7d1093215 100644 --- a/client/src/service/lib/gridfs.ts +++ b/client/src/service/lib/gridfs.ts @@ -17,6 +17,9 @@ export class GridFSStorage { this.bucket = bucket; this.uid = String(uid); } + Collection() { + return mongoose.connection.db.collection(`${this.bucket}.files`); + } GridFSBucket() { return new mongoose.mongo.GridFSBucket(mongoose.connection.db, { bucketName: this.bucket diff --git a/client/src/service/moduleDispatch/kb/search.ts b/client/src/service/moduleDispatch/kb/search.ts index 8281e09e0..6813846f6 100644 --- a/client/src/service/moduleDispatch/kb/search.ts +++ b/client/src/service/moduleDispatch/kb/search.ts @@ -5,7 +5,7 @@ import { getVector } from '@/pages/api/openapi/plugin/vector'; import { countModelPrice } from '@/service/events/pushBill'; import type { SelectedKbType } from '@/types/plugin'; import type { QuoteItemType } from '@/types/chat'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; type KBSearchProps = { kbList: SelectedKbType; @@ -42,7 +42,7 @@ export async function dispatchKBSearch(props: Record): Promise `'${item.kbId}'`) .join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${ vectors[0] diff --git a/client/src/service/pg.ts b/client/src/service/pg.ts index 090be6fb6..eb3a1561c 100644 --- a/client/src/service/pg.ts +++ b/client/src/service/pg.ts @@ -1,6 +1,6 @@ import { Pool } from 'pg'; import type { QueryResultRow } from 'pg'; -import { PgTrainingTableName } from '@/constants/plugin'; +import { PgDatasetTableName } from '@/constants/plugin'; import { addLog } from './utils/tools'; import { DatasetItemType } from '@/types/plugin'; @@ -174,7 +174,7 @@ export const insertKbItem = ({ vector: number[]; })[]; }) => { - return PgClient.insert(PgTrainingTableName, { + return PgClient.insert(PgDatasetTableName, { values: data.map((item) => [ { key: 'user_id', value: userId }, { key: 'kb_id', value: kbId }, @@ -192,7 +192,7 @@ export async function initPg() { await connectPg(); await PgClient.query(` CREATE EXTENSION IF NOT EXISTS vector; - CREATE TABLE IF NOT EXISTS ${PgTrainingTableName} ( + CREATE TABLE IF NOT EXISTS ${PgDatasetTableName} ( id BIGSERIAL PRIMARY KEY, vector VECTOR(1536) NOT NULL, user_id VARCHAR(50) NOT NULL, @@ -202,9 +202,9 @@ export async function initPg() { q TEXT NOT NULL, a TEXT ); - CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id); - CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id); - CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgTrainingTableName} (md5(q), md5(a), user_id, kb_id); + CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgDatasetTableName} USING HASH (user_id); + CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgDatasetTableName} USING HASH (kb_id); + CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgDatasetTableName} (md5(q), md5(a), user_id, kb_id); `); console.log('init pg successful'); } catch (error) { diff --git a/client/src/store/static.ts b/client/src/store/static.ts index 6e28194aa..7c5ae31fa 100644 --- a/client/src/store/static.ts +++ b/client/src/store/static.ts @@ -8,6 +8,7 @@ import { getInitData } from '@/api/system'; import { delay } from '@/utils/tools'; import { FeConfigsType } from '@/types'; +export let systemVersion = '0.0.0'; export let chatModelList: ChatModelItemType[] = []; export let qaModel: QAModelItemType = { model: 'gpt-3.5-turbo-16k', @@ -28,6 +29,7 @@ export const clientInitData = async (): Promise => { qaModel = res.qaModel; vectorModelList = res.vectorModels; feConfigs = res.feConfigs; + systemVersion = res.systemVersion; return res; } catch (error) { diff --git a/client/src/utils/file.ts b/client/src/utils/file.ts index 008d50607..dc0159154 100644 --- a/client/src/utils/file.ts +++ b/client/src/utils/file.ts @@ -2,7 +2,7 @@ import mammoth from 'mammoth'; import Papa from 'papaparse'; import { getOpenAiEncMap } from './plugin/openai'; import { getErrText } from './tools'; -import { uploadImg, postUploadFiles } from '@/api/system'; +import { uploadImg, postUploadFiles } from '@/api/support/file'; /** * upload file to mongo gridfs diff --git a/docSite/content/docs/installation/upgrading/44 copy.md b/docSite/content/docs/installation/upgrading/44 copy.md new file mode 100644 index 000000000..5a6309747 --- /dev/null +++ b/docSite/content/docs/installation/upgrading/44 copy.md @@ -0,0 +1,23 @@ +--- +title: '升级到 V4.4.1' +description: 'FastGPT 从旧版本升级到 V4.4.1 操作指南' +icon: 'upgrade' +draft: false +toc: true +weight: 994 +--- + +## 执行初始化 API + +发起 1 个 HTTP 请求（记得携带 `headers.rootkey`，这个值是环境变量里的） + +1. https://xxxxx/api/admin/initv441 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv441' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +会给初始化 Mongo 的 dataset.files，将所有数据设置为可用。 +