4.7.1-alpha (#1120)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-04-03 18:14:09 +08:00
committed by GitHub
parent 9ae581e09b
commit 8a46372418
76 changed files with 3129 additions and 2104 deletions

View File

@@ -8,7 +8,8 @@ import {
Input,
Link,
Progress,
Grid
Grid,
Image
} from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import { UserUpdateParams } from '@/types/user';
@@ -41,12 +42,14 @@ import {
} from '@/web/support/wallet/sub/constants';
import StandardPlanContentList from '@/components/support/wallet/StandardPlanContentList';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
const StandDetailModal = dynamic(() => import('./standardDetailModal'));
const TeamMenu = dynamic(() => import('@/components/support/user/team/TeamMenu'));
const PayModal = dynamic(() => import('./PayModal'));
const UpdatePswModal = dynamic(() => import('./UpdatePswModal'));
const OpenAIAccountModal = dynamic(() => import('./OpenAIAccountModal'));
const LafAccountModal = dynamic(() => import('@/components/support/laf/LafAccountModal'));
const CommunityModal = dynamic(() => import('@/components/CommunityModal'));
const Account = () => {
@@ -518,7 +521,7 @@ const Other = () => {
const { reset } = useForm<UserUpdateParams>({
defaultValues: userInfo as UserType
});
const { isOpen: isOpenLaf, onClose: onCloseLaf, onOpen: onOpenLaf } = useDisclosure();
const { isOpen: isOpenOpenai, onClose: onCloseOpenai, onOpen: onOpenOpenai } = useDisclosure();
const { isOpen: isOpenConcat, onClose: onCloseConcat, onOpen: onOpenConcat } = useDisclosure();
@@ -537,7 +540,6 @@ const Other = () => {
},
[reset, toast, updateUserInfo]
);
return (
<Box>
<Grid gridGap={4} mt={3}>
@@ -582,6 +584,32 @@ const Other = () => {
</Box>
</Link>
{feConfigs?.lafEnv && userInfo?.team.role === TeamMemberRoleEnum.owner && (
<Flex
bg={'white'}
py={4}
px={6}
border={theme.borders.sm}
borderWidth={'1.5px'}
borderRadius={'md'}
alignItems={'center'}
cursor={'pointer'}
userSelect={'none'}
onClick={onOpenLaf}
>
<Image src="/imgs/module/laf.png" w={'18px'} alt="laf" />
<Box ml={2} flex={1}>
laf
</Box>
<Box
w={'9px'}
h={'9px'}
borderRadius={'50%'}
bg={userInfo?.team.lafAccount?.token ? '#67c13b' : 'myGray.500'}
/>
</Flex>
)}
{feConfigs?.show_openai_account && (
<Flex
bg={'white'}
@@ -620,6 +648,9 @@ const Other = () => {
)}
</Grid>
{isOpenLaf && userInfo && (
<LafAccountModal defaultData={userInfo?.team.lafAccount} onClose={onCloseLaf} />
)}
{isOpenOpenai && userInfo && (
<OpenAIAccountModal
defaultData={userInfo?.openaiAccount}

View File

@@ -2,12 +2,14 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { checkFiles } from '../timerTask/dataset/checkInValidDatasetFiles';
import { addHours } from 'date-fns';
import { checkInvalidCollection } from '../timerTask/dataset/checkInvalidMongoCollection';
import { checkInvalidVector } from '../timerTask/dataset/checkInvalidVector';
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import {
checkInvalidDatasetFiles,
checkInvalidDatasetData,
checkInvalidVector
} from '@/service/common/system/cronTask';
let deleteImageAmount = 0;
async function checkInvalidImg(start: Date, end: Date, limit = 50) {
@@ -60,11 +62,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
(async () => {
try {
console.log('执行脏数据清理任务');
const end = addHours(new Date(), -1);
// 360天 ~ 2小时前
const end = addHours(new Date(), -2);
const start = addHours(new Date(), -360 * 24);
await checkFiles(start, end);
await checkInvalidDatasetFiles(start, end);
await checkInvalidImg(start, end);
await checkInvalidCollection(start, end);
await checkInvalidDatasetData(start, end);
await checkInvalidVector(start, end);
console.log('执行脏数据清理任务完毕');
} catch (error) {

View File

@@ -1,98 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoUsage } from '@fastgpt/service/support/wallet/usage/schema';
import { connectionMongo } from '@fastgpt/service/common/mongo';
import { checkFiles } from '../timerTask/dataset/checkInValidDatasetFiles';
import { addHours } from 'date-fns';
import { checkInvalidCollection } from '../timerTask/dataset/checkInvalidMongoCollection';
import { checkInvalidVector } from '../timerTask/dataset/checkInvalidVector';
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
let deleteImageAmount = 0;
export async function checkInvalidImg(start: Date, end: Date, limit = 50) {
const images = await MongoImage.find(
{
createTime: {
$gte: start,
$lte: end
},
'metadata.relatedId': { $exists: true }
},
'_id teamId metadata'
);
console.log('total images', images.length);
let index = 0;
for await (const image of images) {
try {
// 1. 检测是否有对应的集合
const collection = await MongoDatasetCollection.findOne(
{
teamId: image.teamId,
'metadata.relatedImgId': image.metadata?.relatedId
},
'_id'
);
if (!collection) {
await image.deleteOne();
deleteImageAmount++;
}
index++;
index % 100 === 0 && console.log(index);
} catch (error) {
console.log(error);
}
}
console.log(`检测完成,共删除 ${deleteImageAmount} 个无效图片`);
}
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
// 检查 usage 是否有记录
const totalUsages = await MongoUsage.countDocuments();
if (totalUsages === 0) {
// 重命名 bills 集合成 usages
await connectionMongo.connection.db.renameCollection('bills', 'usages', {
// 强制
dropTarget: true
});
}
(async () => {
try {
console.log('执行脏数据清理任务');
const end = addHours(new Date(), -1);
const start = addHours(new Date(), -360 * 24);
await checkFiles(start, end);
await checkInvalidImg(start, end);
await checkInvalidCollection(start, end);
await checkInvalidVector(start, end);
console.log('执行脏数据清理任务完毕');
} catch (error) {
console.log('执行脏数据清理任务出错了');
}
})();
jsonRes(res, {
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}

View File

@@ -4,7 +4,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { readFileContent } from '@fastgpt/service/common/file/gridfs/controller';
import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
import { authFile } from '@fastgpt/service/support/permission/auth/file';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
@@ -19,7 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { teamId } = await authFile({ req, authToken: true, fileId });
const { rawText } = await readFileContent({
const { rawText } = await readFileContentFromMongo({
teamId,
bucketName: BucketNameEnum.dataset,
fileId,

View File

@@ -90,6 +90,7 @@ export async function initSystemConfig() {
// get config from database
const config: FastGPTConfigFileType = {
feConfigs: {
...fileRes?.feConfigs,
...defaultFeConfigs,
...(dbConfig.feConfigs || {}),
isPlus: !!FastGPTProUrl

View File

@@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { readFileContent } from '@fastgpt/service/common/file/gridfs/controller';
import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
@@ -36,7 +36,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// 1. read file
const { rawText, filename } = await readFileContent({
const { rawText, filename } = await readFileContentFromMongo({
teamId,
bucketName: BucketNameEnum.dataset,
fileId

View File

@@ -3,7 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import {
delFileByFileIdList,
readFileContent
readFileContentFromMongo
} from '@fastgpt/service/common/file/gridfs/controller';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
@@ -47,7 +47,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// 1. read file
const { rawText, filename } = await readFileContent({
const { rawText, filename } = await readFileContentFromMongo({
teamId,
bucketName: BucketNameEnum.dataset,
fileId

View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { authFile } from '@fastgpt/service/support/permission/auth/file';
import { PostPreviewFilesChunksProps } from '@/global/core/dataset/api';
import { readFileContent } from '@fastgpt/service/common/file/gridfs/controller';
import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { parseCsvTable2Chunks } from '@fastgpt/service/core/dataset/training/utils';
@@ -28,7 +28,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { file, teamId } = await authFile({ req, authToken: true, fileId: sourceId });
const fileId = String(file._id);
const { rawText } = await readFileContent({
const { rawText } = await readFileContentFromMongo({
teamId,
bucketName: BucketNameEnum.dataset,
fileId,
@@ -53,7 +53,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
if (type === ImportDataSourceEnum.csvTable) {
const { file, teamId } = await authFile({ req, authToken: true, fileId: sourceId });
const fileId = String(file._id);
const { rawText } = await readFileContent({
const { rawText } = await readFileContentFromMongo({
teamId,
bucketName: BucketNameEnum.dataset,
fileId,

View File

@@ -0,0 +1,69 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { request } from 'https';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
import url from 'url';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { path = [], ...query } = req.query as any;
const queryStr = new URLSearchParams(query).toString();
const requestPath = queryStr
? `/${path?.join('/')}?${new URLSearchParams(query).toString()}`
: `/${path?.join('/')}`;
if (!requestPath) {
throw new Error('url is empty');
}
const lafEnv = global.feConfigs?.lafEnv;
if (!lafEnv) {
throw new Error('lafEnv is empty');
}
const parsedUrl = url.parse(lafEnv);
delete req.headers?.cookie;
delete req.headers?.host;
delete req.headers?.origin;
const requestResult = request({
protocol: parsedUrl.protocol,
hostname: parsedUrl.hostname,
port: parsedUrl.port,
path: requestPath,
method: req.method,
headers: req.headers,
timeout: 30000
});
req.pipe(requestResult);
requestResult.on('response', (response) => {
Object.keys(response.headers).forEach((key) => {
// @ts-ignore
res.setHeader(key, response.headers[key]);
});
response.statusCode && res.writeHead(response.statusCode);
response.pipe(res);
});
requestResult.on('error', (e) => {
res.send(e);
res.end();
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
export const config = {
api: {
bodyParser: false
}
};

View File

@@ -11,7 +11,7 @@ import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSc
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { avatar, timezone, openaiAccount } = req.body as UserUpdateParams;
const { avatar, timezone, openaiAccount, lafAccount } = req.body as UserUpdateParams;
const { tmbId } = await authCert({ req, authToken: true });
const tmb = await MongoTeamMember.findById(tmbId);
@@ -47,7 +47,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
...(avatar && { avatar }),
...(timezone && { timezone }),
openaiAccount: openaiAccount?.key ? openaiAccount : null
openaiAccount: openaiAccount?.key ? openaiAccount : null,
lafAccount: lafAccount?.token ? lafAccount : null
}
);

View File

@@ -1,91 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import {
delFileByFileIdList,
getGFSCollection
} from '@fastgpt/service/common/file/gridfs/controller';
import { addLog } from '@fastgpt/service/common/system/log';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { addHours } from 'date-fns';
/*
check dataset.files data. If there is no match in dataset.collections, delete it
可能异常情况
1. 上传了文件,未成功创建集合
*/
let deleteFileAmount = 0;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { startHour = 24, endHour = 1 } = req.body as {
startHour?: number;
endHour?: number;
limit?: number;
};
await authCert({ req, authRoot: true });
await connectToDatabase();
// start: now - maxDay, end: now - 3 day
const start = addHours(new Date(), -startHour);
const end = addHours(new Date(), -endHour);
deleteFileAmount = 0;
console.log(start, end);
await checkFiles(start, end);
jsonRes(res, {
data: deleteFileAmount,
message: 'success'
});
} catch (error) {
addLog.error(`check valid dataset files error`, error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function checkFiles(start: Date, end: Date) {
const collection = getGFSCollection('dataset');
const where = {
uploadDate: { $gte: start, $lte: end }
};
// 1. get all file _id
const files = await collection
.find(where, {
projection: {
metadata: 1,
_id: 1
}
})
.toArray();
console.log('total files', files.length);
let index = 0;
for await (const file of files) {
try {
// 2. find fileId in dataset.collections
const hasCollection = await MongoDatasetCollection.countDocuments({
teamId: file.metadata.teamId,
fileId: file._id
});
// 3. if not found, delete file
if (hasCollection === 0) {
await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(file._id)] });
console.log('delete file', file._id);
deleteFileAmount++;
}
index++;
index % 100 === 0 && console.log(index);
} catch (error) {
console.log(error);
}
}
console.log(`检测完成,共删除 ${deleteFileAmount} 个无效文件`);
}

View File

@@ -1,96 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { addLog } from '@fastgpt/service/common/system/log';
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { addHours } from 'date-fns';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
/*
检测无效的 Mongo 数据
异常情况:
1. 训练过程删除知识库,可能导致还会有新的数据插入,导致无效。
*/
let deleteAmount = 0;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { startHour = 3, endHour = 1 } = req.body as { startHour?: number; endHour?: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
// start: now - maxDay, end: now - endHour
const start = addHours(new Date(), -startHour);
const end = addHours(new Date(), -endHour);
deleteAmount = 0;
await checkInvalidCollection(start, end);
jsonRes(res, {
data: deleteAmount,
message: 'success'
});
} catch (error) {
addLog.error(`check Invalid user error`, error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function checkInvalidCollection(start: Date, end: Date) {
// 1. 获取时间范围的所有data
const rows = await MongoDatasetData.find(
{
updateTime: {
$gte: start,
$lte: end
}
},
'_id teamId collectionId'
).lean();
// 2. 合并所有的collectionId
const map = new Map<string, { teamId: string; collectionId: string }>();
for (const item of rows) {
const collectionId = String(item.collectionId);
if (!map.has(collectionId)) {
map.set(collectionId, { teamId: item.teamId, collectionId });
}
}
const list = Array.from(map.values());
console.log('total collections', list.length);
let index = 0;
for await (const item of list) {
try {
// 3. 查看该collection是否存在不存在则删除对应的数据
const collection = await MongoDatasetCollection.findOne({ _id: item.collectionId });
if (!collection) {
const result = await Promise.all([
MongoDatasetTraining.deleteMany({
teamId: item.teamId,
collectionId: item.collectionId
}),
MongoDatasetData.deleteMany({
teamId: item.teamId,
collectionId: item.collectionId
}),
deleteDatasetDataVector({
teamId: item.teamId,
collectionIds: [String(item.collectionId)]
})
]);
console.log(result);
console.log('collection is not found', item);
continue;
}
} catch (error) {}
console.log(++index);
}
}

View File

@@ -1,86 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { addLog } from '@fastgpt/service/common/system/log';
import {
deleteDatasetDataVector,
getVectorDataByTime
} from '@fastgpt/service/common/vectorStore/controller';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { addHours } from 'date-fns';
/*
检测无效的 Vector 数据.
异常情况:
1. 插入数据时vector成功mongo失败
2. 更新数据,也会有插入 vector
*/
let deletedVectorAmount = 0;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { startHour = 5, endHour = 1 } = req.body as { startHour?: number; endHour?: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
// start: now - maxDay, end: now - endHour
const start = addHours(new Date(), -startHour);
const end = addHours(new Date(), -endHour);
deletedVectorAmount = 0;
await checkInvalidVector(start, end);
jsonRes(res, {
data: deletedVectorAmount,
message: 'success'
});
} catch (error) {
addLog.error(`check Invalid user error`, error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function checkInvalidVector(start: Date, end: Date) {
// 1. get all vector data
const rows = await getVectorDataByTime(start, end);
console.log('total data', rows.length);
let index = 0;
for await (const item of rows) {
if (!item.teamId || !item.datasetId || !item.id) {
console.log('error data', item);
continue;
}
try {
// 2. find dataset.data
const hasData = await MongoDatasetData.countDocuments({
teamId: item.teamId,
datasetId: item.datasetId,
'indexes.dataId': item.id
});
// 3. if not found, delete vector
if (hasData === 0) {
await deleteDatasetDataVector({
teamId: item.teamId,
id: item.id
});
console.log('delete vector data', item.id);
deletedVectorAmount++;
}
index++;
index % 100 === 0 && console.log(index);
} catch (error) {
console.log(error);
}
}
console.log(`检测完成,共删除 ${deletedVectorAmount} 个无效 向量 数据`);
}

View File

@@ -117,3 +117,5 @@ export const RenderUploadFiles = ({
</>
) : null;
};
export default RenderUploadFiles;

View File

@@ -249,7 +249,7 @@ const InputDataModal = ({
return openConfirm(onDeleteData)();
}
if (e === TabEnum.doc) {
return window.open(getDocPath('/docs/course/datasetengine'), '_blank');
return window.open(getDocPath('/docs/course/dataset_engine'), '_blank');
}
setCurrentTab(e);
}}