From 23f22cda1828066c01779e541da18e83cb91c4ca Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Mon, 29 Jul 2024 09:47:30 +0800 Subject: [PATCH] Add mongo forward read. Fix markdown code block. (#2174) * perf: read from mongo forward node * fix: code block ui * perf: markdown code css --- .../service/common/file/image/controller.ts | 6 +- packages/service/common/mongo/utils.ts | 6 +- pnpm-lock.yaml | 41 ++++- .../app/src/components/Markdown/CodeLight.tsx | 7 +- .../src/components/Markdown/index.module.scss | 5 +- .../app/src/components/Markdown/index.tsx | 21 ++- .../app/src/pages/api/core/app/getChatLogs.ts | 174 +++++++++--------- .../src/pages/api/core/dataset/exportAll.ts | 6 +- .../training/getDatasetTrainingQueue.ts | 15 +- .../api/core/dataset/training/getQueueLen.ts | 28 +-- 10 files changed, 191 insertions(+), 118 deletions(-) diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index 6f6710a30..5c37a6165 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -3,6 +3,7 @@ import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; import { MongoImage } from './schema'; import { ClientSession } from '../../../common/mongo'; import { guessBase64ImageType } from '../utils'; +import { readFromSecondary } from '../../mongo/utils'; export function getMongoImgUrl(id: string, extension: string) { return `${imageBaseUrl}${id}.${extension}`; @@ -44,10 +45,13 @@ export async function uploadMongoImg({ export async function readMongoImg({ id }: { id: string }) { const formatId = id.replace(/\.[^/.]+$/, ''); - const data = await MongoImage.findById(formatId); + const data = await MongoImage.findById(formatId, undefined, { + ...readFromSecondary + }); if (!data) { return Promise.reject('Image not found'); } + return { binary: data.binary, mime: data.metadata?.mime ?? guessBase64ImageType(data.binary.toString('base64')) diff --git a/packages/service/common/mongo/utils.ts b/packages/service/common/mongo/utils.ts index 77f5bf9f7..294bfdd47 100644 --- a/packages/service/common/mongo/utils.ts +++ b/packages/service/common/mongo/utils.ts @@ -1,4 +1,6 @@ +import { ReadPreference } from './index'; + export const readFromSecondary = { - readPreference: 'secondaryPreferred', - readConcern: 'local' + readPreference: ReadPreference.SECONDARY_PREFERRED, // primary | primaryPreferred | secondary | secondaryPreferred | nearest + readConcern: 'local' as any // local | majority | linearizable | available }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cf87dfdc4..c3e316568 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -61,7 +61,7 @@ importers: version: 4.0.2 next: specifier: 14.2.5 - version: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) + version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) openai: specifier: 4.53.0 version: 4.53.0(encoding@0.1.13) @@ -180,7 +180,7 @@ importers: version: 1.4.5-lts.1 next: specifier: 14.2.5 - version: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) + version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) nextjs-cors: specifier: ^2.2.0 version: 2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)) @@ -10036,7 +10036,7 @@ snapshots: '@chakra-ui/react': 2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@emotion/cache': 11.11.0 '@emotion/react': 11.11.1(@types/react@18.3.1)(react@18.3.1) - next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) + next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) react: 18.3.1 '@chakra-ui/number-input@2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)': @@ -16652,7 +16652,7 @@ snapshots: hoist-non-react-statics: 3.3.2 i18next: 23.11.5 i18next-fs-backend: 2.3.1 - next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) + next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) react: 18.3.1 react-i18next: 14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -16682,10 +16682,36 @@ snapshots: - '@babel/core' - babel-plugin-macros + next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8): + dependencies: + '@next/env': 14.2.5 + '@swc/helpers': 0.5.5 + busboy: 1.6.0 + caniuse-lite: 1.0.30001642 + graceful-fs: 4.2.11 + postcss: 8.4.31 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + styled-jsx: 5.1.1(react@18.3.1) + optionalDependencies: + '@next/swc-darwin-arm64': 14.2.5 + '@next/swc-darwin-x64': 14.2.5 + '@next/swc-linux-arm64-gnu': 14.2.5 + '@next/swc-linux-arm64-musl': 14.2.5 + '@next/swc-linux-x64-gnu': 14.2.5 + '@next/swc-linux-x64-musl': 14.2.5 + '@next/swc-win32-arm64-msvc': 14.2.5 + '@next/swc-win32-ia32-msvc': 14.2.5 + '@next/swc-win32-x64-msvc': 14.2.5 + sass: 1.77.8 + transitivePeerDependencies: + - '@babel/core' + - babel-plugin-macros + nextjs-cors@2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)): dependencies: cors: 2.8.5 - next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) + next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8) nextjs-node-loader@1.1.5(webpack@5.92.1): dependencies: @@ -18058,6 +18084,11 @@ snapshots: optionalDependencies: '@babel/core': 7.24.9 + styled-jsx@5.1.1(react@18.3.1): + dependencies: + client-only: 0.0.1 + react: 18.3.1 + stylis@4.2.0: {} stylis@4.3.2: {} diff --git a/projects/app/src/components/Markdown/CodeLight.tsx b/projects/app/src/components/Markdown/CodeLight.tsx index 87541d5fc..2c2e59b07 100644 --- a/projects/app/src/components/Markdown/CodeLight.tsx +++ b/projects/app/src/components/Markdown/CodeLight.tsx @@ -287,18 +287,18 @@ const codeLight: { [key: string]: React.CSSProperties } = { const CodeLight = ({ children, className, - inline, + codeBlock, match }: { children: React.ReactNode & React.ReactNode[]; className?: string; - inline?: boolean; + codeBlock?: boolean; match: RegExpExecArray | null; }) => { const { t } = useTranslation(); const { copyData } = useCopyData(); - if (!inline) { + if (codeBlock) { const codeBoxName = useMemo(() => { const input = match?.['input'] || ''; if (!input) return match?.[1]; @@ -312,7 +312,6 @@ const CodeLight = ({ my={3} borderRadius={'md'} overflow={'overlay'} - bg={'myGray.900'} boxShadow={ '0px 0px 1px 0px rgba(19, 51, 107, 0.08), 0px 1px 2px 0px rgba(19, 51, 107, 0.05)' } diff --git a/projects/app/src/components/Markdown/index.module.scss b/projects/app/src/components/Markdown/index.module.scss index c71595799..52332b6c1 100644 --- a/projects/app/src/components/Markdown/index.module.scss +++ b/projects/app/src/components/Markdown/index.module.scss @@ -336,7 +336,7 @@ } .markdown pre code, .markdown pre tt { - background-color: transparent; + background-color: transparent !important; border: medium none; } .markdown hr { @@ -360,13 +360,12 @@ margin: 0; border: none; border-radius: 0; - background-color: #292b33 !important; + background-color: var(--chakra-colors-gray-900) !important; overflow-x: auto; color: #fff; } pre code { - background-color: #292b33 !important; width: 100%; } diff --git a/projects/app/src/components/Markdown/index.tsx b/projects/app/src/components/Markdown/index.tsx index aa044e7fb..f308fc4e0 100644 --- a/projects/app/src/components/Markdown/index.tsx +++ b/projects/app/src/components/Markdown/index.tsx @@ -36,7 +36,7 @@ const Markdown = ({ const components = useMemo( () => ({ img: Image, - pre: 'div', + pre: RewritePre, p: (pProps: any) =>

, code: Code, a: A @@ -70,8 +70,7 @@ export default React.memo(Markdown); /* Custom dom */ const Code = React.memo(function Code(e: any) { - const { inline, className, children } = e; - + const { className, codeBlock, children } = e; const match = /language-(\w+)/.exec(className || ''); const codeType = match?.[1]; @@ -92,11 +91,11 @@ const Code = React.memo(function Code(e: any) { } return ( - + {children} ); - }, [codeType, className, inline, match, children, strChildren]); + }, [codeType, className, codeBlock, match, children, strChildren]); return Component; }); @@ -149,3 +148,15 @@ const A = React.memo(function A({ children, ...props }: any) { return {children}; }); + +function RewritePre({ children }: any) { + const modifiedChildren = React.Children.map(children, (child) => { + if (React.isValidElement(child)) { + // @ts-ignore + return React.cloneElement(child, { codeBlock: true }); + } + return child; + }); + + return <>{modifiedChildren}; +} diff --git a/projects/app/src/pages/api/core/app/getChatLogs.ts b/projects/app/src/pages/api/core/app/getChatLogs.ts index 9efec2e2d..8149f413b 100644 --- a/projects/app/src/pages/api/core/app/getChatLogs.ts +++ b/projects/app/src/pages/api/core/app/getChatLogs.ts @@ -9,6 +9,7 @@ import { authApp } from '@fastgpt/service/support/permission/app/auth'; import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema'; import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; +import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; async function handler( req: NextApiRequest, @@ -39,101 +40,106 @@ async function handler( }; const [data, total] = await Promise.all([ - MongoChat.aggregate([ - { $match: where }, - { - $sort: { - userBadFeedbackCount: -1, - userGoodFeedbackCount: -1, - customFeedbacksCount: -1, - updateTime: -1 - } - }, - { $skip: (pageNum - 1) * pageSize }, - { $limit: pageSize }, - { - $lookup: { - from: ChatItemCollectionName, - let: { chatId: '$chatId' }, - pipeline: [ - { - $match: { - $expr: { - $and: [ - { $eq: ['$appId', new Types.ObjectId(appId)] }, - { $eq: ['$chatId', '$$chatId'] } - ] + MongoChat.aggregate( + [ + { $match: where }, + { + $sort: { + userBadFeedbackCount: -1, + userGoodFeedbackCount: -1, + customFeedbacksCount: -1, + updateTime: -1 + } + }, + { $skip: (pageNum - 1) * pageSize }, + { $limit: pageSize }, + { + $lookup: { + from: ChatItemCollectionName, + let: { chatId: '$chatId' }, + pipeline: [ + { + $match: { + $expr: { + $and: [ + { $eq: ['$appId', new Types.ObjectId(appId)] }, + { $eq: ['$chatId', '$$chatId'] } + ] + } + } + }, + { + $project: { + userGoodFeedback: 1, + userBadFeedback: 1, + customFeedbacks: 1, + adminFeedback: 1 + } + } + ], + as: 'chatitems' + } + }, + { + $addFields: { + userGoodFeedbackCount: { + $size: { + $filter: { + input: '$chatitems', + as: 'item', + cond: { $ifNull: ['$$item.userGoodFeedback', false] } } } }, - { - $project: { - userGoodFeedback: 1, - userBadFeedback: 1, - customFeedbacks: 1, - adminFeedback: 1 + userBadFeedbackCount: { + $size: { + $filter: { + input: '$chatitems', + as: 'item', + cond: { $ifNull: ['$$item.userBadFeedback', false] } + } } - } - ], - as: 'chatitems' - } - }, - { - $addFields: { - userGoodFeedbackCount: { - $size: { - $filter: { - input: '$chatitems', - as: 'item', - cond: { $ifNull: ['$$item.userGoodFeedback', false] } + }, + customFeedbacksCount: { + $size: { + $filter: { + input: '$chatitems', + as: 'item', + cond: { $gt: [{ $size: { $ifNull: ['$$item.customFeedbacks', []] } }, 0] } + } } - } - }, - userBadFeedbackCount: { - $size: { - $filter: { - input: '$chatitems', - as: 'item', - cond: { $ifNull: ['$$item.userBadFeedback', false] } - } - } - }, - customFeedbacksCount: { - $size: { - $filter: { - input: '$chatitems', - as: 'item', - cond: { $gt: [{ $size: { $ifNull: ['$$item.customFeedbacks', []] } }, 0] } - } - } - }, - markCount: { - $size: { - $filter: { - input: '$chatitems', - as: 'item', - cond: { $ifNull: ['$$item.adminFeedback', false] } + }, + markCount: { + $size: { + $filter: { + input: '$chatitems', + as: 'item', + cond: { $ifNull: ['$$item.adminFeedback', false] } + } } } } + }, + { + $project: { + _id: 1, + id: '$chatId', + title: 1, + source: 1, + time: '$updateTime', + messageCount: { $size: '$chatitems' }, + userGoodFeedbackCount: 1, + userBadFeedbackCount: 1, + customFeedbacksCount: 1, + markCount: 1 + } } - }, + ], { - $project: { - _id: 1, - id: '$chatId', - title: 1, - source: 1, - time: '$updateTime', - messageCount: { $size: '$chatitems' }, - userGoodFeedbackCount: 1, - userBadFeedbackCount: 1, - customFeedbacksCount: 1, - markCount: 1 - } + ...readFromSecondary } - ]), - MongoChat.countDocuments(where) + ), + MongoChat.countDocuments(where, { ...readFromSecondary }) ]); return { diff --git a/projects/app/src/pages/api/core/dataset/exportAll.ts b/projects/app/src/pages/api/core/dataset/exportAll.ts index a31751d59..0c52ea6d7 100644 --- a/projects/app/src/pages/api/core/dataset/exportAll.ts +++ b/projects/app/src/pages/api/core/dataset/exportAll.ts @@ -11,6 +11,7 @@ import { import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; async function handler(req: NextApiRequest, res: NextApiResponse) { let { datasetId } = req.query as { @@ -53,7 +54,10 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { teamId, datasetId: { $in: datasets.map((d) => d._id) } }, - 'q a' + 'q a', + { + ...readFromSecondary + } ) .limit(50000) .cursor(); diff --git a/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts b/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts index 1b2ed2b5c..ba05dc6bf 100644 --- a/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts +++ b/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts @@ -4,6 +4,7 @@ import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; export type getDatasetTrainingQueueResponse = { rebuildingCount: number; @@ -24,8 +25,18 @@ async function handler( }); const [rebuildingCount, trainingCount] = await Promise.all([ - MongoDatasetData.countDocuments({ rebuilding: true, teamId, datasetId }), - MongoDatasetTraining.countDocuments({ teamId, datasetId }) + MongoDatasetData.countDocuments( + { rebuilding: true, teamId, datasetId }, + { + ...readFromSecondary + } + ), + MongoDatasetTraining.countDocuments( + { teamId, datasetId }, + { + ...readFromSecondary + } + ) ]); return { diff --git a/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts b/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts index bc17435c0..504176dd6 100644 --- a/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts +++ b/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts @@ -3,6 +3,7 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/sch import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { GetTrainingQueueProps } from '@/global/core/dataset/api'; import { NextAPI } from '@/service/middleware/entry'; +import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; async function handler(req: NextApiRequest) { await authCert({ req, authToken: true }); @@ -10,20 +11,25 @@ async function handler(req: NextApiRequest) { // get queue data // 分别统计 model = vectorModel和agentModel的数量 - const data = await MongoDatasetTraining.aggregate([ - { - $match: { - lockTime: { $lt: new Date('2040/1/1') }, - $or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }] + const data = await MongoDatasetTraining.aggregate( + [ + { + $match: { + lockTime: { $lt: new Date('2040/1/1') }, + $or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }] + } + }, + { + $group: { + _id: '$model', + count: { $sum: 1 } + } } - }, + ], { - $group: { - _id: '$model', - count: { $sum: 1 } - } + ...readFromSecondary } - ]); + ); const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0; const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;