From 58745f8c35094e4fcc200687a9698103e8033115 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 15 Nov 2024 10:35:04 +0800 Subject: [PATCH] 4.8.14 test (#3164) * perf: match base 64 image * perf: register plugins --- packages/global/common/string/markdown.ts | 25 ++++++++++++++++- packages/plugins/register.ts | 2 ++ packages/service/common/file/read/utils.ts | 14 +++++++--- packages/service/worker/htmlStr2Md/utils.ts | 18 ++---------- .../support/user/inform/SystemMsgModal.tsx | 6 ++-- .../nodes/render/Handle/ConnectionHandle.tsx | 28 +++++++++++-------- .../service/common/system/volumnMongoWatch.ts | 4 +-- projects/app/src/service/core/app/plugin.ts | 19 ++++++++----- projects/app/src/service/mongo.ts | 4 +-- 9 files changed, 76 insertions(+), 44 deletions(-) diff --git a/packages/global/common/string/markdown.ts b/packages/global/common/string/markdown.ts index 388fb0233..4c26c6ff0 100644 --- a/packages/global/common/string/markdown.ts +++ b/packages/global/common/string/markdown.ts @@ -1,5 +1,6 @@ import { batchRun } from '../fn/utils'; -import { simpleText } from './tools'; +import { getNanoid, simpleText } from './tools'; +import type { ImageType } from '../../../service/worker/readFile/type'; /* Delete redundant text in markdown */ export const simpleMarkdownText = (rawText: string) => { @@ -92,3 +93,25 @@ export const markdownProcess = async ({ return simpleMarkdownText(imageProcess); }; + +export const matchMdImgTextAndUpload = (text: string) => { + const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g; + const imageList: ImageType[] = []; + const images = Array.from(text.match(base64Regex) || []); + for (const image of images) { + const uuid = `IMAGE_${getNanoid(12)}_IMAGE`; + const mime = image.split(';')[0].split(':')[1]; + const base64 = image.split(',')[1]; + text = text.replace(image, uuid); + imageList.push({ + uuid, + base64, + mime + }); + } + + return { + text, + imageList + }; +}; diff --git a/packages/plugins/register.ts b/packages/plugins/register.ts index baa3b2c97..826a41721 100644 --- a/packages/plugins/register.ts +++ b/packages/plugins/register.ts @@ -46,6 +46,8 @@ export const getCommunityPlugins = () => { }; export const getSystemPluginTemplates = () => { + if (!global.systemPlugins) return []; + const oldPlugins = global.communityPlugins ?? []; return [...oldPlugins, ...cloneDeep(global.systemPlugins)]; }; diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index 578f078b6..ee898c1a6 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -4,12 +4,12 @@ import FormData from 'form-data'; import { WorkerNameEnum, runWorker } from '../../../worker/utils'; import fs from 'fs'; -import { detectFileEncoding } from '@fastgpt/global/common/file/tools'; import type { ReadFileResponse } from '../../../worker/readFile/type'; import axios from 'axios'; import { addLog } from '../../system/log'; import { batchRun } from '@fastgpt/global/common/fn/utils'; import { addHours } from 'date-fns'; +import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown'; export type readRawTextByLocalFileParams = { teamId: string; @@ -79,6 +79,7 @@ export const readRawContentByFileBuffer = async ({ data: { page: number; markdown: string; + duration: number; }; }>(customReadfileUrl, data, { timeout: 600000, @@ -90,10 +91,12 @@ export const readRawContentByFileBuffer = async ({ addLog.info(`Use custom read file service, time: ${Date.now() - start}ms`); const rawText = response.data.markdown; + const { text, imageList } = matchMdImgTextAndUpload(rawText); return { - rawText, - formatText: rawText + rawText: text, + formatText: rawText, + imageList }; }; @@ -120,6 +123,9 @@ export const readRawContentByFileBuffer = async ({ } }); rawText = rawText.replace(item.uuid, src); + if (formatText) { + formatText = formatText.replace(item.uuid, src); + } }); } @@ -128,7 +134,7 @@ export const readRawContentByFileBuffer = async ({ if (isQAImport) { rawText = rawText || ''; } else { - rawText = formatText || ''; + rawText = formatText || rawText; } } diff --git a/packages/service/worker/htmlStr2Md/utils.ts b/packages/service/worker/htmlStr2Md/utils.ts index 165b54b6d..38112b92d 100644 --- a/packages/service/worker/htmlStr2Md/utils.ts +++ b/packages/service/worker/htmlStr2Md/utils.ts @@ -1,5 +1,6 @@ import TurndownService from 'turndown'; import { ImageType } from '../readFile/type'; +import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown'; // @ts-ignore const turndownPluginGfm = require('joplin-turndown-plugin-gfm'); @@ -24,23 +25,10 @@ export const html2md = ( turndownService.remove(['i', 'script', 'iframe', 'style']); turndownService.use(turndownPluginGfm.gfm); - const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g; - const imageList: ImageType[] = []; - const images = Array.from(html.match(base64Regex) || []); - for (const image of images) { - const uuid = crypto.randomUUID(); - const mime = image.split(';')[0].split(':')[1]; - const base64 = image.split(',')[1]; - html = html.replace(image, uuid); - imageList.push({ - uuid, - base64, - mime - }); - } + const { text, imageList } = matchMdImgTextAndUpload(html); return { - rawText: turndownService.turndown(html), + rawText: turndownService.turndown(text), imageList }; } catch (error) { diff --git a/projects/app/src/components/support/user/inform/SystemMsgModal.tsx b/projects/app/src/components/support/user/inform/SystemMsgModal.tsx index cf880b4ca..1bf98e8ce 100644 --- a/projects/app/src/components/support/user/inform/SystemMsgModal.tsx +++ b/projects/app/src/components/support/user/inform/SystemMsgModal.tsx @@ -1,12 +1,12 @@ import React, { useCallback } from 'react'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { useUserStore } from '@/web/support/user/useUserStore'; -import { useQuery } from '@tanstack/react-query'; import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; import { LOGO_ICON } from '@fastgpt/global/common/system/constants'; import { getSystemMsgModalData } from '@/web/support/user/inform/api'; import dynamic from 'next/dynamic'; +import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; const Markdown = dynamic(() => import('@/components/Markdown'), { ssr: false }); const SystemMsgModal = ({}: {}) => { @@ -15,7 +15,9 @@ const SystemMsgModal = ({}: {}) => { const { isOpen, onOpen, onClose } = useDisclosure(); - const { data } = useQuery(['initSystemMsgModal', systemMsgReadId], getSystemMsgModalData, { + const { data } = useRequest2(getSystemMsgModalData, { + refreshDeps: [systemMsgReadId], + manual: false, onSuccess(res) { if (res?.content && (!systemMsgReadId || res.id !== systemMsgReadId)) { onOpen(); diff --git a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/Handle/ConnectionHandle.tsx b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/Handle/ConnectionHandle.tsx index 6137d094c..e7c0746a3 100644 --- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/Handle/ConnectionHandle.tsx +++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/Handle/ConnectionHandle.tsx @@ -6,6 +6,7 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import { useContextSelector } from 'use-context-selector'; import { WorkflowContext } from '../../../../context'; import { WorkflowNodeEdgeContext } from '../../../../context/workflowInitContext'; +import { FlowNodeItemType } from '@fastgpt/global/core/workflow/type/node'; export const ConnectionSourceHandle = ({ nodeId, @@ -141,15 +142,23 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle const { connectingEdge, nodeList } = useContextSelector(WorkflowContext, (ctx) => ctx); const { LeftHandle, rightHandle, topHandle, bottomHandle } = useMemo(() => { - const node = nodeList.find((node) => node.nodeId === nodeId); - const connectingNode = nodeList.find((node) => node.nodeId === connectingEdge?.nodeId); + let node: FlowNodeItemType | undefined = undefined, + connectingNode: FlowNodeItemType | undefined = undefined; + for (const item of nodeList) { + if (item.nodeId === nodeId) { + node = item; + } + if (item.nodeId === connectingEdge?.nodeId) { + connectingNode = item; + } + if (node && (connectingNode || !connectingEdge?.nodeId)) break; + } - const connectingNodeSourceNodeIdMap = new Map(); let forbidConnect = false; - edges.forEach((edge) => { - if (edge.target === connectingNode?.nodeId) { - connectingNodeSourceNodeIdMap.set(edge.source, 1); - } else if (edge.target === nodeId) { + for (const edge of edges) { + if (forbidConnect) break; + + if (edge.target === nodeId) { // Node has be connected tool, it cannot be connect by other handle if (edge.targetHandle === NodeOutputKeyEnum.selectedTools) { forbidConnect = true; @@ -163,7 +172,7 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle forbidConnect = true; } } - }); + } const showHandle = (() => { if (forbidConnect) return false; @@ -178,9 +187,6 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle // Not the same parent node if (connectingNode && connectingNode?.parentNodeId !== node?.parentNodeId) return false; - // Unable to connect to the source node - if (connectingNodeSourceNodeIdMap.has(nodeId)) return false; - return true; })(); diff --git a/projects/app/src/service/common/system/volumnMongoWatch.ts b/projects/app/src/service/common/system/volumnMongoWatch.ts index 5b3ed2983..37ae25f24 100644 --- a/projects/app/src/service/common/system/volumnMongoWatch.ts +++ b/projects/app/src/service/common/system/volumnMongoWatch.ts @@ -1,4 +1,4 @@ -import { getSystemPluginCb, getSystemPlugins } from '@/service/core/app/plugin'; +import { getSystemPluginCb } from '@/service/core/app/plugin'; import { initSystemConfig } from '.'; import { createDatasetTrainingMongoWatch } from '@/service/core/dataset/training/utils'; import { MongoSystemConfigs } from '@fastgpt/service/common/system/config/schema'; @@ -29,7 +29,7 @@ const refetchSystemPlugins = () => { changeStream.on('change', async (change) => { setTimeout(() => { try { - getSystemPlugins(true); + getSystemPluginCb(true); } catch (error) {} }, 5000); }); diff --git a/projects/app/src/service/core/app/plugin.ts b/projects/app/src/service/core/app/plugin.ts index 95b757940..e84ce1ee6 100644 --- a/projects/app/src/service/core/app/plugin.ts +++ b/projects/app/src/service/core/app/plugin.ts @@ -11,7 +11,8 @@ const getCommercialPlugins = () => { return GET('/core/app/plugin/getSystemPlugins'); }; export const getSystemPlugins = async (refresh = false) => { - if (isProduction && global.systemPlugins && !refresh) return cloneDeep(global.systemPlugins); + if (isProduction && global.systemPlugins && global.systemPlugins.length > 0 && !refresh) + return cloneDeep(global.systemPlugins); try { if (!global.systemPlugins) { @@ -22,8 +23,6 @@ export const getSystemPlugins = async (refresh = false) => { addLog.info(`Load system plugin successfully: ${global.systemPlugins.length}`); - getSystemPluginCb(); - return cloneDeep(global.systemPlugins); } catch (error) { //@ts-ignore @@ -56,16 +55,22 @@ const getCommercialCb = async () => { {} ); }; -export const getSystemPluginCb = async () => { - if (isProduction && global.systemPluginCb) return global.systemPluginCb; + +export const getSystemPluginCb = async (refresh = false) => { + if ( + isProduction && + global.systemPluginCb && + Object.keys(global.systemPluginCb).length > 0 && + !refresh + ) + return global.systemPluginCb; try { global.systemPluginCb = {}; + await getSystemPlugins(); global.systemPluginCb = FastGPTProUrl ? await getCommercialCb() : await getCommunityCb(); return global.systemPluginCb; } catch (error) { - //@ts-ignore - global.systemPluginCb = undefined; return Promise.reject(error); } }; diff --git a/projects/app/src/service/mongo.ts b/projects/app/src/service/mongo.ts index ec79e07d8..bd2d99551 100644 --- a/projects/app/src/service/mongo.ts +++ b/projects/app/src/service/mongo.ts @@ -12,7 +12,7 @@ import { startMongoWatch } from './common/system/volumnMongoWatch'; import { startTrainingQueue } from './core/dataset/training/utils'; import { systemStartCb } from '@fastgpt/service/common/system/tools'; import { addLog } from '@fastgpt/service/common/system/log'; -import { getSystemPlugins } from './core/app/plugin'; +import { getSystemPluginCb } from './core/app/plugin'; /** * This function is equivalent to the entry to the service @@ -34,7 +34,7 @@ export function connectToDatabase() { //init system configļ¼›init vector databaseļ¼›init root user await Promise.all([getInitConfig(), initVectorStore(), initRootUser()]); - getSystemPlugins(); + getSystemPluginCb(); startMongoWatch(); // cron startCron();