4.8.14 test (#3164)

* perf: match base 64 image

* perf: register plugins
This commit is contained in:
Archer
2024-11-15 10:35:04 +08:00
committed by GitHub
parent f699061dea
commit 58745f8c35
9 changed files with 76 additions and 44 deletions

View File

@@ -1,5 +1,6 @@
import { batchRun } from '../fn/utils';
import { simpleText } from './tools';
import { getNanoid, simpleText } from './tools';
import type { ImageType } from '../../../service/worker/readFile/type';
/* Delete redundant text in markdown */
export const simpleMarkdownText = (rawText: string) => {
@@ -92,3 +93,25 @@ export const markdownProcess = async ({
return simpleMarkdownText(imageProcess);
};
export const matchMdImgTextAndUpload = (text: string) => {
const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g;
const imageList: ImageType[] = [];
const images = Array.from(text.match(base64Regex) || []);
for (const image of images) {
const uuid = `IMAGE_${getNanoid(12)}_IMAGE`;
const mime = image.split(';')[0].split(':')[1];
const base64 = image.split(',')[1];
text = text.replace(image, uuid);
imageList.push({
uuid,
base64,
mime
});
}
return {
text,
imageList
};
};

View File

@@ -46,6 +46,8 @@ export const getCommunityPlugins = () => {
};
export const getSystemPluginTemplates = () => {
if (!global.systemPlugins) return [];
const oldPlugins = global.communityPlugins ?? [];
return [...oldPlugins, ...cloneDeep(global.systemPlugins)];
};

View File

@@ -4,12 +4,12 @@ import FormData from 'form-data';
import { WorkerNameEnum, runWorker } from '../../../worker/utils';
import fs from 'fs';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import type { ReadFileResponse } from '../../../worker/readFile/type';
import axios from 'axios';
import { addLog } from '../../system/log';
import { batchRun } from '@fastgpt/global/common/fn/utils';
import { addHours } from 'date-fns';
import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
export type readRawTextByLocalFileParams = {
teamId: string;
@@ -79,6 +79,7 @@ export const readRawContentByFileBuffer = async ({
data: {
page: number;
markdown: string;
duration: number;
};
}>(customReadfileUrl, data, {
timeout: 600000,
@@ -90,10 +91,12 @@ export const readRawContentByFileBuffer = async ({
addLog.info(`Use custom read file service, time: ${Date.now() - start}ms`);
const rawText = response.data.markdown;
const { text, imageList } = matchMdImgTextAndUpload(rawText);
return {
rawText,
formatText: rawText
rawText: text,
formatText: rawText,
imageList
};
};
@@ -120,6 +123,9 @@ export const readRawContentByFileBuffer = async ({
}
});
rawText = rawText.replace(item.uuid, src);
if (formatText) {
formatText = formatText.replace(item.uuid, src);
}
});
}
@@ -128,7 +134,7 @@ export const readRawContentByFileBuffer = async ({
if (isQAImport) {
rawText = rawText || '';
} else {
rawText = formatText || '';
rawText = formatText || rawText;
}
}

View File

@@ -1,5 +1,6 @@
import TurndownService from 'turndown';
import { ImageType } from '../readFile/type';
import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
// @ts-ignore
const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
@@ -24,23 +25,10 @@ export const html2md = (
turndownService.remove(['i', 'script', 'iframe', 'style']);
turndownService.use(turndownPluginGfm.gfm);
const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g;
const imageList: ImageType[] = [];
const images = Array.from(html.match(base64Regex) || []);
for (const image of images) {
const uuid = crypto.randomUUID();
const mime = image.split(';')[0].split(':')[1];
const base64 = image.split(',')[1];
html = html.replace(image, uuid);
imageList.push({
uuid,
base64,
mime
});
}
const { text, imageList } = matchMdImgTextAndUpload(html);
return {
rawText: turndownService.turndown(html),
rawText: turndownService.turndown(text),
imageList
};
} catch (error) {

View File

@@ -1,12 +1,12 @@
import React, { useCallback } from 'react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useQuery } from '@tanstack/react-query';
import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { LOGO_ICON } from '@fastgpt/global/common/system/constants';
import { getSystemMsgModalData } from '@/web/support/user/inform/api';
import dynamic from 'next/dynamic';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
const Markdown = dynamic(() => import('@/components/Markdown'), { ssr: false });
const SystemMsgModal = ({}: {}) => {
@@ -15,7 +15,9 @@ const SystemMsgModal = ({}: {}) => {
const { isOpen, onOpen, onClose } = useDisclosure();
const { data } = useQuery(['initSystemMsgModal', systemMsgReadId], getSystemMsgModalData, {
const { data } = useRequest2(getSystemMsgModalData, {
refreshDeps: [systemMsgReadId],
manual: false,
onSuccess(res) {
if (res?.content && (!systemMsgReadId || res.id !== systemMsgReadId)) {
onOpen();

View File

@@ -6,6 +6,7 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '../../../../context';
import { WorkflowNodeEdgeContext } from '../../../../context/workflowInitContext';
import { FlowNodeItemType } from '@fastgpt/global/core/workflow/type/node';
export const ConnectionSourceHandle = ({
nodeId,
@@ -141,15 +142,23 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
const { connectingEdge, nodeList } = useContextSelector(WorkflowContext, (ctx) => ctx);
const { LeftHandle, rightHandle, topHandle, bottomHandle } = useMemo(() => {
const node = nodeList.find((node) => node.nodeId === nodeId);
const connectingNode = nodeList.find((node) => node.nodeId === connectingEdge?.nodeId);
let node: FlowNodeItemType | undefined = undefined,
connectingNode: FlowNodeItemType | undefined = undefined;
for (const item of nodeList) {
if (item.nodeId === nodeId) {
node = item;
}
if (item.nodeId === connectingEdge?.nodeId) {
connectingNode = item;
}
if (node && (connectingNode || !connectingEdge?.nodeId)) break;
}
const connectingNodeSourceNodeIdMap = new Map<string, number>();
let forbidConnect = false;
edges.forEach((edge) => {
if (edge.target === connectingNode?.nodeId) {
connectingNodeSourceNodeIdMap.set(edge.source, 1);
} else if (edge.target === nodeId) {
for (const edge of edges) {
if (forbidConnect) break;
if (edge.target === nodeId) {
// Node has be connected tool, it cannot be connect by other handle
if (edge.targetHandle === NodeOutputKeyEnum.selectedTools) {
forbidConnect = true;
@@ -163,7 +172,7 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
forbidConnect = true;
}
}
});
}
const showHandle = (() => {
if (forbidConnect) return false;
@@ -178,9 +187,6 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
// Not the same parent node
if (connectingNode && connectingNode?.parentNodeId !== node?.parentNodeId) return false;
// Unable to connect to the source node
if (connectingNodeSourceNodeIdMap.has(nodeId)) return false;
return true;
})();

View File

@@ -1,4 +1,4 @@
import { getSystemPluginCb, getSystemPlugins } from '@/service/core/app/plugin';
import { getSystemPluginCb } from '@/service/core/app/plugin';
import { initSystemConfig } from '.';
import { createDatasetTrainingMongoWatch } from '@/service/core/dataset/training/utils';
import { MongoSystemConfigs } from '@fastgpt/service/common/system/config/schema';
@@ -29,7 +29,7 @@ const refetchSystemPlugins = () => {
changeStream.on('change', async (change) => {
setTimeout(() => {
try {
getSystemPlugins(true);
getSystemPluginCb(true);
} catch (error) {}
}, 5000);
});

View File

@@ -11,7 +11,8 @@ const getCommercialPlugins = () => {
return GET<SystemPluginTemplateItemType[]>('/core/app/plugin/getSystemPlugins');
};
export const getSystemPlugins = async (refresh = false) => {
if (isProduction && global.systemPlugins && !refresh) return cloneDeep(global.systemPlugins);
if (isProduction && global.systemPlugins && global.systemPlugins.length > 0 && !refresh)
return cloneDeep(global.systemPlugins);
try {
if (!global.systemPlugins) {
@@ -22,8 +23,6 @@ export const getSystemPlugins = async (refresh = false) => {
addLog.info(`Load system plugin successfully: ${global.systemPlugins.length}`);
getSystemPluginCb();
return cloneDeep(global.systemPlugins);
} catch (error) {
//@ts-ignore
@@ -56,16 +55,22 @@ const getCommercialCb = async () => {
{}
);
};
export const getSystemPluginCb = async () => {
if (isProduction && global.systemPluginCb) return global.systemPluginCb;
export const getSystemPluginCb = async (refresh = false) => {
if (
isProduction &&
global.systemPluginCb &&
Object.keys(global.systemPluginCb).length > 0 &&
!refresh
)
return global.systemPluginCb;
try {
global.systemPluginCb = {};
await getSystemPlugins();
global.systemPluginCb = FastGPTProUrl ? await getCommercialCb() : await getCommunityCb();
return global.systemPluginCb;
} catch (error) {
//@ts-ignore
global.systemPluginCb = undefined;
return Promise.reject(error);
}
};

View File

@@ -12,7 +12,7 @@ import { startMongoWatch } from './common/system/volumnMongoWatch';
import { startTrainingQueue } from './core/dataset/training/utils';
import { systemStartCb } from '@fastgpt/service/common/system/tools';
import { addLog } from '@fastgpt/service/common/system/log';
import { getSystemPlugins } from './core/app/plugin';
import { getSystemPluginCb } from './core/app/plugin';
/**
* This function is equivalent to the entry to the service
@@ -34,7 +34,7 @@ export function connectToDatabase() {
//init system configinit vector databaseinit root user
await Promise.all([getInitConfig(), initVectorStore(), initRootUser()]);
getSystemPlugins();
getSystemPluginCb();
startMongoWatch();
// cron
startCron();