mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-22 20:37:48 +00:00
4.8.14 test (#3164)
* perf: match base 64 image * perf: register plugins
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { batchRun } from '../fn/utils';
|
||||
import { simpleText } from './tools';
|
||||
import { getNanoid, simpleText } from './tools';
|
||||
import type { ImageType } from '../../../service/worker/readFile/type';
|
||||
|
||||
/* Delete redundant text in markdown */
|
||||
export const simpleMarkdownText = (rawText: string) => {
|
||||
@@ -92,3 +93,25 @@ export const markdownProcess = async ({
|
||||
|
||||
return simpleMarkdownText(imageProcess);
|
||||
};
|
||||
|
||||
export const matchMdImgTextAndUpload = (text: string) => {
|
||||
const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g;
|
||||
const imageList: ImageType[] = [];
|
||||
const images = Array.from(text.match(base64Regex) || []);
|
||||
for (const image of images) {
|
||||
const uuid = `IMAGE_${getNanoid(12)}_IMAGE`;
|
||||
const mime = image.split(';')[0].split(':')[1];
|
||||
const base64 = image.split(',')[1];
|
||||
text = text.replace(image, uuid);
|
||||
imageList.push({
|
||||
uuid,
|
||||
base64,
|
||||
mime
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
text,
|
||||
imageList
|
||||
};
|
||||
};
|
||||
|
@@ -46,6 +46,8 @@ export const getCommunityPlugins = () => {
|
||||
};
|
||||
|
||||
export const getSystemPluginTemplates = () => {
|
||||
if (!global.systemPlugins) return [];
|
||||
|
||||
const oldPlugins = global.communityPlugins ?? [];
|
||||
return [...oldPlugins, ...cloneDeep(global.systemPlugins)];
|
||||
};
|
||||
|
@@ -4,12 +4,12 @@ import FormData from 'form-data';
|
||||
|
||||
import { WorkerNameEnum, runWorker } from '../../../worker/utils';
|
||||
import fs from 'fs';
|
||||
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
|
||||
import type { ReadFileResponse } from '../../../worker/readFile/type';
|
||||
import axios from 'axios';
|
||||
import { addLog } from '../../system/log';
|
||||
import { batchRun } from '@fastgpt/global/common/fn/utils';
|
||||
import { addHours } from 'date-fns';
|
||||
import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
|
||||
|
||||
export type readRawTextByLocalFileParams = {
|
||||
teamId: string;
|
||||
@@ -79,6 +79,7 @@ export const readRawContentByFileBuffer = async ({
|
||||
data: {
|
||||
page: number;
|
||||
markdown: string;
|
||||
duration: number;
|
||||
};
|
||||
}>(customReadfileUrl, data, {
|
||||
timeout: 600000,
|
||||
@@ -90,10 +91,12 @@ export const readRawContentByFileBuffer = async ({
|
||||
addLog.info(`Use custom read file service, time: ${Date.now() - start}ms`);
|
||||
|
||||
const rawText = response.data.markdown;
|
||||
const { text, imageList } = matchMdImgTextAndUpload(rawText);
|
||||
|
||||
return {
|
||||
rawText,
|
||||
formatText: rawText
|
||||
rawText: text,
|
||||
formatText: rawText,
|
||||
imageList
|
||||
};
|
||||
};
|
||||
|
||||
@@ -120,6 +123,9 @@ export const readRawContentByFileBuffer = async ({
|
||||
}
|
||||
});
|
||||
rawText = rawText.replace(item.uuid, src);
|
||||
if (formatText) {
|
||||
formatText = formatText.replace(item.uuid, src);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -128,7 +134,7 @@ export const readRawContentByFileBuffer = async ({
|
||||
if (isQAImport) {
|
||||
rawText = rawText || '';
|
||||
} else {
|
||||
rawText = formatText || '';
|
||||
rawText = formatText || rawText;
|
||||
}
|
||||
}
|
||||
|
||||
|
@@ -1,5 +1,6 @@
|
||||
import TurndownService from 'turndown';
|
||||
import { ImageType } from '../readFile/type';
|
||||
import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
|
||||
// @ts-ignore
|
||||
const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
|
||||
|
||||
@@ -24,23 +25,10 @@ export const html2md = (
|
||||
turndownService.remove(['i', 'script', 'iframe', 'style']);
|
||||
turndownService.use(turndownPluginGfm.gfm);
|
||||
|
||||
const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g;
|
||||
const imageList: ImageType[] = [];
|
||||
const images = Array.from(html.match(base64Regex) || []);
|
||||
for (const image of images) {
|
||||
const uuid = crypto.randomUUID();
|
||||
const mime = image.split(';')[0].split(':')[1];
|
||||
const base64 = image.split(',')[1];
|
||||
html = html.replace(image, uuid);
|
||||
imageList.push({
|
||||
uuid,
|
||||
base64,
|
||||
mime
|
||||
});
|
||||
}
|
||||
const { text, imageList } = matchMdImgTextAndUpload(html);
|
||||
|
||||
return {
|
||||
rawText: turndownService.turndown(html),
|
||||
rawText: turndownService.turndown(text),
|
||||
imageList
|
||||
};
|
||||
} catch (error) {
|
||||
|
@@ -1,12 +1,12 @@
|
||||
import React, { useCallback } from 'react';
|
||||
import MyModal from '@fastgpt/web/components/common/MyModal';
|
||||
import { useUserStore } from '@/web/support/user/useUserStore';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { LOGO_ICON } from '@fastgpt/global/common/system/constants';
|
||||
import { getSystemMsgModalData } from '@/web/support/user/inform/api';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
const Markdown = dynamic(() => import('@/components/Markdown'), { ssr: false });
|
||||
|
||||
const SystemMsgModal = ({}: {}) => {
|
||||
@@ -15,7 +15,9 @@ const SystemMsgModal = ({}: {}) => {
|
||||
|
||||
const { isOpen, onOpen, onClose } = useDisclosure();
|
||||
|
||||
const { data } = useQuery(['initSystemMsgModal', systemMsgReadId], getSystemMsgModalData, {
|
||||
const { data } = useRequest2(getSystemMsgModalData, {
|
||||
refreshDeps: [systemMsgReadId],
|
||||
manual: false,
|
||||
onSuccess(res) {
|
||||
if (res?.content && (!systemMsgReadId || res.id !== systemMsgReadId)) {
|
||||
onOpen();
|
||||
|
@@ -6,6 +6,7 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { WorkflowContext } from '../../../../context';
|
||||
import { WorkflowNodeEdgeContext } from '../../../../context/workflowInitContext';
|
||||
import { FlowNodeItemType } from '@fastgpt/global/core/workflow/type/node';
|
||||
|
||||
export const ConnectionSourceHandle = ({
|
||||
nodeId,
|
||||
@@ -141,15 +142,23 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
|
||||
const { connectingEdge, nodeList } = useContextSelector(WorkflowContext, (ctx) => ctx);
|
||||
|
||||
const { LeftHandle, rightHandle, topHandle, bottomHandle } = useMemo(() => {
|
||||
const node = nodeList.find((node) => node.nodeId === nodeId);
|
||||
const connectingNode = nodeList.find((node) => node.nodeId === connectingEdge?.nodeId);
|
||||
let node: FlowNodeItemType | undefined = undefined,
|
||||
connectingNode: FlowNodeItemType | undefined = undefined;
|
||||
for (const item of nodeList) {
|
||||
if (item.nodeId === nodeId) {
|
||||
node = item;
|
||||
}
|
||||
if (item.nodeId === connectingEdge?.nodeId) {
|
||||
connectingNode = item;
|
||||
}
|
||||
if (node && (connectingNode || !connectingEdge?.nodeId)) break;
|
||||
}
|
||||
|
||||
const connectingNodeSourceNodeIdMap = new Map<string, number>();
|
||||
let forbidConnect = false;
|
||||
edges.forEach((edge) => {
|
||||
if (edge.target === connectingNode?.nodeId) {
|
||||
connectingNodeSourceNodeIdMap.set(edge.source, 1);
|
||||
} else if (edge.target === nodeId) {
|
||||
for (const edge of edges) {
|
||||
if (forbidConnect) break;
|
||||
|
||||
if (edge.target === nodeId) {
|
||||
// Node has be connected tool, it cannot be connect by other handle
|
||||
if (edge.targetHandle === NodeOutputKeyEnum.selectedTools) {
|
||||
forbidConnect = true;
|
||||
@@ -163,7 +172,7 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
|
||||
forbidConnect = true;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const showHandle = (() => {
|
||||
if (forbidConnect) return false;
|
||||
@@ -178,9 +187,6 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
|
||||
// Not the same parent node
|
||||
if (connectingNode && connectingNode?.parentNodeId !== node?.parentNodeId) return false;
|
||||
|
||||
// Unable to connect to the source node
|
||||
if (connectingNodeSourceNodeIdMap.has(nodeId)) return false;
|
||||
|
||||
return true;
|
||||
})();
|
||||
|
||||
|
@@ -1,4 +1,4 @@
|
||||
import { getSystemPluginCb, getSystemPlugins } from '@/service/core/app/plugin';
|
||||
import { getSystemPluginCb } from '@/service/core/app/plugin';
|
||||
import { initSystemConfig } from '.';
|
||||
import { createDatasetTrainingMongoWatch } from '@/service/core/dataset/training/utils';
|
||||
import { MongoSystemConfigs } from '@fastgpt/service/common/system/config/schema';
|
||||
@@ -29,7 +29,7 @@ const refetchSystemPlugins = () => {
|
||||
changeStream.on('change', async (change) => {
|
||||
setTimeout(() => {
|
||||
try {
|
||||
getSystemPlugins(true);
|
||||
getSystemPluginCb(true);
|
||||
} catch (error) {}
|
||||
}, 5000);
|
||||
});
|
||||
|
@@ -11,7 +11,8 @@ const getCommercialPlugins = () => {
|
||||
return GET<SystemPluginTemplateItemType[]>('/core/app/plugin/getSystemPlugins');
|
||||
};
|
||||
export const getSystemPlugins = async (refresh = false) => {
|
||||
if (isProduction && global.systemPlugins && !refresh) return cloneDeep(global.systemPlugins);
|
||||
if (isProduction && global.systemPlugins && global.systemPlugins.length > 0 && !refresh)
|
||||
return cloneDeep(global.systemPlugins);
|
||||
|
||||
try {
|
||||
if (!global.systemPlugins) {
|
||||
@@ -22,8 +23,6 @@ export const getSystemPlugins = async (refresh = false) => {
|
||||
|
||||
addLog.info(`Load system plugin successfully: ${global.systemPlugins.length}`);
|
||||
|
||||
getSystemPluginCb();
|
||||
|
||||
return cloneDeep(global.systemPlugins);
|
||||
} catch (error) {
|
||||
//@ts-ignore
|
||||
@@ -56,16 +55,22 @@ const getCommercialCb = async () => {
|
||||
{}
|
||||
);
|
||||
};
|
||||
export const getSystemPluginCb = async () => {
|
||||
if (isProduction && global.systemPluginCb) return global.systemPluginCb;
|
||||
|
||||
export const getSystemPluginCb = async (refresh = false) => {
|
||||
if (
|
||||
isProduction &&
|
||||
global.systemPluginCb &&
|
||||
Object.keys(global.systemPluginCb).length > 0 &&
|
||||
!refresh
|
||||
)
|
||||
return global.systemPluginCb;
|
||||
|
||||
try {
|
||||
global.systemPluginCb = {};
|
||||
await getSystemPlugins();
|
||||
global.systemPluginCb = FastGPTProUrl ? await getCommercialCb() : await getCommunityCb();
|
||||
return global.systemPluginCb;
|
||||
} catch (error) {
|
||||
//@ts-ignore
|
||||
global.systemPluginCb = undefined;
|
||||
return Promise.reject(error);
|
||||
}
|
||||
};
|
||||
|
@@ -12,7 +12,7 @@ import { startMongoWatch } from './common/system/volumnMongoWatch';
|
||||
import { startTrainingQueue } from './core/dataset/training/utils';
|
||||
import { systemStartCb } from '@fastgpt/service/common/system/tools';
|
||||
import { addLog } from '@fastgpt/service/common/system/log';
|
||||
import { getSystemPlugins } from './core/app/plugin';
|
||||
import { getSystemPluginCb } from './core/app/plugin';
|
||||
|
||||
/**
|
||||
* This function is equivalent to the entry to the service
|
||||
@@ -34,7 +34,7 @@ export function connectToDatabase() {
|
||||
//init system config;init vector database;init root user
|
||||
await Promise.all([getInitConfig(), initVectorStore(), initRootUser()]);
|
||||
|
||||
getSystemPlugins();
|
||||
getSystemPluginCb();
|
||||
startMongoWatch();
|
||||
// cron
|
||||
startCron();
|
||||
|
Reference in New Issue
Block a user