mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-24 13:53:50 +00:00
perf: forbid image to base64 (#3038)
* perf: forbid image to base64 * update file upload path * feat: support promptCall use image * fix: echarts load * update doc
This commit is contained in:
@@ -1,7 +1,5 @@
|
||||
import * as echarts from 'echarts';
|
||||
import json5 from 'json5';
|
||||
import { getFileSavePath } from '../../../utils';
|
||||
import * as fs from 'fs';
|
||||
import { SystemPluginSpecialResponse } from '../../../type.d';
|
||||
|
||||
type Props = {
|
||||
@@ -82,25 +80,23 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT
|
||||
|
||||
chart.setOption(option);
|
||||
// 生成 Base64 图像
|
||||
const base64Image = chart.getDataURL();
|
||||
const svgData = decodeURIComponent(base64Image.split(',')[1]);
|
||||
const base64Image = chart.getDataURL({
|
||||
type: 'png',
|
||||
pixelRatio: 2 // 可以设置更高的像素比以获得更清晰的图像
|
||||
});
|
||||
const svgContent = decodeURIComponent(base64Image.split(',')[1]);
|
||||
const base64 = `data:image/svg+xml;base64,${Buffer.from(svgContent).toString('base64')}`;
|
||||
|
||||
const fileName = `chart_${Date.now()}.svg`;
|
||||
const filePath = getFileSavePath(fileName);
|
||||
fs.writeFileSync(filePath, svgData);
|
||||
// 释放图表实例
|
||||
chart.dispose();
|
||||
|
||||
return filePath;
|
||||
return base64;
|
||||
};
|
||||
|
||||
const main = async ({ title, xAxis, yAxis, chartType }: Props): Response => {
|
||||
const filePath = await generateChart(title, xAxis, yAxis, chartType);
|
||||
const base64 = await generateChart(title, xAxis, yAxis, chartType);
|
||||
return {
|
||||
result: {
|
||||
type: 'SYSTEM_PLUGIN_FILE',
|
||||
path: filePath,
|
||||
contentType: 'image/svg+xml'
|
||||
type: 'SYSTEM_PLUGIN_BASE64',
|
||||
value: base64,
|
||||
extension: 'svg'
|
||||
}
|
||||
};
|
||||
};
|
||||
|
6
packages/plugins/type.d.ts
vendored
6
packages/plugins/type.d.ts
vendored
@@ -4,9 +4,9 @@ import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type
|
||||
|
||||
export type SystemPluginResponseType = Promise<Record<string, any>>;
|
||||
export type SystemPluginSpecialResponse = {
|
||||
type: 'SYSTEM_PLUGIN_FILE';
|
||||
path: string;
|
||||
contentType: string;
|
||||
type: 'SYSTEM_PLUGIN_BASE64';
|
||||
value: string;
|
||||
extension: string;
|
||||
};
|
||||
|
||||
declare global {
|
||||
|
@@ -1,15 +0,0 @@
|
||||
import path from 'path';
|
||||
import * as fs from 'fs';
|
||||
|
||||
const isProduction = process.env.NODE_ENV === 'production';
|
||||
|
||||
export const getFileSavePath = (name: string) => {
|
||||
if (isProduction) {
|
||||
return `/app/plugin_file/${name}`;
|
||||
}
|
||||
const filePath = path.join(process.cwd(), 'local', 'plugin_file', name);
|
||||
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
|
||||
return filePath;
|
||||
};
|
@@ -12,6 +12,7 @@ import { gridFsStream2Buffer, stream2Encoding } from './utils';
|
||||
import { addLog } from '../../system/log';
|
||||
import { readFromSecondary } from '../../mongo/utils';
|
||||
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export function getGFSCollection(bucket: `${BucketNameEnum}`) {
|
||||
MongoDatasetFileSchema;
|
||||
@@ -76,6 +77,59 @@ export async function uploadFile({
|
||||
|
||||
return String(stream.id);
|
||||
}
|
||||
export async function uploadFileFromBase64Img({
|
||||
bucketName,
|
||||
teamId,
|
||||
tmbId,
|
||||
base64,
|
||||
filename,
|
||||
metadata = {}
|
||||
}: {
|
||||
bucketName: `${BucketNameEnum}`;
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
base64: string;
|
||||
filename: string;
|
||||
metadata?: Record<string, any>;
|
||||
}) {
|
||||
if (!base64) return Promise.reject(`filePath is empty`);
|
||||
if (!filename) return Promise.reject(`filename is empty`);
|
||||
|
||||
const base64Data = base64.split(',')[1];
|
||||
const contentType = base64.split(',')?.[0]?.split?.(':')?.[1];
|
||||
const buffer = Buffer.from(base64Data, 'base64');
|
||||
const readableStream = new Readable({
|
||||
read() {
|
||||
this.push(buffer);
|
||||
this.push(null);
|
||||
}
|
||||
});
|
||||
|
||||
const { stream: readStream, encoding } = await stream2Encoding(readableStream);
|
||||
|
||||
// Add default metadata
|
||||
metadata.teamId = teamId;
|
||||
metadata.tmbId = tmbId;
|
||||
metadata.encoding = encoding;
|
||||
|
||||
// create a gridfs bucket
|
||||
const bucket = getGridBucket(bucketName);
|
||||
|
||||
const stream = bucket.openUploadStream(filename, {
|
||||
metadata,
|
||||
contentType
|
||||
});
|
||||
|
||||
// save to gridfs
|
||||
await new Promise((resolve, reject) => {
|
||||
readStream
|
||||
.pipe(stream as any)
|
||||
.on('finish', resolve)
|
||||
.on('error', reject);
|
||||
});
|
||||
|
||||
return String(stream.id);
|
||||
}
|
||||
|
||||
export async function getFileById({
|
||||
bucketName,
|
||||
|
@@ -104,6 +104,9 @@ export const loadRequestMessages = async ({
|
||||
}) => {
|
||||
// Load image to base64
|
||||
const loadImageToBase64 = async (messages: ChatCompletionContentPart[]) => {
|
||||
if (process.env.MULTIPLE_DATA_TO_BASE64 === 'false') {
|
||||
return messages;
|
||||
}
|
||||
return Promise.all(
|
||||
messages.map(async (item) => {
|
||||
if (item.type === 'image_url') {
|
||||
|
@@ -185,7 +185,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
|
||||
// array, replace last element
|
||||
const lastText = lastMessage.content[lastMessage.content.length - 1];
|
||||
if (lastText.type === 'text') {
|
||||
lastMessage.content = replaceVariable(Prompt_Tool_Call, {
|
||||
lastText.text = replaceVariable(Prompt_Tool_Call, {
|
||||
question: lastText.text
|
||||
});
|
||||
} else {
|
||||
|
@@ -176,17 +176,29 @@ export const runToolWithPromptCall = async (
|
||||
);
|
||||
|
||||
const lastMessage = messages[messages.length - 1];
|
||||
if (typeof lastMessage.content !== 'string') {
|
||||
if (typeof lastMessage.content === 'string') {
|
||||
lastMessage.content = replaceVariable(lastMessage.content, {
|
||||
toolsPrompt
|
||||
});
|
||||
} else if (Array.isArray(lastMessage.content)) {
|
||||
// array, replace last element
|
||||
const lastText = lastMessage.content[lastMessage.content.length - 1];
|
||||
if (lastText.type === 'text') {
|
||||
lastText.text = replaceVariable(lastText.text, {
|
||||
toolsPrompt
|
||||
});
|
||||
} else {
|
||||
return Promise.reject('Prompt call invalid input');
|
||||
}
|
||||
} else {
|
||||
return Promise.reject('Prompt call invalid input');
|
||||
}
|
||||
lastMessage.content = replaceVariable(lastMessage.content, {
|
||||
toolsPrompt
|
||||
});
|
||||
|
||||
const filterMessages = await filterGPTMessageByMaxTokens({
|
||||
messages,
|
||||
maxTokens: toolModel.maxContext - 500 // filter token. not response maxToken
|
||||
});
|
||||
|
||||
const [requestMessages, max_tokens] = await Promise.all([
|
||||
loadRequestMessages({
|
||||
messages: filterMessages,
|
||||
@@ -398,11 +410,27 @@ export const runToolWithPromptCall = async (
|
||||
: undefined;
|
||||
|
||||
// get the next user prompt
|
||||
lastMessage.content += `${replaceAnswer}
|
||||
if (typeof lastMessage.content === 'string') {
|
||||
lastMessage.content += `${replaceAnswer}
|
||||
TOOL_RESPONSE: """
|
||||
${workflowInteractiveResponseItem ? `{{${INTERACTIVE_STOP_SIGNAL}}}` : toolsRunResponse.toolResponsePrompt}
|
||||
"""
|
||||
ANSWER: `;
|
||||
} else if (Array.isArray(lastMessage.content)) {
|
||||
// array, replace last element
|
||||
const lastText = lastMessage.content[lastMessage.content.length - 1];
|
||||
if (lastText.type === 'text') {
|
||||
lastText.text += `${replaceAnswer}
|
||||
TOOL_RESPONSE: """
|
||||
${workflowInteractiveResponseItem ? `{{${INTERACTIVE_STOP_SIGNAL}}}` : toolsRunResponse.toolResponsePrompt}
|
||||
"""
|
||||
ANSWER: `;
|
||||
} else {
|
||||
return Promise.reject('Prompt call invalid input');
|
||||
}
|
||||
} else {
|
||||
return Promise.reject('Prompt call invalid input');
|
||||
}
|
||||
|
||||
const runTimes = (response?.runTimes || 0) + toolsRunResponse.toolResponse.runTimes;
|
||||
const toolNodeTokens = response?.toolNodeTokens ? response.toolNodeTokens + tokens : tokens;
|
||||
|
@@ -18,11 +18,11 @@ import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/util
|
||||
import { getSystemPluginCb } from '../../../../../plugins/register';
|
||||
import { ContentTypes } from '@fastgpt/global/core/workflow/constants';
|
||||
import { replaceEditorVariable } from '@fastgpt/global/core/workflow/utils';
|
||||
import { uploadFile } from '../../../../common/file/gridfs/controller';
|
||||
import { uploadFileFromBase64Img } from '../../../../common/file/gridfs/controller';
|
||||
import { ReadFileBaseUrl } from '@fastgpt/global/common/file/constants';
|
||||
import { createFileToken } from '../../../../support/permission/controller';
|
||||
import { removeFilesByPaths } from '../../../../common/file/utils';
|
||||
import { JSONPath } from 'jsonpath-plus';
|
||||
import type { SystemPluginSpecialResponse } from '../../../../../plugins/type';
|
||||
|
||||
type PropsArrType = {
|
||||
key: string;
|
||||
@@ -376,27 +376,25 @@ async function replaceSystemPluginResponse({
|
||||
tmbId: string;
|
||||
}) {
|
||||
for await (const key of Object.keys(response)) {
|
||||
if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_FILE') {
|
||||
const fileObj = response[key];
|
||||
const filename = fileObj.path.split('/').pop() || `${tmbId}-${Date.now()}`;
|
||||
if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_BASE64') {
|
||||
const fileObj = response[key] as SystemPluginSpecialResponse;
|
||||
const filename = `${tmbId}-${Date.now()}.${fileObj.extension}`;
|
||||
try {
|
||||
const fileId = await uploadFile({
|
||||
const fileId = await uploadFileFromBase64Img({
|
||||
teamId,
|
||||
tmbId,
|
||||
bucketName: 'chat',
|
||||
path: fileObj.path,
|
||||
base64: fileObj.value,
|
||||
filename,
|
||||
contentType: fileObj.contentType,
|
||||
metadata: {}
|
||||
});
|
||||
response[key] = `${ReadFileBaseUrl}?filename=${filename}&token=${await createFileToken({
|
||||
response[key] = `${ReadFileBaseUrl}/${filename}?token=${await createFileToken({
|
||||
bucketName: 'chat',
|
||||
teamId,
|
||||
tmbId,
|
||||
fileId
|
||||
})}`;
|
||||
} catch (error) {}
|
||||
removeFilesByPaths([fileObj.path]);
|
||||
}
|
||||
}
|
||||
return response;
|
||||
|
Reference in New Issue
Block a user