From 912b264a472c7a45a1c91fbed343bec7381aab31 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 1 Nov 2024 14:29:20 +0800 Subject: [PATCH] perf: forbid image to base64 (#3038) * perf: forbid image to base64 * update file upload path * feat: support promptCall use image * fix: echarts load * update doc --- .../zh-cn/docs/development/upgrading/4813.md | 14 +++++ .../plugins/src/drawing/baseChart/index.ts | 26 ++++---- packages/plugins/type.d.ts | 6 +- packages/plugins/utils.ts | 15 ----- .../service/common/file/gridfs/controller.ts | 54 ++++++++++++++++ packages/service/core/chat/utils.ts | 3 + .../workflow/dispatch/agent/runTool/index.ts | 2 +- .../dispatch/agent/runTool/promptCall.ts | 38 +++++++++-- .../core/workflow/dispatch/tools/http468.ts | 18 +++--- projects/app/Dockerfile | 2 +- .../pages/api/common/file/read/[filename].ts | 63 +++++++++++++++++++ .../app/src/pages/api/common/file/upload.ts | 14 ++--- 12 files changed, 197 insertions(+), 58 deletions(-) create mode 100644 docSite/content/zh-cn/docs/development/upgrading/4813.md delete mode 100644 packages/plugins/utils.ts create mode 100644 projects/app/src/pages/api/common/file/read/[filename].ts diff --git a/docSite/content/zh-cn/docs/development/upgrading/4813.md b/docSite/content/zh-cn/docs/development/upgrading/4813.md new file mode 100644 index 000000000..f68084014 --- /dev/null +++ b/docSite/content/zh-cn/docs/development/upgrading/4813.md @@ -0,0 +1,14 @@ +--- +title: 'V4.8.13(进行中)' +description: 'FastGPT V4.8.13 更新说明' +icon: 'upgrade' +draft: false +toc: true +weight: 811 +--- + +## 更新说明 + +1. +2. 优化 - 知识库上传文件，优化报错提示 +3. 修复 - BI 图表生成无法写入文件 diff --git a/packages/plugins/src/drawing/baseChart/index.ts b/packages/plugins/src/drawing/baseChart/index.ts index 06d62ac6a..f3e34b7ac 100644 --- a/packages/plugins/src/drawing/baseChart/index.ts +++ b/packages/plugins/src/drawing/baseChart/index.ts @@ -1,7 +1,5 @@ import * as echarts from 'echarts'; import json5 from 'json5'; -import { getFileSavePath } from '../../../utils'; -import * as fs from 'fs'; import { SystemPluginSpecialResponse } from '../../../type.d'; type Props = { @@ -82,25 +80,23 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT chart.setOption(option); // 生成 Base64 图像 - const base64Image = chart.getDataURL(); - const svgData = decodeURIComponent(base64Image.split(',')[1]); + const base64Image = chart.getDataURL({ + type: 'png', + pixelRatio: 2 // 可以设置更高的像素比以获得更清晰的图像 + }); + const svgContent = decodeURIComponent(base64Image.split(',')[1]); + const base64 = `data:image/svg+xml;base64,${Buffer.from(svgContent).toString('base64')}`; - const fileName = `chart_${Date.now()}.svg`; - const filePath = getFileSavePath(fileName); - fs.writeFileSync(filePath, svgData); - // 释放图表实例 - chart.dispose(); - - return filePath; + return base64; }; const main = async ({ title, xAxis, yAxis, chartType }: Props): Response => { - const filePath = await generateChart(title, xAxis, yAxis, chartType); + const base64 = await generateChart(title, xAxis, yAxis, chartType); return { result: { - type: 'SYSTEM_PLUGIN_FILE', - path: filePath, - contentType: 'image/svg+xml' + type: 'SYSTEM_PLUGIN_BASE64', + value: base64, + extension: 'svg' } }; }; diff --git a/packages/plugins/type.d.ts b/packages/plugins/type.d.ts index b125a9fa7..201c6fdd0 100644 --- a/packages/plugins/type.d.ts +++ b/packages/plugins/type.d.ts @@ -4,9 +4,9 @@ import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type export type SystemPluginResponseType = Promise>; export type SystemPluginSpecialResponse = { - type: 'SYSTEM_PLUGIN_FILE'; - path: string; - contentType: string; + type: 'SYSTEM_PLUGIN_BASE64'; + value: string; + extension: string; }; declare global { diff --git a/packages/plugins/utils.ts b/packages/plugins/utils.ts deleted file mode 100644 index c86bfcc8f..000000000 --- a/packages/plugins/utils.ts +++ /dev/null @@ -1,15 +0,0 @@ -import path from 'path'; -import * as fs from 'fs'; - -const isProduction = process.env.NODE_ENV === 'production'; - -export const getFileSavePath = (name: string) => { - if (isProduction) { - return `/app/plugin_file/${name}`; - } - const filePath = path.join(process.cwd(), 'local', 'plugin_file', name); - - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - - return filePath; -}; diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index cf7acfd1c..541f07fff 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -12,6 +12,7 @@ import { gridFsStream2Buffer, stream2Encoding } from './utils'; import { addLog } from '../../system/log'; import { readFromSecondary } from '../../mongo/utils'; import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools'; +import { Readable } from 'stream'; export function getGFSCollection(bucket: `${BucketNameEnum}`) { MongoDatasetFileSchema; @@ -76,6 +77,59 @@ export async function uploadFile({ return String(stream.id); } +export async function uploadFileFromBase64Img({ + bucketName, + teamId, + tmbId, + base64, + filename, + metadata = {} +}: { + bucketName: `${BucketNameEnum}`; + teamId: string; + tmbId: string; + base64: string; + filename: string; + metadata?: Record; +}) { + if (!base64) return Promise.reject(`filePath is empty`); + if (!filename) return Promise.reject(`filename is empty`); + + const base64Data = base64.split(',')[1]; + const contentType = base64.split(',')?.[0]?.split?.(':')?.[1]; + const buffer = Buffer.from(base64Data, 'base64'); + const readableStream = new Readable({ + read() { + this.push(buffer); + this.push(null); + } + }); + + const { stream: readStream, encoding } = await stream2Encoding(readableStream); + + // Add default metadata + metadata.teamId = teamId; + metadata.tmbId = tmbId; + metadata.encoding = encoding; + + // create a gridfs bucket + const bucket = getGridBucket(bucketName); + + const stream = bucket.openUploadStream(filename, { + metadata, + contentType + }); + + // save to gridfs + await new Promise((resolve, reject) => { + readStream + .pipe(stream as any) + .on('finish', resolve) + .on('error', reject); + }); + + return String(stream.id); +} export async function getFileById({ bucketName, diff --git a/packages/service/core/chat/utils.ts b/packages/service/core/chat/utils.ts index 9443ccde5..56b74aa47 100644 --- a/packages/service/core/chat/utils.ts +++ b/packages/service/core/chat/utils.ts @@ -104,6 +104,9 @@ export const loadRequestMessages = async ({ }) => { // Load image to base64 const loadImageToBase64 = async (messages: ChatCompletionContentPart[]) => { + if (process.env.MULTIPLE_DATA_TO_BASE64 === 'false') { + return messages; + } return Promise.all( messages.map(async (item) => { if (item.type === 'image_url') { diff --git a/packages/service/core/workflow/dispatch/agent/runTool/index.ts b/packages/service/core/workflow/dispatch/agent/runTool/index.ts index 2b34f86dd..74bfa25e0 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/index.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/index.ts @@ -185,7 +185,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< // array, replace last element const lastText = lastMessage.content[lastMessage.content.length - 1]; if (lastText.type === 'text') { - lastMessage.content = replaceVariable(Prompt_Tool_Call, { + lastText.text = replaceVariable(Prompt_Tool_Call, { question: lastText.text }); } else { diff --git a/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts b/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts index dac147e9a..9c176ecc9 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts @@ -176,17 +176,29 @@ export const runToolWithPromptCall = async ( ); const lastMessage = messages[messages.length - 1]; - if (typeof lastMessage.content !== 'string') { + if (typeof lastMessage.content === 'string') { + lastMessage.content = replaceVariable(lastMessage.content, { + toolsPrompt + }); + } else if (Array.isArray(lastMessage.content)) { + // array, replace last element + const lastText = lastMessage.content[lastMessage.content.length - 1]; + if (lastText.type === 'text') { + lastText.text = replaceVariable(lastText.text, { + toolsPrompt + }); + } else { + return Promise.reject('Prompt call invalid input'); + } + } else { return Promise.reject('Prompt call invalid input'); } - lastMessage.content = replaceVariable(lastMessage.content, { - toolsPrompt - }); const filterMessages = await filterGPTMessageByMaxTokens({ messages, maxTokens: toolModel.maxContext - 500 // filter token. not response maxToken }); + const [requestMessages, max_tokens] = await Promise.all([ loadRequestMessages({ messages: filterMessages, @@ -398,11 +410,27 @@ export const runToolWithPromptCall = async ( : undefined; // get the next user prompt - lastMessage.content += `${replaceAnswer} + if (typeof lastMessage.content === 'string') { + lastMessage.content += `${replaceAnswer} TOOL_RESPONSE: """ ${workflowInteractiveResponseItem ? `{{${INTERACTIVE_STOP_SIGNAL}}}` : toolsRunResponse.toolResponsePrompt} """ ANSWER: `; + } else if (Array.isArray(lastMessage.content)) { + // array, replace last element + const lastText = lastMessage.content[lastMessage.content.length - 1]; + if (lastText.type === 'text') { + lastText.text += `${replaceAnswer} +TOOL_RESPONSE: """ +${workflowInteractiveResponseItem ? `{{${INTERACTIVE_STOP_SIGNAL}}}` : toolsRunResponse.toolResponsePrompt} +""" +ANSWER: `; + } else { + return Promise.reject('Prompt call invalid input'); + } + } else { + return Promise.reject('Prompt call invalid input'); + } const runTimes = (response?.runTimes || 0) + toolsRunResponse.toolResponse.runTimes; const toolNodeTokens = response?.toolNodeTokens ? response.toolNodeTokens + tokens : tokens; diff --git a/packages/service/core/workflow/dispatch/tools/http468.ts b/packages/service/core/workflow/dispatch/tools/http468.ts index 7a8cf0d47..145ded177 100644 --- a/packages/service/core/workflow/dispatch/tools/http468.ts +++ b/packages/service/core/workflow/dispatch/tools/http468.ts @@ -18,11 +18,11 @@ import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/util import { getSystemPluginCb } from '../../../../../plugins/register'; import { ContentTypes } from '@fastgpt/global/core/workflow/constants'; import { replaceEditorVariable } from '@fastgpt/global/core/workflow/utils'; -import { uploadFile } from '../../../../common/file/gridfs/controller'; +import { uploadFileFromBase64Img } from '../../../../common/file/gridfs/controller'; import { ReadFileBaseUrl } from '@fastgpt/global/common/file/constants'; import { createFileToken } from '../../../../support/permission/controller'; -import { removeFilesByPaths } from '../../../../common/file/utils'; import { JSONPath } from 'jsonpath-plus'; +import type { SystemPluginSpecialResponse } from '../../../../../plugins/type'; type PropsArrType = { key: string; @@ -376,27 +376,25 @@ async function replaceSystemPluginResponse({ tmbId: string; }) { for await (const key of Object.keys(response)) { - if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_FILE') { - const fileObj = response[key]; - const filename = fileObj.path.split('/').pop() || `${tmbId}-${Date.now()}`; + if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_BASE64') { + const fileObj = response[key] as SystemPluginSpecialResponse; + const filename = `${tmbId}-${Date.now()}.${fileObj.extension}`; try { - const fileId = await uploadFile({ + const fileId = await uploadFileFromBase64Img({ teamId, tmbId, bucketName: 'chat', - path: fileObj.path, + base64: fileObj.value, filename, - contentType: fileObj.contentType, metadata: {} }); - response[key] = `${ReadFileBaseUrl}?filename=${filename}&token=${await createFileToken({ + response[key] = `${ReadFileBaseUrl}/${filename}?token=${await createFileToken({ bucketName: 'chat', teamId, tmbId, fileId })}`; } catch (error) {} - removeFilesByPaths([fileObj.path]); } } return response; diff --git a/projects/app/Dockerfile b/projects/app/Dockerfile index 43f10c95d..3970f3795 100644 --- a/projects/app/Dockerfile +++ b/projects/app/Dockerfile @@ -70,9 +70,9 @@ COPY --from=maindeps /app/node_modules/@zilliz/milvus2-sdk-node ./node_modules/@ # copy package.json to version file COPY --from=builder /app/projects/app/package.json ./package.json + # copy config COPY ./projects/app/data /app/data - RUN chown -R nextjs:nodejs /app/data ENV NODE_ENV=production diff --git a/projects/app/src/pages/api/common/file/read/[filename].ts b/projects/app/src/pages/api/common/file/read/[filename].ts new file mode 100644 index 000000000..1e4a96842 --- /dev/null +++ b/projects/app/src/pages/api/common/file/read/[filename].ts @@ -0,0 +1,63 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authFileToken } from '@fastgpt/service/support/permission/controller'; +import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { stream2Encoding } from '@fastgpt/service/common/file/gridfs/utils'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + const { token, filename } = req.query as { token: string; filename: string }; + + const { fileId, bucketName } = await authFileToken(token); + + if (!fileId) { + throw new Error('fileId is empty'); + } + + const [file, fileStream] = await Promise.all([ + getFileById({ bucketName, fileId }), + getDownloadStream({ bucketName, fileId }) + ]); + + if (!file) { + return Promise.reject(CommonErrEnum.fileNotFound); + } + + const { stream, encoding } = await (async () => { + if (file.metadata?.encoding) { + return { + stream: fileStream, + encoding: file.metadata.encoding + }; + } + return stream2Encoding(fileStream); + })(); + + res.setHeader('Content-Type', `${file.contentType}; charset=${encoding}`); + res.setHeader('Cache-Control', 'public, max-age=31536000'); + res.setHeader('Content-Disposition', `inline; filename="${encodeURIComponent(filename)}"`); + + stream.pipe(res); + + stream.on('error', () => { + res.status(500).end(); + }); + stream.on('end', () => { + res.end(); + }); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } +} +export const config = { + api: { + responseLimit: '100mb' + } +}; diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts index 11c00bba0..7d6c840f3 100644 --- a/projects/app/src/pages/api/common/file/upload.ts +++ b/projects/app/src/pages/api/common/file/upload.ts @@ -57,14 +57,12 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { jsonRes(res, { data: { fileId, - previewUrl: `${ReadFileBaseUrl}?filename=${file.originalname}&token=${await createFileToken( - { - bucketName, - teamId, - tmbId, - fileId - } - )}` + previewUrl: `${ReadFileBaseUrl}/${file.originalname}?token=${await createFileToken({ + bucketName, + teamId, + tmbId, + fileId + })}` } }); } catch (error) {