perf: forbid image to base64 (#3038)

* perf: forbid image to base64

* update file upload path

* feat: support promptCall use image

* fix: echarts load

* update doc

packages/plugins/src/drawing/baseChart/index.ts

@@ -1,7 +1,5 @@
 import * as echarts from 'echarts';
 import json5 from 'json5';
-import { getFileSavePath } from '../../../utils';
-import * as fs from 'fs';
 import { SystemPluginSpecialResponse } from '../../../type.d';
 
 type Props = {
@@ -82,25 +80,23 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT
 
   chart.setOption(option);
   // 生成 Base64 图像
-  const base64Image = chart.getDataURL();
-  const svgData = decodeURIComponent(base64Image.split(',')[1]);
+  const base64Image = chart.getDataURL({
+    type: 'png',
+    pixelRatio: 2 // 可以设置更高的像素比以获得更清晰的图像
+  });
+  const svgContent = decodeURIComponent(base64Image.split(',')[1]);
+  const base64 = `data:image/svg+xml;base64,${Buffer.from(svgContent).toString('base64')}`;
 
-  const fileName = `chart_${Date.now()}.svg`;
-  const filePath = getFileSavePath(fileName);
-  fs.writeFileSync(filePath, svgData);
-  // 释放图表实例
-  chart.dispose();
-
-  return filePath;
+  return base64;
 };
 
 const main = async ({ title, xAxis, yAxis, chartType }: Props): Response => {
-  const filePath = await generateChart(title, xAxis, yAxis, chartType);
+  const base64 = await generateChart(title, xAxis, yAxis, chartType);
   return {
     result: {
-      type: 'SYSTEM_PLUGIN_FILE',
-      path: filePath,
-      contentType: 'image/svg+xml'
+      type: 'SYSTEM_PLUGIN_BASE64',
+      value: base64,
+      extension: 'svg'
     }
   };
 };

packages/plugins/type.d.ts

@@ -4,9 +4,9 @@ import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type
 
 export type SystemPluginResponseType = Promise<Record<string, any>>;
 export type SystemPluginSpecialResponse = {
-  type: 'SYSTEM_PLUGIN_FILE';
-  path: string;
-  contentType: string;
+  type: 'SYSTEM_PLUGIN_BASE64';
+  value: string;
+  extension: string;
 };
 
 declare global {

packages/plugins/utils.ts

@@ -1,15 +0,0 @@
-import path from 'path';
-import * as fs from 'fs';
-
-const isProduction = process.env.NODE_ENV === 'production';
-
-export const getFileSavePath = (name: string) => {
-  if (isProduction) {
-    return `/app/plugin_file/${name}`;
-  }
-  const filePath = path.join(process.cwd(), 'local', 'plugin_file', name);
-
-  fs.mkdirSync(path.dirname(filePath), { recursive: true });
-
-  return filePath;
-};

packages/service/common/file/gridfs/controller.ts

@@ -12,6 +12,7 @@ import { gridFsStream2Buffer, stream2Encoding } from './utils';
 import { addLog } from '../../system/log';
 import { readFromSecondary } from '../../mongo/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
+import { Readable } from 'stream';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
   MongoDatasetFileSchema;
@@ -76,6 +77,59 @@ export async function uploadFile({
 
   return String(stream.id);
 }
+export async function uploadFileFromBase64Img({
+  bucketName,
+  teamId,
+  tmbId,
+  base64,
+  filename,
+  metadata = {}
+}: {
+  bucketName: `${BucketNameEnum}`;
+  teamId: string;
+  tmbId: string;
+  base64: string;
+  filename: string;
+  metadata?: Record<string, any>;
+}) {
+  if (!base64) return Promise.reject(`filePath is empty`);
+  if (!filename) return Promise.reject(`filename is empty`);
+
+  const base64Data = base64.split(',')[1];
+  const contentType = base64.split(',')?.[0]?.split?.(':')?.[1];
+  const buffer = Buffer.from(base64Data, 'base64');
+  const readableStream = new Readable({
+    read() {
+      this.push(buffer);
+      this.push(null);
+    }
+  });
+
+  const { stream: readStream, encoding } = await stream2Encoding(readableStream);
+
+  // Add default metadata
+  metadata.teamId = teamId;
+  metadata.tmbId = tmbId;
+  metadata.encoding = encoding;
+
+  // create a gridfs bucket
+  const bucket = getGridBucket(bucketName);
+
+  const stream = bucket.openUploadStream(filename, {
+    metadata,
+    contentType
+  });
+
+  // save to gridfs
+  await new Promise((resolve, reject) => {
+    readStream
+      .pipe(stream as any)
+      .on('finish', resolve)
+      .on('error', reject);
+  });
+
+  return String(stream.id);
+}
 
 export async function getFileById({
   bucketName,

packages/service/core/chat/utils.ts

@@ -104,6 +104,9 @@ export const loadRequestMessages = async ({
 }) => {
   // Load image to base64
   const loadImageToBase64 = async (messages: ChatCompletionContentPart[]) => {
+    if (process.env.MULTIPLE_DATA_TO_BASE64 === 'false') {
+      return messages;
+    }
     return Promise.all(
       messages.map(async (item) => {
         if (item.type === 'image_url') {

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -185,7 +185,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
       // array, replace last element
       const lastText = lastMessage.content[lastMessage.content.length - 1];
       if (lastText.type === 'text') {
-        lastMessage.content = replaceVariable(Prompt_Tool_Call, {
+        lastText.text = replaceVariable(Prompt_Tool_Call, {
           question: lastText.text
         });
       } else {

packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts

@@ -176,17 +176,29 @@ export const runToolWithPromptCall = async (
   );
 
   const lastMessage = messages[messages.length - 1];
-  if (typeof lastMessage.content !== 'string') {
+  if (typeof lastMessage.content === 'string') {
+    lastMessage.content = replaceVariable(lastMessage.content, {
+      toolsPrompt
+    });
+  } else if (Array.isArray(lastMessage.content)) {
+    // array, replace last element
+    const lastText = lastMessage.content[lastMessage.content.length - 1];
+    if (lastText.type === 'text') {
+      lastText.text = replaceVariable(lastText.text, {
+        toolsPrompt
+      });
+    } else {
+      return Promise.reject('Prompt call invalid input');
+    }
+  } else {
     return Promise.reject('Prompt call invalid input');
   }
-  lastMessage.content = replaceVariable(lastMessage.content, {
-    toolsPrompt
-  });
 
   const filterMessages = await filterGPTMessageByMaxTokens({
     messages,
     maxTokens: toolModel.maxContext - 500 // filter token. not response maxToken
   });
+
   const [requestMessages, max_tokens] = await Promise.all([
     loadRequestMessages({
       messages: filterMessages,
@@ -398,11 +410,27 @@ export const runToolWithPromptCall = async (
     : undefined;
 
   // get the next user prompt
-  lastMessage.content += `${replaceAnswer}
+  if (typeof lastMessage.content === 'string') {
+    lastMessage.content += `${replaceAnswer}
 TOOL_RESPONSE: """
 ${workflowInteractiveResponseItem ? `{{${INTERACTIVE_STOP_SIGNAL}}}` : toolsRunResponse.toolResponsePrompt}
 """
 ANSWER: `;
+  } else if (Array.isArray(lastMessage.content)) {
+    // array, replace last element
+    const lastText = lastMessage.content[lastMessage.content.length - 1];
+    if (lastText.type === 'text') {
+      lastText.text += `${replaceAnswer}
+TOOL_RESPONSE: """
+${workflowInteractiveResponseItem ? `{{${INTERACTIVE_STOP_SIGNAL}}}` : toolsRunResponse.toolResponsePrompt}
+"""
+ANSWER: `;
+    } else {
+      return Promise.reject('Prompt call invalid input');
+    }
+  } else {
+    return Promise.reject('Prompt call invalid input');
+  }
 
   const runTimes = (response?.runTimes || 0) + toolsRunResponse.toolResponse.runTimes;
   const toolNodeTokens = response?.toolNodeTokens ? response.toolNodeTokens + tokens : tokens;

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -18,11 +18,11 @@ import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/util
 import { getSystemPluginCb } from '../../../../../plugins/register';
 import { ContentTypes } from '@fastgpt/global/core/workflow/constants';
 import { replaceEditorVariable } from '@fastgpt/global/core/workflow/utils';
-import { uploadFile } from '../../../../common/file/gridfs/controller';
+import { uploadFileFromBase64Img } from '../../../../common/file/gridfs/controller';
 import { ReadFileBaseUrl } from '@fastgpt/global/common/file/constants';
 import { createFileToken } from '../../../../support/permission/controller';
-import { removeFilesByPaths } from '../../../../common/file/utils';
 import { JSONPath } from 'jsonpath-plus';
+import type { SystemPluginSpecialResponse } from '../../../../../plugins/type';
 
 type PropsArrType = {
   key: string;
@@ -376,27 +376,25 @@ async function replaceSystemPluginResponse({
   tmbId: string;
 }) {
   for await (const key of Object.keys(response)) {
-    if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_FILE') {
-      const fileObj = response[key];
-      const filename = fileObj.path.split('/').pop() || `${tmbId}-${Date.now()}`;
+    if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_BASE64') {
+      const fileObj = response[key] as SystemPluginSpecialResponse;
+      const filename = `${tmbId}-${Date.now()}.${fileObj.extension}`;
       try {
-        const fileId = await uploadFile({
+        const fileId = await uploadFileFromBase64Img({
           teamId,
           tmbId,
           bucketName: 'chat',
-          path: fileObj.path,
+          base64: fileObj.value,
           filename,
-          contentType: fileObj.contentType,
           metadata: {}
         });
-        response[key] = `${ReadFileBaseUrl}?filename=${filename}&token=${await createFileToken({
+        response[key] = `${ReadFileBaseUrl}/${filename}?token=${await createFileToken({
           bucketName: 'chat',
           teamId,
           tmbId,
           fileId
         })}`;
       } catch (error) {}
-      removeFilesByPaths([fileObj.path]);
     }
   }
   return response;