HTTP support jsonPath; System plugin support save file. (#2969)

* perf: system plugin auto save file * feat: http support jsonPath * fix: assistant response * reset milvus version * fix: textarea register * fix: global variable * delete tip * doc
2025-07-27 00:17:31 +00:00 · 2024-10-23 00:40:54 +08:00
parent 718108a552
commit 9b0706ed92
30 changed files with 374 additions and 163 deletions
--- a/packages/global/core/workflow/template/system/http468.ts
+++ b/packages/global/core/workflow/template/system/http468.ts
@@ -114,7 +114,9 @@ export const HttpNode468: FlowNodeTemplateType = {
  ],
  outputs: [
    {
-      ...Output_Template_AddOutput
+      ...Output_Template_AddOutput,
+      label: i18nT('workflow:http_extract_output'),
+      description: i18nT('workflow:http_extract_output_description')
    },
    {
      id: NodeOutputKeyEnum.error,
--- a/packages/plugins/package.json
+++ b/packages/plugins/package.json
@@ -10,6 +10,7 @@
    "expr-eval": "^2.0.2",
    "lodash": "^4.17.21",
    "mysql2": "^3.11.3",
+    "json5": "^2.2.3",
    "pg": "^8.10.0",
    "wikijs": "^6.4.1"
  },
--- a/packages/plugins/src/drawing/baseChart/index.ts
+++ b/packages/plugins/src/drawing/baseChart/index.ts
@@ -1,4 +1,8 @@
 import * as echarts from 'echarts';
+import json5 from 'json5';
+import { getFileSavePath } from '../../../utils';
+import * as fs from 'fs';
+import { SystemPluginSpecialResponse } from '../../../type.d';

 type Props = {
  title: string;
@@ -8,7 +12,7 @@ type Props = {
 };

 type Response = Promise<{
-  result: string;
+  result: SystemPluginSpecialResponse;
 }>;

 type SeriesData = {
@@ -37,8 +41,8 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT
  let parsedXAxis: string[] = [];
  let parsedYAxis: number[] = [];
  try {
-    parsedXAxis = JSON.parse(xAxis);
-    parsedYAxis = JSON.parse(yAxis);
+    parsedXAxis = json5.parse(xAxis);
+    parsedYAxis = json5.parse(yAxis);
  } catch (error: any) {
    console.error('解析数据时出错:', error);
    return Promise.reject('Data error');
@@ -78,16 +82,26 @@ const generateChart = async (title: string, xAxis: string, yAxis: string, chartT

  chart.setOption(option);
  // 生成 Base64 图像
-  const base64Image = chart.getDataURL({ type: 'png' });
+  const base64Image = chart.getDataURL();
+  const svgData = decodeURIComponent(base64Image.split(',')[1]);
+
+  const fileName = `chart_${Date.now()}.svg`;
+  const filePath = getFileSavePath(fileName);
+  fs.writeFileSync(filePath, svgData);
  // 释放图表实例
  chart.dispose();

-  return base64Image;
+  return filePath;
 };

 const main = async ({ title, xAxis, yAxis, chartType }: Props): Response => {
+  const filePath = await generateChart(title, xAxis, yAxis, chartType);
  return {
-    result: await generateChart(title, xAxis, yAxis, chartType)
+    result: {
+      type: 'SYSTEM_PLUGIN_FILE',
+      path: filePath,
+      contentType: 'image/svg+xml'
+    }
  };
 };

--- a/packages/plugins/src/drawing/baseChart/template.json
+++ b/packages/plugins/src/drawing/baseChart/template.json
@@ -50,10 +50,10 @@
            "canEdit": true,
            "key": "xAxis",
            "label": "xAxis",
-            "description": "x轴数据，例如：['A', 'B', 'C']",
+            "description": "x轴数据，例如：[\"A\", \"B\", \"C\"]",
            "defaultValue": "",
            "required": true,
-            "toolDescription": "x轴数据，例如：['A', 'B', 'C']",
+            "toolDescription": "x轴数据，例如：[\"A\", \"B\", \"C\"]",
            "list": [
              {
                "label": "",
--- a/packages/plugins/type.d.ts
+++ b/packages/plugins/type.d.ts
@@ -1,7 +1,13 @@
 import { PluginTemplateType } from '@fastgpt/global/core/plugin/type.d';
+import { systemPluginResponseEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type';

 export type SystemPluginResponseType = Promise<Record<string, any>>;
+export type SystemPluginSpecialResponse = {
+  type: 'SYSTEM_PLUGIN_FILE';
+  path: string;
+  contentType: string;
+};

 declare global {
  var systemPlugins: SystemPluginTemplateItemType[];
--- a/packages/plugins/utils.ts
+++ b/packages/plugins/utils.ts
@@ -0,0 +1,15 @@
+import path from 'path';
+import * as fs from 'fs';
+
+const isProduction = process.env.NODE_ENV === 'production';
+
+export const getFileSavePath = (name: string) => {
+  if (isProduction) {
+    return `/app/plugin_file/${name}`;
+  }
+  const filePath = path.join(process.cwd(), 'local', 'plugin_file', name);
+
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+
+  return filePath;
+};
--- a/packages/service/common/string/tiktoken/index.ts
+++ b/packages/service/common/string/tiktoken/index.ts
@@ -25,7 +25,7 @@ export const countGptMessagesTokens = async (
      number
    >({
      name: WorkerNameEnum.countGptMessagesTokens,
-      maxReservedThreads: global.systemEnv?.tokenWorkers || 20
+      maxReservedThreads: global.systemEnv?.tokenWorkers || 50
    });

    const total = await workerController.run({ messages, tools, functionCall });
--- a/packages/service/common/vectorStore/milvus/class.ts
+++ b/packages/service/common/vectorStore/milvus/class.ts
@@ -24,10 +24,7 @@ export class MilvusCtrl {

    global.milvusClient = new MilvusClient({
      address: MILVUS_ADDRESS,
-      token: MILVUS_TOKEN,
-      loaderOptions: {
-        longs: Function
-      }
+      token: MILVUS_TOKEN
    });

    addLog.info(`Milvus connected`);
@@ -329,19 +326,10 @@ export class MilvusCtrl {
      id: string;
      teamId: string;
      datasetId: string;
-      int64: {
-        low: bigint;
-        high: bigint;
-        unsigned: boolean;
-      };
    }[];

    return rows.map((item) => ({
-      id: String({
-        low: BigInt(item.int64.low),
-        high: BigInt(item.int64.high),
-        unsigned: item.int64.unsigned
-      }),
+      id: String(item.id),
      teamId: item.teamId,
      datasetId: item.datasetId
    }));
--- a/packages/service/core/chat/utils.ts
+++ b/packages/service/core/chat/utils.ts
@@ -222,10 +222,9 @@ export const loadRequestMessages = async ({
            };
          }
        }
-        // if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
-        //   if (item.content === undefined && !item.tool_calls && !item.function_call) return;
-        //   if (Array.isArray(item.content) && item.content.length === 0) return;
-        // }
+        if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
+          if (item.content === undefined && !item.tool_calls && !item.function_call) return;
+        }

        return item;
      })
--- a/packages/service/core/workflow/dispatch/tools/http468.ts
+++ b/packages/service/core/workflow/dispatch/tools/http468.ts
@@ -18,6 +18,11 @@ import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/util
 import { getSystemPluginCb } from '../../../../../plugins/register';
 import { ContentTypes } from '@fastgpt/global/core/workflow/constants';
 import { replaceEditorVariable } from '@fastgpt/global/core/workflow/utils';
+import { uploadFile } from '../../../../common/file/gridfs/controller';
+import { ReadFileBaseUrl } from '@fastgpt/global/common/file/constants';
+import { createFileToken } from '../../../../support/permission/controller';
+import { removeFilesByPaths } from '../../../../common/file/utils';
+import { JSONPath } from 'jsonpath-plus';

 type PropsArrType = {
  key: string;
@@ -55,7 +60,7 @@ const contentTypeMap = {

 export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<HttpResponse> => {
  let {
-    runningAppInfo: { id: appId },
+    runningAppInfo: { id: appId, teamId, tmbId },
    chatId,
    responseChatItemId,
    variables,
@@ -204,7 +209,12 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
    const { formatResponse, rawResponse } = await (async () => {
      const systemPluginCb = await getSystemPluginCb();
      if (systemPluginCb[httpReqUrl]) {
-        const pluginResult = await systemPluginCb[httpReqUrl](requestBody);
+        const pluginResult = await replaceSystemPluginResponse({
+          response: await systemPluginCb[httpReqUrl](requestBody),
+          teamId,
+          tmbId
+        });
+
        return {
          formatResponse: pluginResult,
          rawResponse: pluginResult
@@ -222,11 +232,10 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H

    // format output value type
    const results: Record<string, any> = {};
-    for (const key in formatResponse) {
-      const output = node.outputs.find((item) => item.key === key);
-      if (!output) continue;
-      results[key] = valueTypeFormat(formatResponse[key], output.valueType);
-    }
+    node.outputs.forEach((item) => {
+      const key = item.key.startsWith('$') ? item.key : `$.${item.key}`;
+      results[item.key] = JSONPath({ path: key, json: formatResponse })[0];
+    });

    if (typeof formatResponse[NodeOutputKeyEnum.answerText] === 'string') {
      workflowStreamResponse?.({
@@ -293,80 +302,8 @@ async function fetchData({
    data: ['POST', 'PUT', 'PATCH'].includes(method) ? body : undefined
  });

-  /*
-    parse the json:
-    {
-      user: {
-        name: 'xxx',
-        age: 12
-      },
-      list: [
-        {
-          name: 'xxx',
-          age: 50
-        },
-        [{ test: 22 }]
-      ],
-      psw: 'xxx'
-    }
-
-    result: {
-      'user': { name: 'xxx', age: 12 },
-      'user.name': 'xxx',
-      'user.age': 12,
-      'list': [ { name: 'xxx', age: 50 }, [ [Object] ] ],
-      'list[0]': { name: 'xxx', age: 50 },
-      'list[0].name': 'xxx',
-      'list[0].age': 50,
-      'list[1]': [ { test: 22 } ],
-      'list[1][0]': { test: 22 },
-      'list[1][0].test': 22,
-      'psw': 'xxx'
-    }
-  */
-  const parseJson = (obj: Record<string, any>, prefix = '') => {
-    let result: Record<string, any> = {};
-
-    if (Array.isArray(obj)) {
-      for (let i = 0; i < obj.length; i++) {
-        result[`${prefix}[${i}]`] = obj[i];
-
-        if (Array.isArray(obj[i])) {
-          result = {
-            ...result,
-            ...parseJson(obj[i], `${prefix}[${i}]`)
-          };
-        } else if (typeof obj[i] === 'object') {
-          result = {
-            ...result,
-            ...parseJson(obj[i], `${prefix}[${i}].`)
-          };
-        }
-      }
-    } else if (typeof obj == 'object') {
-      for (const key in obj) {
-        result[`${prefix}${key}`] = obj[key];
-
-        if (Array.isArray(obj[key])) {
-          result = {
-            ...result,
-            ...parseJson(obj[key], `${prefix}${key}`)
-          };
-        } else if (typeof obj[key] === 'object') {
-          result = {
-            ...result,
-            ...parseJson(obj[key], `${prefix}${key}.`)
-          };
-        }
-      }
-    }
-
-    return result;
-  };
-
  return {
-    formatResponse:
-      typeof response === 'object' && !Array.isArray(response) ? parseJson(response) : {},
+    formatResponse: typeof response === 'object' ? response : {},
    rawResponse: response
  };
 }
@@ -405,3 +342,40 @@ function removeUndefinedSign(obj: Record<string, any>) {
  }
  return obj;
 }
+
+// Replace some special response from system plugin
+async function replaceSystemPluginResponse({
+  response,
+  teamId,
+  tmbId
+}: {
+  response: Record<string, any>;
+  teamId: string;
+  tmbId: string;
+}) {
+  for await (const key of Object.keys(response)) {
+    if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_FILE') {
+      const fileObj = response[key];
+      const filename = fileObj.path.split('/').pop() || `${tmbId}-${Date.now()}`;
+      try {
+        const fileId = await uploadFile({
+          teamId,
+          tmbId,
+          bucketName: 'chat',
+          path: fileObj.path,
+          filename,
+          contentType: fileObj.contentType,
+          metadata: {}
+        });
+        response[key] = `${ReadFileBaseUrl}?filename=${filename}&token=${await createFileToken({
+          bucketName: 'chat',
+          teamId,
+          tmbId,
+          fileId
+        })}`;
+      } catch (error) {}
+      removeFilesByPaths([fileObj.path]);
+    }
+  }
+  return response;
+}
--- a/packages/service/package.json
+++ b/packages/service/package.json
@@ -20,6 +20,7 @@
    "iconv-lite": "^0.6.3",
    "joplin-turndown-plugin-gfm": "^1.0.12",
    "json5": "^2.2.3",
+    "jsonpath-plus": "^10.1.0",
    "jsonwebtoken": "^9.0.2",
    "lodash": "^4.17.21",
    "mammoth": "^1.6.0",
--- a/packages/web/hooks/useI18n.ts
+++ b/packages/web/hooks/useI18n.ts
@@ -26,12 +26,11 @@ export const useI18nLng = () => {
    if (!navigator || !localStorage) return;
    if (getCookie(LANG_KEY)) return onChangeLng(getCookie(LANG_KEY) as string);

-    const languageMap = {
+    const languageMap: Record<string, string> = {
      zh: 'zh',
      'zh-CN': 'zh'
    };

-    // @ts-ignore
    const lang = languageMap[navigator.language] || 'en';

    // currentLng not in userLang
--- a/packages/web/i18n/en/app.json
+++ b/packages/web/i18n/en/app.json
@@ -41,6 +41,7 @@
  "file_recover": "File will overwrite current content",
  "file_upload": "File Upload",
  "file_upload_tip": "Once enabled, documents/images can be uploaded. Documents are retained for 7 days, images for 15 days. Using this feature may incur additional costs. To ensure a good experience, please choose an AI model with a larger context length when using this feature.",
+  "global_variables_desc": "Variable description",
  "go_to_chat": "Go to Conversation",
  "go_to_run": "Go to Execution",
  "image_upload": "Image Upload",
@@ -159,4 +160,4 @@
  "workflow.user_file_input_desc": "Links to documents and images uploaded by users.",
  "workflow.user_select": "User Selection",
  "workflow.user_select_tip": "This module can configure multiple options for selection during the dialogue. Different options can lead to different workflow branches."
-}
+}
--- a/packages/web/i18n/en/workflow.json
+++ b/packages/web/i18n/en/workflow.json
@@ -64,6 +64,8 @@
  "full_response_data": "Full Response Data",
  "greater_than": "Greater Than",
  "greater_than_or_equal_to": "Greater Than or Equal To",
+  "http_extract_output": "Output field extraction",
+  "http_extract_output_description": "Specified fields in the response value can be extracted through JSONPath syntax",
  "http_raw_response_description": "Raw HTTP response. Only accepts string or JSON type response data.",
  "http_request": "HTTP Request",
  "http_request_error_info": "HTTP request error information, returns empty on success",
--- a/packages/web/i18n/zh/app.json
+++ b/packages/web/i18n/zh/app.json
@@ -5,11 +5,8 @@
  "app.Version name": "版本名称",
  "app.modules.click to update": "点击更新",
  "app.modules.has new version": "有新版本",
-  "version_back": "回到初始状态",
-  "version_copy": "副本",
  "app.version_current": "当前版本",
  "app.version_initial": "初始版本",
-  "version_initial_copy": "副本-初始状态",
  "app.version_name_tips": "版本名称不能为空",
  "app.version_past": "发布过",
  "app.version_publish_tips": "该版本将被保存至团队云端，同步给整个团队，同时更新所有发布渠道的应用版本",
@@ -44,6 +41,7 @@
  "file_recover": "文件将覆盖当前内容",
  "file_upload": "文件上传",
  "file_upload_tip": "开启后，可以上传文档/图片。文档保留7天，图片保留15天。使用该功能可能产生较多额外费用。为保证使用体验，使用该功能时，请选择上下文长度较大的AI模型。",
+  "global_variables_desc": "变量描述",
  "go_to_chat": "去对话",
  "go_to_run": "去运行",
  "image_upload": "图片上传",
@@ -132,6 +130,9 @@
  "variable.select type_desc": "可以为工作流定义全局变量，常用临时缓存。赋值的方式包括：\n1. 从对话页面的 query 参数获取。\n2. 通过 API 的 variables 对象传递。\n3. 通过【变量更新】节点进行赋值。",
  "variable.textarea_type_desc": "允许用户最多输入4000字的对话框。",
  "version.Revert success": "回滚成功",
+  "version_back": "回到初始状态",
+  "version_copy": "副本",
+  "version_initial_copy": "副本-初始状态",
  "vision_model_title": "启用图片识别",
  "week.Friday": "星期五",
  "week.Monday": "星期一",
@@ -159,4 +160,4 @@
  "workflow.user_file_input_desc": "用户上传的文档和图片链接",
  "workflow.user_select": "用户选择",
  "workflow.user_select_tip": "该模块可配置多个选项，以供对话时选择。不同选项可导向不同工作流支线"
-}
+}
--- a/packages/web/i18n/zh/workflow.json
+++ b/packages/web/i18n/zh/workflow.json
@@ -65,6 +65,8 @@
  "full_response_data": "完整响应数据",
  "greater_than": "大于",
  "greater_than_or_equal_to": "大于等于",
+  "http_extract_output": "输出字段提取",
+  "http_extract_output_description": "可以通过 JSONPath 语法来提取响应值中的指定字段",
  "http_raw_response_description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
  "http_request": "HTTP 请求",
  "http_request_error_info": "HTTP请求错误信息，成功时返回空",