HTTP support jsonPath; System plugin support save file. (#2969)

* perf: system plugin auto save file

* feat: http support jsonPath

* fix: assistant response

* reset milvus version

* fix: textarea register

* fix: global variable

* delete tip

* doc

packages/service/common/string/tiktoken/index.ts

@@ -25,7 +25,7 @@ export const countGptMessagesTokens = async (
       number
     >({
       name: WorkerNameEnum.countGptMessagesTokens,
-      maxReservedThreads: global.systemEnv?.tokenWorkers || 20
+      maxReservedThreads: global.systemEnv?.tokenWorkers || 50
     });
 
     const total = await workerController.run({ messages, tools, functionCall });

packages/service/common/vectorStore/milvus/class.ts

@@ -24,10 +24,7 @@ export class MilvusCtrl {
 
     global.milvusClient = new MilvusClient({
       address: MILVUS_ADDRESS,
-      token: MILVUS_TOKEN,
-      loaderOptions: {
-        longs: Function
-      }
+      token: MILVUS_TOKEN
     });
 
     addLog.info(`Milvus connected`);
@@ -329,19 +326,10 @@ export class MilvusCtrl {
       id: string;
       teamId: string;
       datasetId: string;
-      int64: {
-        low: bigint;
-        high: bigint;
-        unsigned: boolean;
-      };
     }[];
 
     return rows.map((item) => ({
-      id: String({
-        low: BigInt(item.int64.low),
-        high: BigInt(item.int64.high),
-        unsigned: item.int64.unsigned
-      }),
+      id: String(item.id),
       teamId: item.teamId,
       datasetId: item.datasetId
     }));

packages/service/core/chat/utils.ts

@@ -222,10 +222,9 @@ export const loadRequestMessages = async ({
             };
           }
         }
-        // if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
-        //   if (item.content === undefined && !item.tool_calls && !item.function_call) return;
-        //   if (Array.isArray(item.content) && item.content.length === 0) return;
-        // }
+        if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
+          if (item.content === undefined && !item.tool_calls && !item.function_call) return;
+        }
 
         return item;
       })

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -18,6 +18,11 @@ import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/util
 import { getSystemPluginCb } from '../../../../../plugins/register';
 import { ContentTypes } from '@fastgpt/global/core/workflow/constants';
 import { replaceEditorVariable } from '@fastgpt/global/core/workflow/utils';
+import { uploadFile } from '../../../../common/file/gridfs/controller';
+import { ReadFileBaseUrl } from '@fastgpt/global/common/file/constants';
+import { createFileToken } from '../../../../support/permission/controller';
+import { removeFilesByPaths } from '../../../../common/file/utils';
+import { JSONPath } from 'jsonpath-plus';
 
 type PropsArrType = {
   key: string;
@@ -55,7 +60,7 @@ const contentTypeMap = {
 
 export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<HttpResponse> => {
   let {
-    runningAppInfo: { id: appId },
+    runningAppInfo: { id: appId, teamId, tmbId },
     chatId,
     responseChatItemId,
     variables,
@@ -204,7 +209,12 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
     const { formatResponse, rawResponse } = await (async () => {
       const systemPluginCb = await getSystemPluginCb();
       if (systemPluginCb[httpReqUrl]) {
-        const pluginResult = await systemPluginCb[httpReqUrl](requestBody);
+        const pluginResult = await replaceSystemPluginResponse({
+          response: await systemPluginCb[httpReqUrl](requestBody),
+          teamId,
+          tmbId
+        });
+
         return {
           formatResponse: pluginResult,
           rawResponse: pluginResult
@@ -222,11 +232,10 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
 
     // format output value type
     const results: Record<string, any> = {};
-    for (const key in formatResponse) {
-      const output = node.outputs.find((item) => item.key === key);
-      if (!output) continue;
-      results[key] = valueTypeFormat(formatResponse[key], output.valueType);
-    }
+    node.outputs.forEach((item) => {
+      const key = item.key.startsWith('$') ? item.key : `$.${item.key}`;
+      results[item.key] = JSONPath({ path: key, json: formatResponse })[0];
+    });
 
     if (typeof formatResponse[NodeOutputKeyEnum.answerText] === 'string') {
       workflowStreamResponse?.({
@@ -293,80 +302,8 @@ async function fetchData({
     data: ['POST', 'PUT', 'PATCH'].includes(method) ? body : undefined
   });
 
-  /*
-    parse the json:
-    {
-      user: {
-        name: 'xxx',
-        age: 12
-      },
-      list: [
-        {
-          name: 'xxx',
-          age: 50
-        },
-        [{ test: 22 }]
-      ],
-      psw: 'xxx'
-    }
-
-    result: {
-      'user': { name: 'xxx', age: 12 },
-      'user.name': 'xxx',
-      'user.age': 12,
-      'list': [ { name: 'xxx', age: 50 }, [ [Object] ] ],
-      'list[0]': { name: 'xxx', age: 50 },
-      'list[0].name': 'xxx',
-      'list[0].age': 50,
-      'list[1]': [ { test: 22 } ],
-      'list[1][0]': { test: 22 },
-      'list[1][0].test': 22,
-      'psw': 'xxx'
-    }
-  */
-  const parseJson = (obj: Record<string, any>, prefix = '') => {
-    let result: Record<string, any> = {};
-
-    if (Array.isArray(obj)) {
-      for (let i = 0; i < obj.length; i++) {
-        result[`${prefix}[${i}]`] = obj[i];
-
-        if (Array.isArray(obj[i])) {
-          result = {
-            ...result,
-            ...parseJson(obj[i], `${prefix}[${i}]`)
-          };
-        } else if (typeof obj[i] === 'object') {
-          result = {
-            ...result,
-            ...parseJson(obj[i], `${prefix}[${i}].`)
-          };
-        }
-      }
-    } else if (typeof obj == 'object') {
-      for (const key in obj) {
-        result[`${prefix}${key}`] = obj[key];
-
-        if (Array.isArray(obj[key])) {
-          result = {
-            ...result,
-            ...parseJson(obj[key], `${prefix}${key}`)
-          };
-        } else if (typeof obj[key] === 'object') {
-          result = {
-            ...result,
-            ...parseJson(obj[key], `${prefix}${key}.`)
-          };
-        }
-      }
-    }
-
-    return result;
-  };
-
   return {
-    formatResponse:
-      typeof response === 'object' && !Array.isArray(response) ? parseJson(response) : {},
+    formatResponse: typeof response === 'object' ? response : {},
     rawResponse: response
   };
 }
@@ -405,3 +342,40 @@ function removeUndefinedSign(obj: Record<string, any>) {
   }
   return obj;
 }
+
+// Replace some special response from system plugin
+async function replaceSystemPluginResponse({
+  response,
+  teamId,
+  tmbId
+}: {
+  response: Record<string, any>;
+  teamId: string;
+  tmbId: string;
+}) {
+  for await (const key of Object.keys(response)) {
+    if (typeof response[key] === 'object' && response[key].type === 'SYSTEM_PLUGIN_FILE') {
+      const fileObj = response[key];
+      const filename = fileObj.path.split('/').pop() || `${tmbId}-${Date.now()}`;
+      try {
+        const fileId = await uploadFile({
+          teamId,
+          tmbId,
+          bucketName: 'chat',
+          path: fileObj.path,
+          filename,
+          contentType: fileObj.contentType,
+          metadata: {}
+        });
+        response[key] = `${ReadFileBaseUrl}?filename=${filename}&token=${await createFileToken({
+          bucketName: 'chat',
+          teamId,
+          tmbId,
+          fileId
+        })}`;
+      } catch (error) {}
+      removeFilesByPaths([fileObj.path]);
+    }
+  }
+  return response;
+}

packages/service/package.json

@@ -20,6 +20,7 @@
     "iconv-lite": "^0.6.3",
     "joplin-turndown-plugin-gfm": "^1.0.12",
     "json5": "^2.2.3",
+    "jsonpath-plus": "^10.1.0",
     "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
     "mammoth": "^1.6.0",