feature: V4.14.3 (#5970)

* feat(marketplace): update plugin/ download count statistic (#5957) * feat: download count * feat: update ui * fix: ui * chore: update sdk verison * chore: update .env.template * chore: adjust * chore: remove console.log * chore: adjust * Update projects/marketplace/src/pages/index.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update projects/marketplace/src/pages/index.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update projects/app/src/pages/config/tool/marketplace.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: update refresh; feat: marketplace download count per hour --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * download * marketplace code * fix: ui (#5963) * feat: support dataset and files as global variables (#5961) * json & dataset * file * fix file var * fix * fix init * remove * perf: file vars * fix: file uploading errors (#5969) * fix: file uploading errors * fix build * perf: fileselector ux * feat: integrate S3 for dataset with compatibility (#5941) * fix: text split * remove test * feat: integrate S3 for dataset with compatibility * fix: delay s3 files delete timing * fix: remove imageKeys * fix: remove parsed images' TTL * fix: improve codes by pr comments --------- Co-authored-by: archer <545436317@qq.com> * remove log * perf: request limit * chore: s3 migration script (#5971) * test * perf: s3 code * fix: migration script (#5972) * perf: s3 move object * wip: fix s3 bugs (#5976) * fix: incorrect replace origin logic (#5978) * fix: add downloadURL (#5980) * perf: file variable ttl & quick create dataset with temp s3 bucket (#5973) * perf: file variable ttl & quick create dataset with temp s3 bucket * fix * plugin & form input variables (#5979) * plugin & form input variables * fix * docs: 4143.mdx (#5981) * doc: update 4143.mdx (#5982) * fix form input file ttl (#5983) * trans file type (#5986) * trans file type * fix * fix: S3 script early return (#5985) * fix: S3 script typeof * fix: truncate large filename to fit S3 name * perf(permission): add a schema verification for resource permission, tmbId, groupId, orgId should be set at least one of them (#5987) * fix: version & typo (#5988) * fix-v4.14.3 (#5991) * fix: empty alt make replace JWT failed & incorrect image dataset preview url (#5989) * fix: empty alt make replace JWT failed & incorrect image dataset preview url * fix: s3 files recovery script * fix: incorrect chat external url parsing (#5993) --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: Roy <whoeverimf5@gmail.com>
2026-05-05 01:02:59 +08:00 · 2025-11-26 20:47:28 +08:00
parent 91156f13e7
commit 58000324e2
169 changed files with 6247 additions and 2276 deletions
@@ -41,6 +41,9 @@ import { i18nT } from '../../../../../web/i18n/utils';
 import { postTextCensor } from '../../../chat/postTextCensor';
 import { createLLMResponse } from '../../../ai/llm/request';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
+import { replaceDatasetQuoteTextWithJWT } from '../../../dataset/utils';
+import { getFileS3Key } from '../../../../common/s3/utils';
+import { addDays } from 'date-fns';

 export type ChatProps = ModuleDispatchProps<
  AIChatNodeProps & {
@@ -98,6 +101,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
      stringQuoteText //abandon
    }
  } = props;
+
  const { files: inputFiles } = chatValue2RuntimePrompt(query); // Chat box input files

  const modelConstantsData = getLLMModel(model);
@@ -303,6 +307,7 @@ async function filterDatasetQuote({
      : '';

  return {
+    // datasetQuoteText: replaceDatasetQuoteTextWithJWT(datasetQuoteText, addDays(new Date(), 90))
    datasetQuoteText
  };
 }
@@ -31,6 +31,7 @@ import { postTextCensor } from '../../../../chat/postTextCensor';
 import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io';
 import type { McpToolDataType } from '@fastgpt/global/core/app/tool/mcpTool/type';
 import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema';
+import { getFileS3Key } from '../../../../../common/s3/utils';

 type Response = DispatchNodeResultType<{
  [NodeOutputKeyEnum.answerText]: string;
@@ -119,7 +120,10 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
      fileLinks,
      inputFiles: globalFiles,
      hasReadFilesTool,
-      usageId
+      usageId,
+      appId: props.runningAppInfo.id,
+      chatId: props.chatId,
+      uId: props.uid
    });

    const concatenateSystemPrompt = [
@@ -277,7 +281,10 @@ const getMultiInput = async ({
  customPdfParse,
  inputFiles,
  hasReadFilesTool,
-  usageId
+  usageId,
+  appId,
+  chatId,
+  uId
 }: {
  runningUserInfo: ChatDispatchProps['runningUserInfo'];
  histories: ChatItemType[];
@@ -288,6 +295,9 @@ const getMultiInput = async ({
  inputFiles: UserChatItemValueItemType['file'][];
  hasReadFilesTool: boolean;
  usageId?: string;
+  appId: string;
+  chatId?: string;
+  uId: string;
 }) => {
  // Not file quote
  if (!fileLinks || hasReadFilesTool) {
@@ -53,12 +53,16 @@ import type { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/const
 import { createChatUsageRecord, pushChatItemUsage } from '../../../support/wallet/usage/controller';
 import type { RequireOnlyOne } from '@fastgpt/global/common/type/utils';
 import { getS3ChatSource } from '../../../common/s3/sources/chat';
-import { addPreviewUrlToChatItems } from '../../chat/utils';
+import { addPreviewUrlToChatItems, presignVariablesFileUrls } from '../../chat/utils';
 import type { MCPClient } from '../../app/mcp';
 import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
 import { i18nT } from '../../../../web/i18n/utils';
+import { clone } from 'lodash';

-type Props = Omit<ChatDispatchProps, 'workflowDispatchDeep' | 'timezone' | 'externalProvider'> & {
+type Props = Omit<
+  ChatDispatchProps,
+  'workflowDispatchDeep' | 'timezone' | 'externalProvider' | 'cloneVariables'
+> & {
  runtimeNodes: RuntimeNodeItemType[];
  runtimeEdges: RuntimeEdgeItemType[];
  defaultSkipNodeQueue?: WorkflowDebugResponse['skipNodeQueue'];
@@ -144,7 +148,7 @@ export async function dispatchWorkFlow({
  }

  // Get default variables
-
+  const cloneVariables = clone(data.variables);
  const defaultVariables = {
    ...externalProvider.externalWorkflowVariables,
    ...(await getSystemVariables({
@@ -168,7 +172,8 @@ export async function dispatchWorkFlow({
    workflowDispatchDeep: 0,
    usageId: newUsageId,
    concatUsage,
-    mcpClientMemory
+    mcpClientMemory,
+    cloneVariables
  }).finally(() => {
    if (streamCheckTimer) {
      clearInterval(streamCheckTimer);
@@ -203,7 +208,8 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
    usageId,
    concatUsage,
    runningUserInfo: { teamId },
-    mcpClientMemory
+    mcpClientMemory,
+    cloneVariables
  } = data;

  // Over max depth
@@ -225,6 +231,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
      [DispatchNodeResponseKeyEnum.toolResponses]: null,
      [DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
        variables,
+        cloneVariables,
        removeObj: externalProvider.externalWorkflowVariables,
        userVariablesConfigs: data.chatConfig?.variables
      }),
@@ -1057,6 +1064,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
    [DispatchNodeResponseKeyEnum.toolResponses]: workflowQueue.toolRunResponse,
    [DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
      variables,
+      cloneVariables,
      removeObj: externalProvider.externalWorkflowVariables,
      userVariablesConfigs: data.chatConfig?.variables
    }),
@@ -1092,6 +1100,15 @@ const getSystemVariables = async ({
      const actualValue = anyValueDecrypt(val);
      variablesMap[item.key] = valueTypeFormat(actualValue, item.valueType);
    }
+    //  文件类型全局变量，签发成 string[] 格式
+    else if (item.type === VariableInputEnum.file) {
+      const vars = await presignVariablesFileUrls({
+        variables,
+        variableConfig: [item]
+      });
+
+      variablesMap[item.key] = vars?.[item.key]?.map((item: any) => item.url);
+    }
    // API
    else if (variables[item.label] !== undefined) {
      variablesMap[item.key] = valueTypeFormat(variables[item.label], item.valueType);
@@ -14,7 +14,7 @@ type Response = DispatchNodeResultType<{
  [NodeOutputKeyEnum.userFiles]: string[];
 }>;

-export const dispatchWorkflowStart = (props: Record<string, any>): Response => {
+export const dispatchWorkflowStart = async (props: Record<string, any>): Promise<Response> => {
  const {
    query,
    variables,
@@ -8,6 +8,8 @@ import type {
 } from '@fastgpt/global/core/workflow/runtime/type';
 import type { UserInputFormItemType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
 import { addLog } from '../../../../common/system/log';
+import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
+import { anyValueDecrypt } from '../../../../common/secret/utils';

 type Props = ModuleDispatchProps<{
  [NodeInputKeyEnum.description]: string;
@@ -47,7 +49,7 @@ export const dispatchFormInput = async (props: Props): Promise<FormInputResponse
  node.isEntry = false;

  const { text } = chatValue2RuntimePrompt(query);
-  const userInputVal = (() => {
+  const rawUserInputVal: Record<string, any> = (() => {
    try {
      return JSON.parse(text);
    } catch (error) {
@@ -56,6 +58,32 @@ export const dispatchFormInput = async (props: Props): Promise<FormInputResponse
    }
  })();

+  const userInputVal = Object.entries(rawUserInputVal).reduce(
+    (acc, [key, value]) => {
+      const inputConfig = userInputForms.find((form) => form.key === key);
+
+      if (inputConfig?.type === FlowNodeInputTypeEnum.password) {
+        acc[key] = anyValueDecrypt(value);
+      } else if (inputConfig?.type === FlowNodeInputTypeEnum.fileSelect) {
+        if (Array.isArray(value)) {
+          acc[key] = value.map((file: any) => {
+            if (typeof file === 'object' && file.url) {
+              return file.url;
+            }
+            return file;
+          });
+        } else {
+          acc[key] = value;
+        }
+      } else {
+        acc[key] = value;
+      }
+
+      return acc;
+    },
+    {} as Record<string, any>
+  );
+
  return {
    data: {
      ...userInputVal,
@@ -7,15 +7,21 @@ import axios from 'axios';
 import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
-import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
+import { readS3FileContentByBuffer } from '../../../../common/file/read/utils';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { type ChatItemType, type UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { addLog } from '../../../../common/system/log';
 import { addRawTextBuffer, getRawTextBuffer } from '../../../../common/buffer/rawText/controller';
-import { addMinutes } from 'date-fns';
+import { addDays, addMinutes } from 'date-fns';
 import { getNodeErrResponse } from '../utils';
 import { isInternalAddress } from '../../../../common/system/utils';
+import { replaceDatasetQuoteTextWithJWT } from '../../../dataset/utils';
+import { getFileS3Key } from '../../../../common/s3/utils';
+import { S3ChatSource } from '../../../../common/s3/sources/chat';
+import path from 'path';
+import { S3Buckets } from '../../../../common/s3/constants';
+import { S3Sources } from '../../../../common/s3/type';

 type Props = ModuleDispatchProps<{
  [NodeInputKeyEnum.fileUrlList]: string[];
@@ -152,11 +158,9 @@ export const getFileContentFromLinks = async ({
    .map((url) => {
      try {
        // Check is system upload file
-        if (url.startsWith('/') || (requestOrigin && url.startsWith(requestOrigin))) {
-          //  Remove the origin(Make intranet requests directly)
-          if (requestOrigin && url.startsWith(requestOrigin)) {
-            url = url.replace(requestOrigin, '');
-          }
+        const parsedURL = new URL(url, 'http://localhost:3000');
+        if (requestOrigin && parsedURL.origin === requestOrigin) {
+          url = url.replace(requestOrigin, '');
        }

        return url;
@@ -185,6 +189,7 @@ export const getFileContentFromLinks = async ({
          if (isInternalAddress(url)) {
            return Promise.reject('Url is invalid');
          }
+
          // Get file buffer data
          const response = await axios.get(url, {
            baseURL: serverRequestBaseUrl,
@@ -193,21 +198,39 @@ export const getFileContentFromLinks = async ({

          const buffer = Buffer.from(response.data, 'binary');

+          const urlObj = new URL(url, 'http://localhost:3000');
+          const isChatExternalUrl = urlObj.pathname.startsWith(
+            `/${S3Buckets.private}/${S3Sources.chat}/`
+          );
+
          // Get file name
-          const filename = (() => {
+          const { filename, extension, imageParsePrefix } = (() => {
            const contentDisposition = response.headers['content-disposition'];
            if (contentDisposition) {
              const filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
              const matches = filenameRegex.exec(contentDisposition);
              if (matches != null && matches[1]) {
-                return decodeURIComponent(matches[1].replace(/['"]/g, ''));
+                const filename = decodeURIComponent(matches[1].replace(/['"]/g, ''));
+                return {
+                  filename,
+                  extension: path.extname(filename).replace('.', ''),
+                  imageParsePrefix: `` // TODO: 需要根据是否是聊天对话里面的外部链接来决定
+                };
              }
            }

-            return url;
+            if (isChatExternalUrl) {
+              const filename = urlObj.pathname.split('/').pop() || 'file';
+              const extension = path.extname(filename).replace('.', '');
+              return {
+                filename,
+                extension,
+                imageParsePrefix: getFileS3Key.temp({ teamId, filename }).fileParsedPrefix
+              };
+            }
+
+            return S3ChatSource.parseChatUrl(url);
          })();
-          // Extension
-          const extension = parseFileExtensionFromUrl(filename);

          // Get encoding
          const encoding = (() => {
@@ -223,8 +246,7 @@ export const getFileContentFromLinks = async ({
            return detectFileEncoding(buffer);
          })();

-          // Read file
-          const { rawText } = await readRawContentByFileBuffer({
+          const { rawText } = await readS3FileContentByBuffer({
            extension,
            teamId,
            tmbId,
@@ -232,18 +254,27 @@ export const getFileContentFromLinks = async ({
            encoding,
            customPdfParse,
            getFormatText: true,
+            imageKeyOptions: imageParsePrefix
+              ? {
+                  prefix: imageParsePrefix,
+                  // 聊天对话里面上传的外部链接，解析出来的图片过期时间设置为1天，而且是存储在临时文件夹的
+                  expiredTime: isChatExternalUrl ? addDays(new Date(), 1) : undefined
+                }
+              : undefined,
            usageId
          });

+          const replacedText = replaceDatasetQuoteTextWithJWT(rawText, addDays(new Date(), 90));
+
          // Add to buffer
          addRawTextBuffer({
            sourceId: url,
            sourceName: filename,
-            text: rawText,
+            text: replacedText,
            expiredTime: addMinutes(new Date(), 20)
          });

-          return formatResponseObject({ filename, url, content: rawText });
+          return formatResponseObject({ filename, url, content: replacedText });
        } catch (error) {
          return formatResponseObject({
            filename: '',
@@ -14,6 +14,8 @@ import { type ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/
 import { runtimeSystemVar2StoreType } from '../utils';
 import { isValidReferenceValue } from '@fastgpt/global/core/workflow/utils';
 import { valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils';
+import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
+import { z } from 'zod';

 type Props = ModuleDispatchProps<{
  [NodeInputKeyEnum.updateList]: TUpdateListItem[];
@@ -25,6 +27,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
    chatConfig,
    params,
    variables,
+    cloneVariables,
    runtimeNodes,
    workflowStreamResponse,
    externalProvider,
@@ -33,6 +36,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>

  const { updateList } = params;
  const nodeIds = runtimeNodes.map((node) => node.nodeId);
+  const urlSchema = z.string().url();

  const result = updateList.map((item) => {
    const variable = item.variable;
@@ -62,11 +66,19 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>

        return valueTypeFormat(val, item.valueType);
      } else {
-        return getReferenceVariableValue({
+        const val = getReferenceVariableValue({
          value: item.value,
          variables,
          nodes: runtimeNodes
        });
+
+        if (
+          Array.isArray(val) &&
+          val.every((url) => typeof url === 'string' && urlSchema.safeParse(url).success)
+        ) {
+          return val.map((url) => parseUrlToFileType(url)).filter(Boolean);
+        }
+        return val;
      }
    })();

@@ -94,6 +106,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
      event: SseResponseEventEnum.updateVariables,
      data: runtimeSystemVar2StoreType({
        variables,
+        cloneVariables,
        removeObj: externalProvider.externalWorkflowVariables,
        userVariablesConfigs: chatConfig?.variables
      })
@@ -1,6 +1,6 @@
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
-import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
+import type { ChatItemType, UserChatItemFileItemType } from '@fastgpt/global/core/chat/type.d';
 import { NodeOutputKeyEnum, VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
 import type { VariableItemType } from '@fastgpt/global/core/app/type';
 import { encryptSecret } from '../../../common/secret/aes256gcm';
@@ -119,10 +119,12 @@ export const checkQuoteQAValue = (quoteQA?: SearchDataResponseItemType[]) => {
 /* remove system variable */
 export const runtimeSystemVar2StoreType = ({
  variables,
+  cloneVariables,
  removeObj = {},
  userVariablesConfigs = []
 }: {
  variables: Record<string, any>;
+  cloneVariables: Record<string, any>;
  removeObj?: Record<string, string>;
  userVariablesConfigs?: VariableItemType[];
 }) => {
@@ -152,6 +154,10 @@ export const runtimeSystemVar2StoreType = ({
        };
      }
    }
+    // Remove URL from file variables
+    else if (item.type === VariableInputEnum.file) {
+      copyVariables[item.key] = cloneVariables[item.key];
+    }
  });

  return copyVariables;