* refactor: fastgpt object storage & global proxy (#6155)

* feat: migrate to fastgpt storage sdk

* chore: rename env variable

* chore: move to sdk dir

* docs: object storage

* CHORE

* chore: storage mocks

* chore: update docker-compose

* fix: global proxy agent

* fix: update COS proxy

* refactor: use fetch instead of http.request

* fix: axios request base url

* fix: axios proxy request behavior

* fix: bumps axios

* fix: patch axios for proxy

* fix: replace axios with proxied axios

* fix: upload txt file encoding

* clean code

* fix: use "minio" for minio adapter (#6205)

* fix: use minio client to delete files when using minio vendor (#6206)

* doc

* feat: filter citations and add response button control (#6170)

* feat: filter citations and add response button control

* i18n

* fix

* fix test

* perf: chat api code

* fix: workflow edge overlap and auto-align in folded loop nodes (#6204)

* fix: workflow edge overlap and auto-align in folded loop nodes

* sort

* fix

* fix edge

* fix icon

* perf: s3 file name

* perf: admin get app api

* perf: catch user error

* fix: refactor useOrg hook to use debounced search key (#6180)

* chore: comment minio adapter (#6207)

* chore: filename with suffix random id

* perf: s3 storage code

* fix: encode filename when copy object

---------

Co-authored-by: archer <545436317@qq.com>

* fix: node card link

* json

* perf: chat index;

* index

* chat item soft delete (#6216)

* chat item soft delete

* temp

* fix

* remove code

* perf: delete chat item

---------

Co-authored-by: archer <545436317@qq.com>

* feat: select wheather filter sensitive info when export apps (#6222)

* fix some bugs (#6210)

* fix v4.14.5 bugs

* type

* fix

* fix

* custom feedback

* fix

* code

* fix

* remove invalid function

---------

Co-authored-by: archer <545436317@qq.com>

* perf: test

* fix file default local upload (#6223)

* docs: improve object storage introduction (#6224)

* doc

---------

Co-authored-by: roy <whoeverimf5@gmail.com>
Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
This commit is contained in:
Archer
2026-01-09 18:25:02 +08:00
committed by GitHub
parent 56ec86e2fe
commit c93c3937e1
281 changed files with 11562 additions and 2527 deletions
@@ -131,7 +131,8 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
assistantResponses,
runTimes,
workflowInteractiveResponse,
system_memories
system_memories,
customFeedbacks
} = await runWorkflow({
...props,
usageId: undefined,
@@ -205,7 +206,8 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
totalPoints: usagePoints
}
],
[DispatchNodeResponseKeyEnum.toolResponses]: text
[DispatchNodeResponseKeyEnum.toolResponses]: text,
[DispatchNodeResponseKeyEnum.customFeedbacks]: customFeedbacks
};
} catch (error) {
return getNodeErrResponse({ error });
@@ -22,6 +22,7 @@ import { getNodeErrResponse } from '../utils';
import { splitCombineToolId } from '@fastgpt/global/core/app/tool/utils';
import { getAppVersionById } from '../../../../core/app/version/controller';
import { runHTTPTool } from '../../../app/http';
import { getS3ChatSource } from '../../../../common/s3/sources/chat';
type SystemInputConfigType = {
type: SystemToolSecretInputTypeEnum;
@@ -53,6 +54,12 @@ export const dispatchRunTool = async (props: RunToolProps): Promise<RunToolRespo
node: { name, avatar, toolConfig, version, catchError }
} = props;
const {
uid: uId,
chatId = '',
runningAppInfo: { id: appId }
} = props;
const systemToolId = toolConfig?.systemTool?.toolId;
let toolInput: Record<string, any> = {};
@@ -109,7 +116,8 @@ export const dispatchRunTool = async (props: RunToolProps): Promise<RunToolRespo
},
tool: {
id: formatToolId,
version: version || tool.versionList?.[0]?.value || ''
version: version || tool.versionList?.[0]?.value || '',
prefix: getS3ChatSource().getToolFilePrefix({ appId, chatId, uId })
},
time: variables.cTime
},
@@ -24,7 +24,7 @@ import type {
} from '@fastgpt/global/core/workflow/runtime/type';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d';
import { getErrText, UserError } from '@fastgpt/global/common/error/utils';
import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
import { ChatItemValueTypeEnum, ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils';
import {
checkNodeRunStatus,
@@ -57,13 +57,12 @@ import { addPreviewUrlToChatItems, presignVariablesFileUrls } from '../../chat/u
import type { MCPClient } from '../../app/mcp';
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { i18nT } from '../../../../web/i18n/utils';
import { clone } from 'lodash';
import { validateFileUrlDomain } from '../../../common/security/fileUrlValidator';
import { delAgentRuntimeStopSign, shouldWorkflowStop } from './workflowStatus';
type Props = Omit<
ChatDispatchProps,
'checkIsStopping' | 'workflowDispatchDeep' | 'timezone' | 'externalProvider' | 'cloneVariables'
'checkIsStopping' | 'workflowDispatchDeep' | 'timezone' | 'externalProvider'
> & {
runtimeNodes: RuntimeNodeItemType[];
runtimeEdges: RuntimeEdgeItemType[];
@@ -139,10 +138,11 @@ export async function dispatchWorkFlow({
// Add preview url to query
...query.map(async (item) => {
if (item.type !== ChatItemValueTypeEnum.file || !item.file?.key) return;
item.file.url = await getS3ChatSource().createGetChatFileURL({
const { url } = await getS3ChatSource().createGetChatFileURL({
key: item.file.key,
external: true
});
item.file.url = url;
}),
// Remove stopping sign
delAgentRuntimeStopSign({
@@ -181,7 +181,6 @@ export async function dispatchWorkFlow({
}
// Get default variables
const cloneVariables = clone(data.variables);
const defaultVariables = {
...externalProvider.externalWorkflowVariables,
...(await getSystemVariables({
@@ -227,8 +226,7 @@ export async function dispatchWorkFlow({
workflowDispatchDeep: 0,
usageId: newUsageId,
concatUsage,
mcpClientMemory,
cloneVariables
mcpClientMemory
}).finally(async () => {
if (streamCheckTimer) {
clearInterval(streamCheckTimer);
@@ -272,8 +270,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
usageId,
concatUsage,
runningUserInfo: { teamId },
mcpClientMemory,
cloneVariables
mcpClientMemory
} = data;
// Over max depth
@@ -295,7 +292,6 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
[DispatchNodeResponseKeyEnum.toolResponses]: null,
[DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
variables,
cloneVariables,
removeObj: externalProvider.externalWorkflowVariables,
userVariablesConfigs: data.chatConfig?.variables
}),
@@ -342,6 +338,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
}
| undefined;
system_memories: Record<string, any> = {}; // Workflow node memories
customFeedbackList: string[] = []; // Custom feedbacks collected from nodes
// Debug
debugNextStepRunNodes: RuntimeNodeItemType[] = []; // 记录 Debug 模式下,下一个阶段需要执行的节点。
@@ -719,7 +716,8 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
assistantResponses,
rewriteHistories,
runTimes = 1,
system_memories: newMemories
system_memories: newMemories,
customFeedbacks
}: NodeResponseCompleteType) => {
// Add run times
this.workflowRunTimes += runTimes;
@@ -736,6 +734,11 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
this.chatResponses.push(responseData);
}
// Collect custom feedbacks
if (customFeedbacks && Array.isArray(customFeedbacks)) {
this.customFeedbackList = this.customFeedbackList.concat(customFeedbacks);
}
// Push usage in real time. Avoid a workflow usage a large number of points
if (nodeDispatchUsages) {
if (usageId) {
@@ -1119,7 +1122,6 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
[DispatchNodeResponseKeyEnum.toolResponses]: workflowQueue.toolRunResponse,
[DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
variables,
cloneVariables,
removeObj: externalProvider.externalWorkflowVariables,
userVariablesConfigs: data.chatConfig?.variables
}),
@@ -1127,6 +1129,8 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
Object.keys(workflowQueue.system_memories).length > 0
? workflowQueue.system_memories
: undefined,
[DispatchNodeResponseKeyEnum.customFeedbacks]:
workflowQueue.customFeedbackList.length > 0 ? workflowQueue.customFeedbackList : undefined,
durationSeconds
};
};
@@ -51,6 +51,7 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
const outputValueArr = interactiveData ? interactiveData.loopResult : [];
const loopResponseDetail: ChatHistoryItemResType[] = [];
let assistantResponses: AIChatItemValueItemType[] = [];
const customFeedbacks: string[] = [];
let totalPoints = 0;
let newVariables: Record<string, any> = props.variables;
let interactiveResponse: WorkflowInteractiveResponseType | undefined = undefined;
@@ -116,6 +117,11 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
assistantResponses.push(...response.assistantResponses);
totalPoints += response.flowUsages.reduce((acc, usage) => acc + usage.totalPoints, 0);
// Collect custom feedbacks
if (response[DispatchNodeResponseKeyEnum.customFeedbacks]) {
customFeedbacks.push(...response[DispatchNodeResponseKeyEnum.customFeedbacks]);
}
// Concat new variables
newVariables = {
...newVariables,
@@ -163,6 +169,8 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
}
]
: [],
[DispatchNodeResponseKeyEnum.newVariables]: newVariables
[DispatchNodeResponseKeyEnum.newVariables]: newVariables,
[DispatchNodeResponseKeyEnum.customFeedbacks]:
customFeedbacks.length > 0 ? customFeedbacks : undefined
};
};
@@ -132,35 +132,41 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
appId: String(plugin.id),
...(externalProvider ? externalProvider.externalWorkflowVariables : {})
};
const { flowResponses, flowUsages, assistantResponses, runTimes, system_memories } =
await runWorkflow({
...props,
usageId: undefined,
// Rewrite stream mode
...(system_forbid_stream
? {
stream: false,
workflowStreamResponse: undefined
}
: {}),
runningAppInfo: {
id: String(plugin.id),
name: plugin.name,
// 如果系统插件有 teamId 和 tmbId,则使用系统插件的 teamId 和 tmbId(管理员指定了插件作为系统插件)
teamId: plugin.teamId || runningAppInfo.teamId,
tmbId: plugin.tmbId || runningAppInfo.tmbId,
isChildApp: true
},
const {
flowResponses,
flowUsages,
assistantResponses,
runTimes,
system_memories,
[DispatchNodeResponseKeyEnum.customFeedbacks]: customFeedbacks
} = await runWorkflow({
...props,
usageId: undefined,
// Rewrite stream mode
...(system_forbid_stream
? {
stream: false,
workflowStreamResponse: undefined
}
: {}),
runningAppInfo: {
id: String(plugin.id),
name: plugin.name,
// 如果系统插件有 teamId 和 tmbId,则使用系统插件的 teamId 和 tmbId(管理员指定了插件作为系统插件)
teamId: plugin.teamId || runningAppInfo.teamId,
tmbId: plugin.tmbId || runningAppInfo.tmbId,
isChildApp: true
},
variables: runtimeVariables,
query: serverGetWorkflowToolRunUserQuery({
pluginInputs: getWorkflowToolInputsFromStoreNodes(plugin.nodes),
variables: runtimeVariables,
query: serverGetWorkflowToolRunUserQuery({
pluginInputs: getWorkflowToolInputsFromStoreNodes(plugin.nodes),
variables: runtimeVariables,
files
}).value,
chatConfig: {},
runtimeNodes,
runtimeEdges: storeEdges2RuntimeEdges(plugin.edges)
});
files
}).value,
chatConfig: {},
runtimeNodes,
runtimeEdges: storeEdges2RuntimeEdges(plugin.edges)
});
const output = flowResponses.find((item) => item.moduleType === FlowNodeTypeEnum.pluginOutput);
const usagePoints = await computedAppToolUsage({
@@ -200,7 +206,8 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
acc[key] = output.pluginOutput![key];
return acc;
}, {})
: null
: null,
[DispatchNodeResponseKeyEnum.customFeedbacks]: customFeedbacks
};
} catch (error) {
return getNodeErrResponse({
@@ -42,11 +42,12 @@ export const dispatchPluginInput = async (
for (let i = 0; i < val.length; i++) {
const fileItem = val[i];
if (fileItem.key && !fileItem.url) {
val[i].url = await getS3ChatSource().createGetChatFileURL({
const { url } = await getS3ChatSource().createGetChatFileURL({
key: fileItem.key,
external: true,
expiredHours: 1
});
val[i].url = url;
}
}
params[key] = val.map((item) => item.url);
@@ -1,7 +1,7 @@
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import axios from 'axios';
import { axios } from '../../../../common/api/axios';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { SandboxCodeTypeEnum } from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
import { getErrText } from '@fastgpt/global/common/error/utils';
@@ -1,12 +1,7 @@
import {
DispatchNodeResponseKeyEnum,
SseResponseEventEnum
} from '@fastgpt/global/core/workflow/runtime/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import { addCustomFeedbacks } from '../../../chat/controller';
import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.textareaInput]: string;
@@ -15,37 +10,13 @@ type Response = DispatchNodeResultType<{}>;
export const dispatchCustomFeedback = (props: Record<string, any>): Response => {
const {
runningAppInfo: { id: appId },
chatId,
responseChatItemId: dataId,
stream,
workflowStreamResponse,
params: { system_textareaInput: feedbackText = '' }
} = props as Props;
setTimeout(() => {
addCustomFeedbacks({
appId,
chatId,
dataId,
feedbacks: [feedbackText]
});
}, 60000);
if (stream) {
if (!chatId || !dataId) {
workflowStreamResponse?.({
event: SseResponseEventEnum.fastAnswer,
data: textAdaptGptResponse({
text: `\n\n**自定义反馈成功: (仅调试模式下展示该内容)**: "${feedbackText}"\n\n`
})
});
}
}
return {
[DispatchNodeResponseKeyEnum.nodeResponse]: {
textOutput: feedbackText
}
},
[DispatchNodeResponseKeyEnum.customFeedbacks]: [feedbackText]
};
};
@@ -8,7 +8,6 @@ import {
WorkflowIOValueTypeEnum
} from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import axios from 'axios';
import { valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils';
import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import type {
@@ -29,6 +28,7 @@ import { SERVICE_LOCAL_HOST } from '../../../../common/system/tools';
import { formatHttpError } from '../utils';
import { isInternalAddress } from '../../../../common/system/utils';
import { serviceRequestMaxContentLength } from '../../../../common/system/constants';
import { axios } from '../../../../common/api/axios';
type PropsArrType = {
key: string;
@@ -3,7 +3,7 @@ import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import axios from 'axios';
import { axios } from '../../../../common/api/axios';
import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
@@ -1,7 +1,7 @@
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import axios from 'axios';
import { axios } from '../../../../common/api/axios';
import { valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils';
import { SERVICE_LOCAL_HOST } from '../../../../common/system/tools';
import { addLog } from '../../../../common/system/log';
@@ -14,8 +14,6 @@ import { type ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/
import { runtimeSystemVar2StoreType } from '../utils';
import { isValidReferenceValue } from '@fastgpt/global/core/workflow/utils';
import { valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { z } from 'zod';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.updateList]: TUpdateListItem[];
@@ -27,7 +25,6 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
chatConfig,
params,
variables,
cloneVariables,
runtimeNodes,
workflowStreamResponse,
externalProvider,
@@ -36,7 +33,6 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
const { updateList } = params;
const nodeIds = runtimeNodes.map((node) => node.nodeId);
const urlSchema = z.string().url();
const result = updateList.map((item) => {
const variable = item.variable;
@@ -66,19 +62,11 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
return valueTypeFormat(val, item.valueType);
} else {
const val = getReferenceVariableValue({
return getReferenceVariableValue({
value: item.value,
variables,
nodes: runtimeNodes
});
if (
Array.isArray(val) &&
val.every((url) => typeof url === 'string' && urlSchema.safeParse(url).success)
) {
return val.map((url) => parseUrlToFileType(url)).filter(Boolean);
}
return val;
}
})();
@@ -106,7 +94,6 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
event: SseResponseEventEnum.updateVariables,
data: runtimeSystemVar2StoreType({
variables,
cloneVariables,
removeObj: externalProvider.externalWorkflowVariables,
userVariablesConfigs: chatConfig?.variables
})
+1
View File
@@ -37,6 +37,7 @@ export type DispatchFlowResponse = {
[DispatchNodeResponseKeyEnum.assistantResponses]: AIChatItemValueItemType[];
[DispatchNodeResponseKeyEnum.runTimes]: number;
[DispatchNodeResponseKeyEnum.memories]?: Record<string, any>;
[DispatchNodeResponseKeyEnum.customFeedbacks]?: string[];
[DispatchNodeResponseKeyEnum.newVariables]: Record<string, string>;
durationSeconds: number;
};
@@ -1,9 +1,11 @@
import path from 'path';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { ChatRoleEnum, ChatFileTypeEnum } from '@fastgpt/global/core/chat/constants';
import type { ChatItemType, UserChatItemFileItemType } from '@fastgpt/global/core/chat/type.d';
import { NodeOutputKeyEnum, VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
import type { VariableItemType } from '@fastgpt/global/core/app/type';
import { encryptSecret } from '../../../common/secret/aes256gcm';
import { imageFileType } from '@fastgpt/global/common/file/constants';
import {
type RuntimeEdgeItemType,
type RuntimeNodeItemType,
@@ -120,12 +122,10 @@ export const checkQuoteQAValue = (quoteQA?: SearchDataResponseItemType[]) => {
/* remove system variable */
export const runtimeSystemVar2StoreType = ({
variables,
cloneVariables,
removeObj = {},
userVariablesConfigs = []
}: {
variables: Record<string, any>;
cloneVariables: Record<string, any>;
removeObj?: Record<string, string>;
userVariablesConfigs?: VariableItemType[];
}) => {
@@ -155,9 +155,33 @@ export const runtimeSystemVar2StoreType = ({
};
}
}
// Remove URL from file variables
// Handle file variables
else if (item.type === VariableInputEnum.file) {
copyVariables[item.key] = cloneVariables[item.key];
const currentValue = copyVariables[item.key];
copyVariables[item.key] = currentValue
.map((url: string) => {
try {
const urlObj = new URL(url);
// Extract key: remove bucket prefix (e.g., "/fastgpt-private/")
const key = decodeURIComponent(urlObj.pathname.replace(/^\/[^/]+\//, ''));
const filename = path.basename(key) || 'file';
const extname = path.extname(key).toLowerCase(); // includes the dot, e.g., ".jpg"
// Check if it's an image type
const isImage = extname && imageFileType.includes(extname);
return {
id: path.basename(key, path.extname(key)), // filename without extension
key,
name: filename,
type: isImage ? ChatFileTypeEnum.image : ChatFileTypeEnum.file
};
} catch {
return null;
}
})
.filter((file: any) => file !== null);
}
});