V4.13.0 features (#5693)

* feat: concat usage code (#5657)

* feat: dataset parse queue (#5661)

* feat: chat usage concat (#5669)

* perf: search test usage

* feat: chat usage concat

* fix: ts

* fix: ts

* feat: chat node response store (#5675)

* feat: chat node response store

* limit export

* test

* add ai generate node (#5506)

* add node copilot

* apply code

* update dynamic input & output

* add code test

* usage

* dynamic input border render

* optimize input & output

* optimize code

* update style

* change card to popover

* prompt editor basic

* prompt editor

* handle key down

* update prompt

* merge

* fix

* fix

* fix

* perf: workflow performance (#5677)

* feat: chat node response store

* limit export

* perf: workflow performance

* remove log

* fix: app template get duplicate (#5682)

* fix: dynamic input lock & code param (#5680)

* fix: dynamic input lock & code param

* fix

* fix

* feat: multi node data sync & system tool hot-swapping (#5575)

* Enhance file upload functionality and system tool integration (#5257)

* Enhance file upload functionality and system tool integration

* Add supplementary documents and optimize the upload interface

* Refactor file plugin types and update upload configurations

* Refactor MinIO configuration variables and clean up API plugin handlers for improved readability and consistency

* File name change

* Refactor SystemTools component layout

* fix i18n

* fix

* fix

* fix

* optimize app logs sort (#5310)

* log keys config modal

* multiple select

* api

* fontsize

* code

* chatid

* fix build

* fix

* fix component

* change name

* log keys config

* fix

* delete unused

* fix

* chore: minio service class rewrite

* chore: s3 plugin upload

* feat: system global cache with multi node sync feature

* feat: cache

* chore: move images

* docs: update & remove useless code

* chore: resolve merge conflicts

* chore: adjust the code

* chore: adjust

* deps: upgrade @fastgpt-sdk/plugin to 0.1.17

* perf(s3): s3 config

* fix: cache syncKey refresh

* fix: update @fastgpt-sdk/plugin to v0.1.18 removing mongo definition for fixing vitest

* chore: adjust

---------

Co-authored-by: Ctrlz <143257420+ctrlz526@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Archer <545436317@qq.com>

* perf: s3 api code

* fix: toolbox empty when second open modal

* feat: http tool set (#5599)

* feat: http toolSet manual create front end

* feat: http toolSet manual create i18n

* feat: http toolSet manual create back end

* feat: auth, as tool param, adapt mcp

* fix: delete unused httpPlugin

* fix: delete FlowNodeTypeEnum.httpPlugin

* fix: AppTypeEnum include httpToolSet and httpPlugin

* fix

* delete console

* fix

* output schema

* fix

* fix bg

* fix base url

* fix

---------

Co-authored-by: heheer <zhiyu44@qq.com>

* feat: app count

* perf: type check

* feat: catch error

* perf: plugin hot-swapping (#5688)

* perf: plugin hot-swapping

* chore: adjust code

* perf: cite data auth

* fix http toolset (#5689)

* temp

* fix http tool set

* fix

* template author hide

* dynamic IO ui

* fix: auth test

* fix dynamic input & output (#5690)

Co-authored-by: Archer <545436317@qq.com>

* fix: dynamic output id

* doc

* feat: model permission (#5666)

* feat(permission): model permission definition & api

* chore: support update model's collaborators

* feat: remove unauthedmodel when paste and import

* fix: type error

* fix: test setup global model list

* fix: http tool api

* chore: update fastgpt-sdk version

* chore: remove useless code

* chore: myModelList cache

* perf: user who is not manager can not configure model permission (FE)

* perf: model => Set

* feat: getMyModels moved to opensource code; cache the myModelList

* fix: type error

* fix dynamic input reference select type (#5694)

* remove unique index

* read file usage

* perf: connection error

* fix: abort token count

* fix: debug usage concat

* fix: immer clone object

* fix: immer clone object

* perf: throw error when error chat

* update audit i18n

* fix: 修复识别pptx文件后,返回内容顺序错乱问题 (#5696)

* fix: pptx sort error

* fix prompt editor (#5695)

* fix prompt editor

* fix

* fix: redis cache prefix (#5697)

* fix: redis cache prefix

* fix: cache

* fix: get model collaborator by model.model

* feat: hint for model per

* rename bucket name

* model ui

* doc

* doc

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: Ctrlz <143257420+ctrlz526@users.noreply.github.com>
Co-authored-by: Zeng Qingwen <143274079+fishwww-ww@users.noreply.github.com>
Co-authored-by: heheer <zhiyu44@qq.com>
Co-authored-by: Deepturn <33342819+Deepturn@users.noreply.github.com>
This commit is contained in:
Archer
2025-09-24 22:40:31 +08:00
committed by GitHub
parent 9f8f8dd3de
commit 051455238c
249 changed files with 8797 additions and 4202 deletions

View File

@@ -63,6 +63,7 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
...props,
runningAppInfo: {
id: String(appData._id),
name: appData.name,
teamId: String(appData.teamId),
tmbId: String(appData.tmbId)
},

View File

@@ -48,6 +48,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
lastInteractive,
runningUserInfo,
externalProvider,
usageId,
params: {
model,
systemPrompt,
@@ -117,7 +118,8 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
customPdfParse: chatConfig?.fileSelectConfig?.customPdfParse,
fileLinks,
inputFiles: globalFiles,
hasReadFilesTool
hasReadFilesTool,
usageId
});
const concatenateSystemPrompt = [
@@ -273,7 +275,8 @@ const getMultiInput = async ({
maxFiles,
customPdfParse,
inputFiles,
hasReadFilesTool
hasReadFilesTool,
usageId
}: {
runningUserInfo: ChatDispatchProps['runningUserInfo'];
histories: ChatItemType[];
@@ -283,6 +286,7 @@ const getMultiInput = async ({
customPdfParse?: boolean;
inputFiles: UserChatItemValueItemType['file'][];
hasReadFilesTool: boolean;
usageId?: string;
}) => {
// Not file quote
if (!fileLinks || hasReadFilesTool) {
@@ -309,6 +313,7 @@ const getMultiInput = async ({
requestOrigin,
maxFiles,
customPdfParse,
usageId,
teamId: runningUserInfo.teamId,
tmbId: runningUserInfo.tmbId
});

View File

@@ -74,6 +74,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
runningUserInfo,
workflowStreamResponse,
chatConfig,
usageId,
params: {
model,
temperature,
@@ -131,6 +132,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
requestOrigin,
maxFiles: chatConfig?.fileSelectConfig?.maxFiles || 20,
customPdfParse: chatConfig?.fileSelectConfig?.customPdfParse,
usageId,
runningUserInfo
})
]);
@@ -313,6 +315,7 @@ async function getMultiInput({
requestOrigin,
maxFiles,
customPdfParse,
usageId,
runningUserInfo
}: {
histories: ChatItemType[];
@@ -322,6 +325,7 @@ async function getMultiInput({
requestOrigin?: string;
maxFiles: number;
customPdfParse?: boolean;
usageId?: string;
runningUserInfo: ChatDispatchProps['runningUserInfo'];
}) {
// 旧版本适配====>
@@ -359,9 +363,10 @@ async function getMultiInput({
urls,
requestOrigin,
maxFiles,
customPdfParse,
teamId: runningUserInfo.teamId,
tmbId: runningUserInfo.tmbId
tmbId: runningUserInfo.tmbId,
customPdfParse,
usageId
});
return {

View File

@@ -20,7 +20,7 @@ import { authAppByTmbId } from '../../../../support/permission/app/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { getAppVersionById } from '../../../app/version/controller';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
import { getUserChatInfo } from '../../../../support/user/team/utils';
import { getRunningUserInfoByTmbId } from '../../../../support/user/team/utils';
type Props = ModuleDispatchProps<{
@@ -99,7 +99,7 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
// Rewrite children app variables
const systemVariables = filterSystemVariables(variables);
const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(appData.tmbId);
const { externalProvider } = await getUserChatInfo(appData.tmbId);
const childrenRunVariables = {
...systemVariables,
...childrenAppVariables,
@@ -144,6 +144,7 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
: {}),
runningAppInfo: {
id: String(appData._id),
name: appData.name,
teamId: String(appData.teamId),
tmbId: String(appData.tmbId),
isChildApp: true

View File

@@ -10,6 +10,7 @@ import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { MCPClient } from '../../../app/mcp';
import { getSecretValue } from '../../../../common/secret/utils';
import type { McpToolDataType } from '@fastgpt/global/core/app/mcpTools/type';
import type { HttpToolConfigType } from '@fastgpt/global/core/app/type';
import { APIRunSystemTool } from '../../../app/tool/api';
import { MongoSystemPlugin } from '../../../app/plugin/systemPluginSchema';
import { SystemToolInputTypeEnum } from '@fastgpt/global/core/app/systemTool/constants';
@@ -20,6 +21,7 @@ import { pushTrack } from '../../../../common/middle/tracks/utils';
import { getNodeErrResponse } from '../utils';
import { splitCombinePluginId } from '@fastgpt/global/core/app/plugin/utils';
import { getAppVersionById } from '../../../../core/app/version/controller';
import { runHTTPTool } from '../../../app/http';
type SystemInputConfigType = {
type: SystemToolInputTypeEnum;
@@ -34,7 +36,7 @@ type RunToolProps = ModuleDispatchProps<{
type RunToolResponse = DispatchNodeResultType<
{
[NodeOutputKeyEnum.rawResponse]?: any; // MCP Tool
[NodeOutputKeyEnum.rawResponse]?: any;
[key: string]: any;
},
Record<string, any>
@@ -214,6 +216,60 @@ export const dispatchRunTool = async (props: RunToolProps): Promise<RunToolRespo
},
[DispatchNodeResponseKeyEnum.toolResponses]: result
};
} else if (toolConfig?.httpTool?.toolId) {
const { pluginId } = splitCombinePluginId(toolConfig.httpTool.toolId);
const [parentId, toolSetName, toolName] = pluginId.split('/');
const toolset = await getAppVersionById({
appId: parentId,
versionId: version
});
const toolSetData = toolset.nodes[0].toolConfig?.httpToolSet;
if (!toolSetData || typeof toolSetData !== 'object') {
throw new Error('HTTP tool set not found');
}
const { headerSecret, baseUrl, toolList, customHeaders } = toolSetData;
const httpTool = toolList?.find((tool: HttpToolConfigType) => tool.name === toolName);
if (!httpTool) {
throw new Error(`HTTP tool ${toolName} not found`);
}
const { data, errorMsg } = await runHTTPTool({
baseUrl: baseUrl,
toolPath: httpTool.path,
method: httpTool.method,
params,
headerSecret,
customHeaders: customHeaders
? typeof customHeaders === 'string'
? JSON.parse(customHeaders)
: customHeaders
: undefined
});
if (errorMsg) {
if (catchError) {
return {
error: { [NodeOutputKeyEnum.errorText]: errorMsg },
[DispatchNodeResponseKeyEnum.nodeResponse]: {
toolRes: errorMsg,
moduleLogo: avatar
},
[DispatchNodeResponseKeyEnum.toolResponses]: errorMsg
};
}
throw new Error(errorMsg);
}
return {
data: { [NodeOutputKeyEnum.rawResponse]: data, ...(typeof data === 'object' ? data : {}) },
[DispatchNodeResponseKeyEnum.nodeResponse]: {
toolRes: data,
moduleLogo: avatar
},
[DispatchNodeResponseKeyEnum.toolResponses]: data
};
} else {
// mcp tool (old version compatible)
const { toolData, system_toolData, ...restParams } = params;

View File

@@ -15,7 +15,6 @@ import { type ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type
import { MongoDataset } from '../../../dataset/schema';
import { i18nT } from '../../../../../web/i18n/utils';
import { filterDatasetsByTmbId } from '../../../dataset/utils';
import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
import { getDatasetSearchToolResponsePrompt } from '../../../../../global/core/ai/prompt/dataset';
import { getNodeErrResponse } from '../utils';

View File

@@ -47,8 +47,13 @@ import { rewriteRuntimeWorkFlow, removeSystemVariable } from './utils';
import { getHandleId } from '@fastgpt/global/core/workflow/utils';
import { callbackMap } from './constants';
import { anyValueDecrypt } from '../../../common/secret/utils';
import { getUserChatInfo } from '../../../support/user/team/utils';
import { checkTeamAIPoints } from '../../../support/permission/teamLimit';
import type { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { createChatUsageRecord, pushChatItemUsage } from '../../../support/wallet/usage/controller';
import type { RequireOnlyOne } from '@fastgpt/global/common/type/utils';
type Props = Omit<ChatDispatchProps, 'workflowDispatchDeep'> & {
type Props = Omit<ChatDispatchProps, 'workflowDispatchDeep' | 'timezone' | 'externalProvider'> & {
runtimeNodes: RuntimeNodeItemType[];
runtimeEdges: RuntimeEdgeItemType[];
defaultSkipNodeQueue?: WorkflowDebugResponse['skipNodeQueue'];
@@ -61,8 +66,38 @@ type NodeResponseCompleteType = Omit<NodeResponseType, 'responseData'> & {
};
// Run workflow
export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowResponse> {
const { res, stream, externalProvider } = data;
type WorkflowUsageProps = RequireOnlyOne<{
usageSource: UsageSourceEnum;
concatUsage: (points: number) => any;
usageId: string;
}>;
export async function dispatchWorkFlow({
usageSource,
usageId,
concatUsage,
...data
}: Props & WorkflowUsageProps): Promise<DispatchFlowResponse> {
const { res, stream, runningUserInfo, runningAppInfo, lastInteractive } = data;
await checkTeamAIPoints(runningUserInfo.teamId);
const [{ timezone, externalProvider }, newUsageId] = await Promise.all([
getUserChatInfo(runningUserInfo.tmbId),
(() => {
if (lastInteractive?.usageId) {
return lastInteractive.usageId;
}
if (usageSource) {
return createChatUsageRecord({
appName: runningAppInfo.name,
appId: runningAppInfo.id,
teamId: runningUserInfo.teamId,
tmbId: runningUserInfo.tmbId,
source: usageSource
});
}
return usageId;
})()
]);
let streamCheckTimer: NodeJS.Timeout | null = null;
@@ -96,15 +131,22 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
// Get default variables
const defaultVariables = {
...externalProvider.externalWorkflowVariables,
...getSystemVariables(data)
...getSystemVariables({
...data,
timezone
})
};
// Init some props
return runWorkflow({
...data,
timezone,
externalProvider,
defaultSkipNodeQueue: data.lastInteractive?.skipNodeQueue || data.defaultSkipNodeQueue,
variables: defaultVariables,
workflowDispatchDeep: 0
workflowDispatchDeep: 0,
usageId: newUsageId,
concatUsage
}).finally(() => {
if (streamCheckTimer) {
clearInterval(streamCheckTimer);
@@ -116,6 +158,7 @@ type RunWorkflowProps = ChatDispatchProps & {
runtimeNodes: RuntimeNodeItemType[];
runtimeEdges: RuntimeEdgeItemType[];
defaultSkipNodeQueue?: WorkflowDebugResponse['skipNodeQueue'];
concatUsage?: (points: number) => any;
};
export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowResponse> => {
let {
@@ -129,7 +172,10 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
retainDatasetCite = true,
version = 'v1',
responseDetail = true,
responseAllData = true
responseAllData = true,
usageId,
concatUsage,
runningUserInfo: { teamId }
} = data;
// Over max depth
@@ -487,7 +533,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
data: responseAllData
? formatResponseData
: filterPublicNodeResponseData({
flowResponses: [formatResponseData],
nodeRespones: [formatResponseData],
responseDetail
})[0]
});
@@ -573,6 +619,17 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
}
if (nodeDispatchUsages) {
if (usageId) {
pushChatItemUsage({
teamId,
usageId,
nodeUsages: nodeDispatchUsages
});
}
if (concatUsage) {
concatUsage(nodeDispatchUsages.reduce((sum, item) => sum + (item.totalPoints || 0), 0));
}
this.chatNodeUsages = this.chatNodeUsages.concat(nodeDispatchUsages);
}
@@ -827,7 +884,8 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
...edge,
status: entryNodeIds.includes(edge.target) ? 'active' : edge.status
})),
nodeOutputs
nodeOutputs,
usageId
};
// Tool call, not need interactive response
@@ -945,7 +1003,9 @@ const getSystemVariables = ({
uid,
chatConfig,
variables
}: Props): SystemVariablesType => {
}: Props & {
timezone: string;
}): SystemVariablesType => {
// Get global variables(Label -> key; Key -> key)
const variablesConfig = chatConfig?.variables || [];

View File

@@ -21,7 +21,7 @@ import { getPluginRunUserQuery } from '@fastgpt/global/core/workflow/utils';
import type { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { getChildAppRuntimeById } from '../../../app/plugin/controller';
import { runWorkflow } from '../index';
import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
import { getUserChatInfo } from '../../../../support/user/team/utils';
import { dispatchRunTool } from '../child/runTool';
import type { PluginRuntimeType } from '@fastgpt/global/core/app/plugin/type';
@@ -111,7 +111,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
};
});
const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(runningAppInfo.tmbId);
const { externalProvider } = await getUserChatInfo(runningAppInfo.tmbId);
const runtimeVariables = {
...filterSystemVariables(props.variables),
appId: String(plugin.id),
@@ -129,6 +129,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
: {}),
runningAppInfo: {
id: String(plugin.id),
name: plugin.name,
// 如果系统插件有 teamId 和 tmbId则使用系统插件的 teamId 和 tmbId管理员指定了插件作为系统插件
teamId: plugin.teamId || runningAppInfo.teamId,
tmbId: plugin.tmbId || runningAppInfo.tmbId,

View File

@@ -52,7 +52,8 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
histories,
chatConfig,
node: { version },
params: { fileUrlList = [] }
params: { fileUrlList = [] },
usageId
} = props;
const maxFiles = chatConfig?.fileSelectConfig?.maxFiles || 20;
const customPdfParse = chatConfig?.fileSelectConfig?.customPdfParse || false;
@@ -68,7 +69,8 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
maxFiles,
teamId,
tmbId,
customPdfParse
customPdfParse,
usageId
});
return {
@@ -119,7 +121,8 @@ export const getFileContentFromLinks = async ({
maxFiles,
teamId,
tmbId,
customPdfParse
customPdfParse,
usageId
}: {
urls: string[];
requestOrigin?: string;
@@ -127,6 +130,7 @@ export const getFileContentFromLinks = async ({
teamId: string;
tmbId: string;
customPdfParse?: boolean;
usageId?: string;
}) => {
const parseUrlList = urls
// Remove invalid urls
@@ -225,7 +229,8 @@ export const getFileContentFromLinks = async ({
buffer,
encoding,
customPdfParse,
getFormatText: true
getFormatText: true,
usageId
});
// Add to buffer

View File

@@ -18,11 +18,13 @@ import {
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { type SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { getMCPToolRuntimeNode } from '@fastgpt/global/core/app/mcpTools/utils';
import { getHTTPToolRuntimeNode } from '@fastgpt/global/core/app/httpTools/utils';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { MongoApp } from '../../../core/app/schema';
import { getMCPChildren } from '../../../core/app/mcp';
import { getSystemToolRunTimeNodeFromSystemToolset } from '../utils';
import type { localeType } from '@fastgpt/global/common/i18n/type';
import type { HttpToolConfigType } from '@fastgpt/global/core/app/type';
export const getWorkflowResponseWrite = ({
res,
@@ -197,7 +199,8 @@ export const rewriteRuntimeWorkFlow = async ({
for (const toolSetNode of toolSetNodes) {
nodeIdsToRemove.add(toolSetNode.nodeId);
const systemToolId = toolSetNode.toolConfig?.systemToolSet?.toolId;
const mcpToolsetVal = toolSetNode.toolConfig?.mcpToolSet ?? toolSetNode.inputs[0].value;
const mcpToolsetVal = toolSetNode.toolConfig?.mcpToolSet ?? toolSetNode.inputs?.[0]?.value;
const httpToolsetVal = toolSetNode.toolConfig?.httpToolSet;
const incomingEdges = edges.filter((edge) => edge.target === toolSetNode.nodeId);
const pushEdges = (nodeId: string) => {
@@ -243,6 +246,22 @@ export const rewriteRuntimeWorkFlow = async ({
});
pushEdges(newToolNode.nodeId);
});
} else if (httpToolsetVal) {
const parentId = toolSetNode.pluginId || '';
httpToolsetVal.toolList.forEach((tool: HttpToolConfigType, index: number) => {
const newToolNode = getHTTPToolRuntimeNode({
tool: {
...tool,
name: `${toolSetNode.name}/${tool.name}`
},
nodeId: `${parentId}${index}`,
avatar: toolSetNode.avatar,
parentId
});
nodes.push(newToolNode);
pushEdges(newToolNode.nodeId);
});
}
}