perf: workflow runtime (#6562)

* perf: workflow runtime * perf: lable input and dispatch workflow * fix: workflow dispatch * fix: workflow dispatch * fix: workflow dispatch * fix: workflow dispatch * perf: workflow runtime * perf: workflow runtime
2026-04-27 02:08:10 +08:00 · 2026-03-15 14:43:48 +08:00
parent 6ea65f644b
commit 007ca09772
16 changed files with 889 additions and 92 deletions
@@ -67,7 +67,9 @@ export async function createSinks(options: CreateSinksOptions): Promise<CreateSi
          timestampStyle: 'reset',

          categorySeparator: ':',
-          timestamp: () => dayjs().format('YYYY-MM-DD HH:mm:ss')
+          timestamp: () => dayjs().format('YYYY-MM-DD HH:mm:ss'),
+          // Full depth for nested objects (e.g. Zod errors) in console output
+          inspectOptions: { depth: 5 }
        })
      }),
      (record) => levelFilter(record, consoleLevel)
@@ -366,9 +366,16 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
      string,
      { node: RuntimeNodeItemType; skippedNodeIdList: Set<string> }
    >();
-    private runningNodeCount = 0;
    private maxConcurrency: number;
    private resolve: (e: WorkflowQueue) => void;
+    private processingActive = false; // 标记是否正在处理队列
+
+    // Buffer
+    // 可以根据 nodeId 获取所有的 source 边和 target 边
+    private edgeIndex = {
+      bySource: new Map<string, RuntimeEdgeItemType[]>(),
+      byTarget: new Map<string, RuntimeEdgeItemType[]>()
+    };

    constructor({
      maxConcurrency = 10,
@@ -388,6 +395,20 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
        if (!node) return;
        this.addSkipNode(node, new Set(skippedNodeIdList));
      });
+
+      // 一次性构建索引 - O(m)
+      const filteredEdges = filterWorkflowEdges(runtimeEdges);
+      filteredEdges.forEach((edge) => {
+        if (!this.edgeIndex.bySource.has(edge.source)) {
+          this.edgeIndex.bySource.set(edge.source, []);
+        }
+        this.edgeIndex.bySource.get(edge.source)!.push(edge);
+
+        if (!this.edgeIndex.byTarget.has(edge.target)) {
+          this.edgeIndex.byTarget.set(edge.target, []);
+        }
+        this.edgeIndex.byTarget.get(edge.target)!.push(edge);
+      });
    }

    // Add active node to queue (if already in the queue, it will not be added again)
@@ -397,50 +418,79 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
      }
      this.activeRunQueue.add(nodeId);

-      this.processActiveNode();
+      // 非递归触发：如果没有正在处理，则启动处理循环
+      if (!this.processingActive) {
+        this.startProcessing();
+      }
    }
-    // Process next active node
-    private async processActiveNode() {
-      // Finish
-      if (this.activeRunQueue.size === 0 && this.runningNodeCount === 0) {
-        if (isDebugMode) {
-          // 没有下一个激活节点，说明debug 进入了一个“即将结束”状态。可以开始处理 skip 节点
-          if (this.debugNextStepRunNodes.length === 0 && this.skipNodeQueue.size > 0) {
-            this.processSkipNodes();
-          } else {
-            this.resolve(this);
+
+    // 迭代处理队列（替代递归的 processActiveNode）
+    private async startProcessing() {
+      // 防止重复启动
+      if (this.processingActive) {
+        return;
+      }
+
+      this.processingActive = true;
+
+      try {
+        const runningNodePromises = new Set<Promise<unknown>>();
+
+        // 迭代循环替代递归
+        while (true) {
+          // 检查结束条件
+          if (this.activeRunQueue.size === 0 && runningNodePromises.size === 0) {
+            if (isDebugMode) {
+              // 没有下一个激活节点，说明debug 进入了一个”即将结束”状态。可以开始处理 skip 节点
+              if (this.debugNextStepRunNodes.length === 0 && this.skipNodeQueue.size > 0) {
+                await this.processSkipNodes();
+                continue;
+              } else {
+                this.resolve(this);
+                break;
+              }
+            }
+
+            // 如果没有交互响应，则开始处理 skip（交互响应的 skip 需要留给后续处理）
+            if (this.skipNodeQueue.size > 0 && !this.nodeInteractiveResponse) {
+              await this.processSkipNodes();
+              continue;
+            } else {
+              this.resolve(this);
+              break;
+            }
+          }
+
+          // 检查并发限制
+          if (this.activeRunQueue.size === 0 || runningNodePromises.size >= this.maxConcurrency) {
+            if (runningNodePromises.size > 0) {
+              // 当上一个节点运行结束时，立即运行下一轮
+              await Promise.race(runningNodePromises);
+            } else {
+              // 理论上不应出现此情况，防御性退回到让出进程
+              await surrenderProcess();
+            }
+            continue;
+          }
+
+          // 处理下一个节点
+          const nodeId = this.activeRunQueue.keys().next().value;
+          const node = nodeId ? this.runtimeNodesMap.get(nodeId) : undefined;
+
+          if (nodeId) {
+            this.activeRunQueue.delete(nodeId);
+          }
+
+          if (node) {
+            // 不再递归调用，异步执行节点（不等待完成）
+            const nodePromise: Promise<unknown> = this.checkNodeCanRun(node).finally(() => {
+              runningNodePromises.delete(nodePromise);
+            });
+            runningNodePromises.add(nodePromise);
          }
-          return;
        }
-
-        // 如果没有交互响应，则开始处理 skip（交互响应的 skip 需要留给后续处理）
-        if (this.skipNodeQueue.size > 0 && !this.nodeInteractiveResponse) {
-          this.processSkipNodes();
-        } else {
-          this.resolve(this);
-        }
-        return;
-      }
-
-      // Over max concurrency（如果 this.activeRunQueue.size === 0 条件触发，代表肯定有节点在运行）
-      if (this.activeRunQueue.size === 0 || this.runningNodeCount >= this.maxConcurrency) {
-        return;
-      }
-
-      await surrenderProcess();
-      const nodeId = this.activeRunQueue.keys().next().value;
-      const node = nodeId ? this.runtimeNodesMap.get(nodeId) : undefined;
-
-      if (nodeId) {
-        this.activeRunQueue.delete(nodeId);
-      }
-      if (node) {
-        this.runningNodeCount++;
-
-        this.checkNodeCanRun(node).finally(() => {
-          this.runningNodeCount--;
-          this.processActiveNode();
-        });
+      } finally {
+        this.processingActive = false;
      }
    }

@@ -453,17 +503,16 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR

      this.skipNodeQueue.set(node.nodeId, { node, skippedNodeIdList: concatSkippedNodeIdList });
    }
+
+    // 迭代处理 skip 节点（每次只处理一个，然后返回主循环检查 active）
    private async processSkipNodes() {
-      // 取一个 node，并且从队列里删除
      await surrenderProcess();
      const skipItem = this.skipNodeQueue.values().next().value;
      if (skipItem) {
        this.skipNodeQueue.delete(skipItem.node.nodeId);
-        this.checkNodeCanRun(skipItem.node, skipItem.skippedNodeIdList).finally(() => {
-          this.processActiveNode();
+        await this.checkNodeCanRun(skipItem.node, skipItem.skippedNodeIdList).catch((error) => {
+          logger.error('Workflow skip node run error', { error, nodeName: skipItem.node.name });
        });
-      } else {
-        this.processActiveNode();
      }
    }

@@ -579,7 +628,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
      // run module
      const dispatchRes: NodeResponseType = await (async () => {
        if (callbackMap[node.flowNodeType]) {
-          const targetEdges = runtimeEdges.filter((item) => item.source === node.nodeId);
+          const targetEdges = this.edgeIndex.bySource.get(node.nodeId) || [];
          const errorHandleId = getHandleId(node.nodeId, 'source_catch', 'right');

          try {
@@ -848,9 +897,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
        // Get next source edges and update status
        const skipHandleId = result[DispatchNodeResponseKeyEnum.skipHandleId] || [];

-        const targetEdges = filterWorkflowEdges(runtimeEdges).filter(
-          (item) => item.source === node.nodeId
-        );
+        const targetEdges = this.edgeIndex.bySource.get(node.nodeId) || [];

        // update edge status
        targetEdges.forEach((edge) => {
@@ -864,23 +911,20 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
        // 同时可以去重
        const nextStepActiveNodesMap = new Map<string, RuntimeNodeItemType>();
        const nextStepSkipNodesMap = new Map<string, RuntimeNodeItemType>();
-        runtimeNodes.forEach((node) => {
-          if (targetEdges.some((item) => item.target === node.nodeId && item.status === 'active')) {
-            nextStepActiveNodesMap.set(node.nodeId, node);
-          }
-          if (
-            targetEdges.some((item) => item.target === node.nodeId && item.status === 'skipped')
-          ) {
-            nextStepSkipNodesMap.set(node.nodeId, node);
+        targetEdges.forEach((edge) => {
+          const targetNode = this.runtimeNodesMap.get(edge.target);
+          if (!targetNode) return;
+
+          if (edge.status === 'active') {
+            nextStepActiveNodesMap.set(targetNode.nodeId, targetNode);
+          } else if (edge.status === 'skipped') {
+            nextStepSkipNodesMap.set(targetNode.nodeId, targetNode);
          }
        });

-        const nextStepActiveNodes = Array.from(nextStepActiveNodesMap.values());
-        const nextStepSkipNodes = Array.from(nextStepSkipNodesMap.values());
-
        return {
-          nextStepActiveNodes,
-          nextStepSkipNodes
+          nextStepActiveNodes: Array.from(nextStepActiveNodesMap.values()),
+          nextStepSkipNodes: Array.from(nextStepSkipNodesMap.values())
        };
      };

@@ -900,11 +944,6 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
        return;
      }

-      logger.debug('Run workflow node', {
-        maxRunTimes: data.maxRunTimes,
-        appId: data.runningAppInfo.id
-      });
-
      // Get node run status by edges
      const status = checkNodeRunStatus({
        nodesMap: this.runtimeNodesMap,
@@ -1111,6 +1150,10 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
  });

  const workflowQueue = await new Promise<WorkflowQueue>((resolve) => {
+    logger.info('Workflow run start', {
+      maxRunTimes: data.maxRunTimes,
+      appId: data.runningAppInfo.id
+    });
    const workflowQueue = new WorkflowQueue({
      resolve,
      defaultSkipNodeQueue: data.lastInteractive?.skipNodeQueue || data.defaultSkipNodeQueue