Tool call support interactive node (#2903)

* feat: tool call support interactive node

* feat: interactive node tool response

* fix: tool call concat

* fix: llm history concat

packages/global/core/ai/type.d.ts

@@ -4,12 +4,14 @@ import type {
   ChatCompletionChunk,
   ChatCompletionMessageParam as SdkChatCompletionMessageParam,
   ChatCompletionToolMessageParam,
-  ChatCompletionAssistantMessageParam,
   ChatCompletionContentPart as SdkChatCompletionContentPart,
-  ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam
+  ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam,
+  ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam,
+  ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam,
+  ChatCompletionContentPartText
 } from 'openai/resources';
 import { ChatMessageTypeEnum } from './constants';
-import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
+import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
 export * from 'openai/resources';
 
 // Extension of ChatCompletionMessageParam, Add file url type
@@ -22,18 +24,31 @@ export type ChatCompletionContentPartFile = {
 export type ChatCompletionContentPart =
   | SdkChatCompletionContentPart
   | ChatCompletionContentPartFile;
-type CustomChatCompletionUserMessageParam = {
-  content: string | Array<ChatCompletionContentPart>;
+type CustomChatCompletionUserMessageParam = Omit<ChatCompletionUserMessageParam, 'content'> & {
   role: 'user';
+  content: string | Array<ChatCompletionContentPart>;
+};
+type CustomChatCompletionToolMessageParam = SdkChatCompletionToolMessageParam & {
+  role: 'tool';
   name?: string;
 };
+type CustomChatCompletionAssistantMessageParam = SdkChatCompletionAssistantMessageParam & {
+  role: 'assistant';
+  interactive?: WorkflowInteractiveResponseType;
+};
 
 export type ChatCompletionMessageParam = (
-  | Exclude<SdkChatCompletionMessageParam, SdkChatCompletionUserMessageParam>
+  | Exclude<
+      SdkChatCompletionMessageParam,
+      | SdkChatCompletionUserMessageParam
+      | SdkChatCompletionToolMessageParam
+      | SdkChatCompletionAssistantMessageParam
+    >
   | CustomChatCompletionUserMessageParam
+  | CustomChatCompletionToolMessageParam
+  | CustomChatCompletionAssistantMessageParam
 ) & {
   dataId?: string;
-  interactive?: InteractiveNodeResponseItemType;
 };
 export type SdkChatCompletionMessageParam = SdkChatCompletionMessageParam;
 
@@ -47,11 +62,12 @@ export type ChatCompletionMessageToolCall = ChatCompletionMessageToolCall & {
   toolName?: string;
   toolAvatar?: string;
 };
-export type ChatCompletionMessageFunctionCall = ChatCompletionAssistantMessageParam.FunctionCall & {
-  id?: string;
-  toolName?: string;
-  toolAvatar?: string;
-};
+export type ChatCompletionMessageFunctionCall =
+  SdkChatCompletionAssistantMessageParam.FunctionCall & {
+    id?: string;
+    toolName?: string;
+    toolAvatar?: string;
+  };
 
 // Stream response
 export type StreamChatType = Stream<ChatCompletionChunk>;

packages/global/core/chat/adapt.ts

@@ -90,8 +90,9 @@ export const chats2GPTMessages = ({
         });
       }
     } else {
+      const aiResults: ChatCompletionMessageParam[] = [];
       //AI
-      item.value.forEach((value) => {
+      item.value.forEach((value, i) => {
         if (value.type === ChatItemValueTypeEnum.tool && value.tools && reserveTool) {
           const tool_calls: ChatCompletionMessageToolCall[] = [];
           const toolResponse: ChatCompletionToolMessageParam[] = [];
@@ -111,28 +112,53 @@ export const chats2GPTMessages = ({
               content: tool.response
             });
           });
-          results = results
-            .concat({
+          aiResults.push({
+            dataId,
+            role: ChatCompletionRequestMessageRoleEnum.Assistant,
+            tool_calls
+          });
+          aiResults.push(...toolResponse);
+        } else if (
+          value.type === ChatItemValueTypeEnum.text &&
+          typeof value.text?.content === 'string'
+        ) {
+          // Concat text
+          const lastValue = item.value[i - 1];
+          const lastResult = aiResults[aiResults.length - 1];
+          if (
+            lastValue &&
+            lastValue.type === ChatItemValueTypeEnum.text &&
+            typeof lastResult.content === 'string'
+          ) {
+            lastResult.content += value.text.content;
+          } else {
+            aiResults.push({
               dataId,
               role: ChatCompletionRequestMessageRoleEnum.Assistant,
-              tool_calls
-            })
-            .concat(toolResponse);
-        } else if (value.text?.content) {
-          results.push({
-            dataId,
-            role: ChatCompletionRequestMessageRoleEnum.Assistant,
-            content: value.text.content
-          });
+              content: value.text.content
+            });
+          }
         } else if (value.type === ChatItemValueTypeEnum.interactive) {
-          results = results.concat({
+          aiResults.push({
             dataId,
             role: ChatCompletionRequestMessageRoleEnum.Assistant,
-            interactive: value.interactive,
-            content: ''
+            interactive: value.interactive
           });
         }
       });
+
+      // Auto add empty assistant message
+      results = results.concat(
+        aiResults.length > 0
+          ? aiResults
+          : [
+              {
+                dataId,
+                role: ChatCompletionRequestMessageRoleEnum.Assistant,
+                content: ''
+              }
+            ]
+      );
     }
   });
 
@@ -215,14 +241,7 @@ export const GPTMessages2Chats = (
         obj === ChatRoleEnum.AI &&
         item.role === ChatCompletionRequestMessageRoleEnum.Assistant
       ) {
-        if (item.content && typeof item.content === 'string') {
-          value.push({
-            type: ChatItemValueTypeEnum.text,
-            text: {
-              content: item.content
-            }
-          });
-        } else if (item.tool_calls && reserveTool) {
+        if (item.tool_calls && reserveTool) {
           // save tool calls
           const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[];
           value.push({
@@ -278,6 +297,18 @@ export const GPTMessages2Chats = (
             type: ChatItemValueTypeEnum.interactive,
             interactive: item.interactive
           });
+        } else if (typeof item.content === 'string') {
+          const lastValue = value[value.length - 1];
+          if (lastValue && lastValue.type === ChatItemValueTypeEnum.text && lastValue.text) {
+            lastValue.text.content += item.content;
+          } else {
+            value.push({
+              type: ChatItemValueTypeEnum.text,
+              text: {
+                content: item.content
+              }
+            });
+          }
         }
       }
 

packages/global/core/chat/type.d.ts

@@ -15,7 +15,7 @@ import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 import { DispatchNodeResponseType } from '../workflow/runtime/type.d';
 import { ChatBoxInputType } from '../../../../projects/app/src/components/core/chat/ChatContainer/ChatBox/type';
-import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
+import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
 
 export type ChatSchema = {
   _id: string;
@@ -73,7 +73,7 @@ export type AIChatItemValueItemType = {
     content: string;
   };
   tools?: ToolModuleResponseItemType[];
-  interactive?: InteractiveNodeResponseItemType;
+  interactive?: WorkflowInteractiveResponseType;
 };
 export type AIChatItemType = {
   obj: ChatRoleEnum.AI;

packages/global/core/chat/utils.ts

@@ -143,3 +143,29 @@ export const getChatSourceByPublishChannel = (publishChannel: PublishChannelEnum
       return ChatSourceEnum.online;
   }
 };
+
+/* 
+  Merge chat responseData
+  1. Same tool mergeSignId (Interactive tool node)
+*/
+export const mergeChatResponseData = (responseDataList: ChatHistoryItemResType[]) => {
+  let lastResponse: ChatHistoryItemResType | undefined = undefined;
+
+  return responseDataList.reduce<ChatHistoryItemResType[]>((acc, curr) => {
+    if (lastResponse && lastResponse.mergeSignId && curr.mergeSignId === lastResponse.mergeSignId) {
+      // 替换 lastResponse
+      const concatResponse: ChatHistoryItemResType = {
+        ...curr,
+        runningTime: +((lastResponse.runningTime || 0) + (curr.runningTime || 0)).toFixed(2),
+        totalPoints: (lastResponse.totalPoints || 0) + (curr.totalPoints || 0),
+        childTotalPoints: (lastResponse.childTotalPoints || 0) + (curr.childTotalPoints || 0),
+        toolCallTokens: (lastResponse.toolCallTokens || 0) + (curr.toolCallTokens || 0),
+        toolDetail: [...(lastResponse.toolDetail || []), ...(curr.toolDetail || [])]
+      };
+      return [...acc.slice(0, -1), concatResponse];
+    } else {
+      lastResponse = curr;
+      return [...acc, curr];
+    }
+  }, []);
+};

packages/global/core/workflow/runtime/type.d.ts

@@ -73,7 +73,7 @@ export type RuntimeNodeItemType = {
   intro?: StoreNodeItemType['intro'];
   flowNodeType: StoreNodeItemType['flowNodeType'];
   showStatus?: StoreNodeItemType['showStatus'];
-  isEntry?: StoreNodeItemType['isEntry'];
+  isEntry?: boolean;
 
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
@@ -108,12 +108,14 @@ export type DispatchNodeResponseType = {
   customOutputs?: Record<string, any>;
   nodeInputs?: Record<string, any>;
   nodeOutputs?: Record<string, any>;
+  mergeSignId?: string;
 
   // bill
   tokens?: number;
   model?: string;
   contextTotalLen?: number;
   totalPoints?: number;
+  childTotalPoints?: number;
 
   // chat
   temperature?: number;

packages/global/core/workflow/runtime/utils.ts

@@ -69,7 +69,7 @@ export const initWorkflowEdgeStatus = (
   histories?: ChatItemType[]
 ): RuntimeEdgeItemType[] => {
   // If there is a history, use the last interactive value
-  if (!!histories) {
+  if (histories && histories.length > 0) {
     const memoryEdges = getLastInteractiveValue(histories)?.memoryEdges;
 
     if (memoryEdges && memoryEdges.length > 0) {
@@ -90,7 +90,7 @@ export const getWorkflowEntryNodeIds = (
   histories?: ChatItemType[]
 ) => {
   // If there is a history, use the last interactive entry node
-  if (!!histories) {
+  if (histories && histories.length > 0) {
     const entryNodeIds = getLastInteractiveValue(histories)?.entryNodeIds;
 
     if (Array.isArray(entryNodeIds) && entryNodeIds.length > 0) {

packages/global/core/workflow/template/system/interactive/formInput.ts

@@ -22,7 +22,7 @@ export const FormInputNode: FlowNodeTemplateType = {
   avatar: 'core/workflow/template/formInput',
   name: i18nT('app:workflow.form_input'),
   intro: i18nT(`app:workflow.form_input_tip`),
-  showStatus: true,
+  isTool: true,
   version: '4811',
   inputs: [
     {

packages/global/core/workflow/template/system/interactive/type.d.ts

@@ -1,8 +1,9 @@
-import { NodeOutputItemType } from '../../../../chat/type';
-import { FlowNodeOutputItemType } from '../../../type/io';
-import { RuntimeEdgeItemType } from '../../../runtime/type';
+import type { NodeOutputItemType } from '../../../../chat/type';
+import type { FlowNodeOutputItemType } from '../../../type/io';
+import type { RuntimeEdgeItemType } from '../../../runtime/type';
 import { FlowNodeInputTypeEnum } from 'core/workflow/node/constant';
 import { WorkflowIOValueTypeEnum } from 'core/workflow/constants';
+import type { ChatCompletionMessageParam } from '../../../../ai/type';
 
 export type UserSelectOptionItemType = {
   key: string;
@@ -32,6 +33,12 @@ type InteractiveBasicType = {
   entryNodeIds: string[];
   memoryEdges: RuntimeEdgeItemType[];
   nodeOutputs: NodeOutputItemType[];
+
+  toolParams?: {
+    entryNodeIds: string[]; // 记录工具中，交互节点的 Id，而不是起始工作流的入口
+    memoryMessages: ChatCompletionMessageParam[]; // 这轮工具中，产生的新的 messages
+    toolCallId: string; // 记录对应 tool 的id，用于后续交互节点可以替换掉 tool 的 response
+  };
 };
 
 type UserSelectInteractive = {
@@ -52,5 +59,5 @@ type UserInputInteractive = {
   };
 };
 
-export type InteractiveNodeResponseItemType = InteractiveBasicType &
-  (UserSelectInteractive | UserInputInteractive);
+export type InteractiveNodeResponseType = UserSelectInteractive | UserInputInteractive;
+export type WorkflowInteractiveResponseType = InteractiveBasicType & InteractiveNodeResponseType;

packages/global/core/workflow/template/system/interactive/userSelect.ts

@@ -23,7 +23,7 @@ export const UserSelectNode: FlowNodeTemplateType = {
   diagram: '/imgs/app/userSelect.svg',
   name: i18nT('app:workflow.user_select'),
   intro: i18nT(`app:workflow.user_select_tip`),
-  showStatus: true,
+  isTool: true,
   version: '489',
   inputs: [
     {