feat: prompt call tool support reason;perf: ai proxy doc (#3982)

* update schema

* perf: ai proxy doc

* feat: prompt call tool support reason

packages/service/core/ai/config/provider/Moonshot.json

@@ -75,6 +75,81 @@
       "showTopP": true,
       "showStopSign": true,
       "responseFormatList": ["text", "json_object"]
+    },
+    {
+      "model": "moonshot-v1-8k-vision-preview",
+      "name": "moonshot-v1-8k-vision-preview",
+      "maxContext": 8000,
+      "maxResponse": 4000,
+      "quoteMaxToken": 6000,
+      "maxTemperature": 1,
+      "vision": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm",
+      "showTopP": true,
+      "showStopSign": true,
+      "responseFormatList": ["text", "json_object"]
+    },
+    {
+      "model": "moonshot-v1-32k-vision-preview",
+      "name": "moonshot-v1-32k-vision-preview",
+      "maxContext": 32000,
+      "maxResponse": 4000,
+      "quoteMaxToken": 32000,
+      "maxTemperature": 1,
+      "vision": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm",
+      "showTopP": true,
+      "showStopSign": true,
+      "responseFormatList": ["text", "json_object"]
+    },
+    {
+      "model": "moonshot-v1-128k-vision-preview",
+      "name": "moonshot-v1-128k-vision-preview",
+      "maxContext": 128000,
+      "maxResponse": 4000,
+      "quoteMaxToken": 60000,
+      "maxTemperature": 1,
+      "vision": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm",
+      "showTopP": true,
+      "showStopSign": true,
+      "responseFormatList": ["text", "json_object"]
     }
   ]
 }

packages/service/core/app/templates/templateSchema.ts

@@ -9,41 +9,23 @@ const AppTemplateSchema = new Schema({
     type: String,
     required: true
   },
-  name: {
-    type: String
-  },
-  intro: {
-    type: String
-  },
-  avatar: {
-    type: String
-  },
-  author: {
-    type: String
-  },
+  name: String,
+  intro: String,
+  avatar: String,
+  author: String,
   tags: {
     type: [String],
     default: undefined
   },
-  type: {
-    type: String
-  },
-  isActive: {
-    type: Boolean
-  },
-  userGuide: {
-    type: Object
-  },
-  isQuickTemplate: {
-    type: Boolean
-  },
+  type: String,
+  isActive: Boolean,
+  userGuide: Object,
+  isQuickTemplate: Boolean,
   order: {
     type: Number,
     default: -1
   },
-  workflow: {
-    type: Object
-  }
+  workflow: Object
 });
 
 AppTemplateSchema.index({ templateId: 1 });

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -55,7 +55,8 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
       userChatInput,
       history = 6,
       fileUrlList: fileLinks,
-      aiChatVision
+      aiChatVision,
+      aiChatReasoning
     }
   } = props;
 
@@ -63,6 +64,9 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
   const useVision = aiChatVision && toolModel.vision;
   const chatHistories = getHistories(history, histories);
 
+  props.params.aiChatVision = aiChatVision && toolModel.vision;
+  props.params.aiChatReasoning = aiChatReasoning && toolModel.reasoning;
+
   const toolNodeIds = filterToolNodeIdByEdges({ nodeId, edges: runtimeEdges });
 
   // Gets the module to which the tool is connected

packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts

@@ -24,7 +24,12 @@ import {
 import { AIChatItemType } from '@fastgpt/global/core/chat/type';
 import { GPTMessages2Chats } from '@fastgpt/global/core/chat/adapt';
 import { formatToolResponse, initToolCallEdges, initToolNodes } from './utils';
-import { computedMaxToken, llmCompletionsBodyFormat } from '../../../../ai/utils';
+import {
+  computedMaxToken,
+  llmCompletionsBodyFormat,
+  parseReasoningContent,
+  parseReasoningStreamContent
+} from '../../../../ai/utils';
 import { WorkflowResponseType } from '../../type';
 import { toolValueTypeList } from '@fastgpt/global/core/workflow/constants';
 import { WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
@@ -58,6 +63,7 @@ export const runToolWithPromptCall = async (
       temperature,
       maxToken,
       aiChatVision,
+      aiChatReasoning,
       aiChatTopP,
       aiChatStopSign,
       aiChatResponseFormat,
@@ -216,7 +222,7 @@ export const runToolWithPromptCall = async (
   const [requestMessages] = await Promise.all([
     loadRequestMessages({
       messages: filterMessages,
-      useVision: toolModel.vision && aiChatVision,
+      useVision: aiChatVision,
       origin: requestOrigin
     })
   ]);
@@ -251,22 +257,46 @@ export const runToolWithPromptCall = async (
     }
   });
 
-  const answer = await (async () => {
+  const { answer, reasoning } = await (async () => {
     if (res && isStreamResponse) {
-      const { answer } = await streamResponse({
+      const { answer, reasoning } = await streamResponse({
         res,
         toolNodes,
         stream: aiResponse,
-        workflowStreamResponse
+        workflowStreamResponse,
+        aiChatReasoning
       });
 
-      return answer;
+      return { answer, reasoning };
     } else {
-      const result = aiResponse as ChatCompletion;
+      const content = aiResponse.choices?.[0]?.message?.content || '';
+      const reasoningContent: string = aiResponse.choices?.[0]?.message?.reasoning_content || '';
 
-      return result.choices?.[0]?.message?.content || '';
+      // API already parse reasoning content
+      if (reasoningContent || !aiChatReasoning) {
+        return {
+          answer: content,
+          reasoning: reasoningContent
+        };
+      }
+
+      const [think, answer] = parseReasoningContent(content);
+      return {
+        answer,
+        reasoning: think
+      };
     }
   })();
+
+  if (stream && !isStreamResponse && aiChatReasoning && reasoning) {
+    workflowStreamResponse?.({
+      event: SseResponseEventEnum.fastAnswer,
+      data: textAdaptGptResponse({
+        reasoning_content: reasoning
+      })
+    });
+  }
+
   const { answer: replaceAnswer, toolJson } = parseAnswer(answer);
   if (!answer && !toolJson) {
     return Promise.reject(getEmptyResponseTip());
@@ -294,11 +324,16 @@ export const runToolWithPromptCall = async (
     }
 
     // No tool is invoked, indicating that the process is over
-    const gptAssistantResponse: ChatCompletionAssistantMessageParam = {
+    const gptAssistantResponse: ChatCompletionMessageParam = {
       role: ChatCompletionRequestMessageRoleEnum.Assistant,
-      content: replaceAnswer
+      content: replaceAnswer,
+      reasoning_text: reasoning
     };
-    const completeMessages = filterMessages.concat(gptAssistantResponse);
+    const completeMessages = filterMessages.concat({
+      ...gptAssistantResponse,
+      reasoning_text: undefined
+    });
+
     const inputTokens = await countGptMessagesTokens(requestMessages);
     const outputTokens = await countGptMessagesTokens([gptAssistantResponse]);
 
@@ -379,9 +414,10 @@ export const runToolWithPromptCall = async (
   })();
 
   // 合并工具调用的结果，使用 functionCall 格式存储。
-  const assistantToolMsgParams: ChatCompletionAssistantMessageParam = {
+  const assistantToolMsgParams: ChatCompletionMessageParam = {
     role: ChatCompletionRequestMessageRoleEnum.Assistant,
-    function_call: toolJson
+    function_call: toolJson,
+    reasoning_text: reasoning
   };
 
   // Only toolCall tokens are counted here, Tool response tokens count towards the next reply
@@ -502,12 +538,14 @@ ANSWER: `;
 async function streamResponse({
   res,
   stream,
-  workflowStreamResponse
+  workflowStreamResponse,
+  aiChatReasoning
 }: {
   res: NextApiResponse;
   toolNodes: ToolNodeItemType[];
   stream: StreamChatType;
   workflowStreamResponse?: WorkflowResponseType;
+  aiChatReasoning?: boolean;
 }) {
   const write = responseWriteController({
     res,
@@ -515,7 +553,9 @@ async function streamResponse({
   });
 
   let startResponseWrite = false;
-  let textAnswer = '';
+  let answer = '';
+  let reasoning = '';
+  const { parsePart, getStartTagBuffer } = parseReasoningStreamContent();
 
   for await (const part of stream) {
     if (res.closed) {
@@ -523,13 +563,21 @@ async function streamResponse({
       break;
     }
 
-    const responseChoice = part.choices?.[0]?.delta;
-    // console.log(responseChoice, '---===');
+    const [reasoningContent, content] = parsePart(part, aiChatReasoning);
+    answer += content;
+    reasoning += reasoningContent;
 
-    if (responseChoice?.content) {
-      const content = responseChoice?.content || '';
-      textAnswer += content;
+    if (aiChatReasoning && reasoningContent) {
+      workflowStreamResponse?.({
+        write,
+        event: SseResponseEventEnum.answer,
+        data: textAdaptGptResponse({
+          reasoning_content: reasoningContent
+        })
+      });
+    }
 
+    if (content) {
       if (startResponseWrite) {
         workflowStreamResponse?.({
           write,
@@ -538,18 +586,20 @@ async function streamResponse({
             text: content
           })
         });
-      } else if (textAnswer.length >= 3) {
-        textAnswer = textAnswer.trim();
-        if (textAnswer.startsWith('0')) {
+      } else if (answer.length >= 3) {
+        answer = answer.trimStart();
+        if (/0(:|：)/.test(answer)) {
           startResponseWrite = true;
+
           // find first : index
-          const firstIndex = textAnswer.indexOf(':');
-          textAnswer = textAnswer.substring(firstIndex + 1).trim();
+          const firstIndex =
+            answer.indexOf('0:') !== -1 ? answer.indexOf('0:') : answer.indexOf('0：');
+          answer = answer.substring(firstIndex + 2).trim();
           workflowStreamResponse?.({
             write,
             event: SseResponseEventEnum.answer,
             data: textAdaptGptResponse({
-              text: textAnswer
+              text: answer
             })
           });
         }
@@ -557,7 +607,23 @@ async function streamResponse({
     }
   }
 
-  return { answer: textAnswer.trim() };
+  if (answer === '') {
+    answer = getStartTagBuffer();
+    if (/0(:|：)/.test(answer)) {
+      // find first : index
+      const firstIndex = answer.indexOf('0:') !== -1 ? answer.indexOf('0:') : answer.indexOf('0：');
+      answer = answer.substring(firstIndex + 2).trim();
+      workflowStreamResponse?.({
+        write,
+        event: SseResponseEventEnum.answer,
+        data: textAdaptGptResponse({
+          text: answer
+        })
+      });
+    }
+  }
+
+  return { answer, reasoning };
 }
 
 const parseAnswer = (
@@ -568,8 +634,7 @@ const parseAnswer = (
 } => {
   str = str.trim();
   // 首先，使用正则表达式提取TOOL_ID和TOOL_ARGUMENTS
-  const prefixReg = /^1(:|：)/;
-  const answerPrefixReg = /^0(:|：)/;
+  const prefixReg = /1(:|：)/;
 
   if (prefixReg.test(str)) {
     const toolString = sliceJsonStr(str);
@@ -585,13 +650,21 @@ const parseAnswer = (
         }
       };
     } catch (error) {
-      return {
-        answer: ERROR_TEXT
-      };
+      if (/^1(:|：)/.test(str)) {
+        return {
+          answer: ERROR_TEXT
+        };
+      } else {
+        return {
+          answer: str
+        };
+      }
     }
   } else {
+    const firstIndex = str.indexOf('0:') !== -1 ? str.indexOf('0:') : str.indexOf('0：');
+    const answer = str.substring(firstIndex + 2).trim();
     return {
-      answer: str.replace(answerPrefixReg, '')
+      answer
     };
   }
 };

packages/service/core/workflow/dispatch/agent/runTool/type.d.ts

@@ -22,6 +22,7 @@ export type DispatchToolModuleProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.aiChatTemperature]: number;
   [NodeInputKeyEnum.aiChatMaxToken]: number;
   [NodeInputKeyEnum.aiChatVision]?: boolean;
+  [NodeInputKeyEnum.aiChatReasoning]?: boolean;
   [NodeInputKeyEnum.aiChatTopP]?: number;
   [NodeInputKeyEnum.aiChatStopSign]?: string;
   [NodeInputKeyEnum.aiChatResponseFormat]?: string;

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -563,6 +563,15 @@ async function streamResponse({
   // if answer is empty, try to get value from startTagBuffer. (Cause: The response content is too short to exceed the minimum parse length)
   if (answer === '') {
     answer = getStartTagBuffer();
+    if (isResponseAnswerText && answer) {
+      workflowStreamResponse?.({
+        write,
+        event: SseResponseEventEnum.answer,
+        data: textAdaptGptResponse({
+          text: answer
+        })
+      });
+    }
   }
 
   return { answer, reasoning };