V4.7-alpha (#985)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/string/tiktoken/index.ts

@@ -1,10 +1,15 @@
 /* Only the token of gpt-3.5-turbo is used */
 import type { ChatItemType } from '../../../core/chat/type';
 import { Tiktoken } from 'js-tiktoken/lite';
-import { adaptChat2GptMessages } from '../../../core/chat/adapt';
-import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constant';
+import { chats2GPTMessages } from '../../../core/chat/adapt';
 import encodingJson from './cl100k_base.json';
-import { ChatMessageItemType } from '../../../core/ai/type';
+import {
+  ChatCompletionMessageParam,
+  ChatCompletionContentPart,
+  ChatCompletionCreateParams,
+  ChatCompletionTool
+} from '../../../core/ai/type';
+import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constants';
 
 /* init tikToken obj */
 export function getTikTokenEnc() {
@@ -29,18 +34,25 @@ export function getTikTokenEnc() {
 
 /* count one prompt tokens */
 export function countPromptTokens(
-  prompt = '',
-  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = '',
-  tools?: any
+  prompt: string | ChatCompletionContentPart[] | null | undefined = '',
+  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = ''
 ) {
   const enc = getTikTokenEnc();
-  const toolText = tools
-    ? JSON.stringify(tools)
-        .replace('"', '')
-        .replace('\n', '')
-        .replace(/( ){2,}/g, ' ')
-    : '';
-  const text = `${role}\n${prompt}\n${toolText}`.trim();
+  const promptText = (() => {
+    if (!prompt) return '';
+    if (typeof prompt === 'string') return prompt;
+    let promptText = '';
+    prompt.forEach((item) => {
+      if (item.type === 'text') {
+        promptText += item.text;
+      } else if (item.type === 'image_url') {
+        promptText += item.image_url.url;
+      }
+    });
+    return promptText;
+  })();
+
+  const text = `${role}\n${promptText}`.trim();
 
   try {
     const encodeText = enc.encode(text);
@@ -50,15 +62,66 @@ export function countPromptTokens(
     return text.length;
   }
 }
+export const countToolsTokens = (
+  tools?: ChatCompletionTool[] | ChatCompletionCreateParams.Function[]
+) => {
+  if (!tools || tools.length === 0) return 0;
+
+  const enc = getTikTokenEnc();
+
+  const toolText = tools
+    ? JSON.stringify(tools)
+        .replace('"', '')
+        .replace('\n', '')
+        .replace(/( ){2,}/g, ' ')
+    : '';
+
+  return enc.encode(toolText).length;
+};
 
 /* count messages tokens */
-export const countMessagesTokens = (messages: ChatItemType[], tools?: any) => {
-  const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
+export const countMessagesTokens = (messages: ChatItemType[]) => {
+  const adaptMessages = chats2GPTMessages({ messages, reserveId: true });
 
-  return countGptMessagesTokens(adaptMessages, tools);
+  return countGptMessagesTokens(adaptMessages);
 };
-export const countGptMessagesTokens = (messages: ChatMessageItemType[], tools?: any) =>
-  messages.reduce((sum, item) => sum + countPromptTokens(item.content, item.role, tools), 0);
+export const countGptMessagesTokens = (
+  messages: ChatCompletionMessageParam[],
+  tools?: ChatCompletionTool[],
+  functionCall?: ChatCompletionCreateParams.Function[]
+) =>
+  messages.reduce((sum, item) => {
+    // Evaluates the text of toolcall and functioncall
+    const functionCallPrompt = (() => {
+      let prompt = '';
+      if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
+        const toolCalls = item.tool_calls;
+        prompt +=
+          toolCalls
+            ?.map((item) => `${item?.function?.name} ${item?.function?.arguments}`.trim())
+            ?.join('') || '';
+
+        const functionCall = item.function_call;
+        prompt += `${functionCall?.name} ${functionCall?.arguments}`.trim();
+      }
+      return prompt;
+    })();
+
+    const contentPrompt = (() => {
+      if (!item.content) return '';
+      if (typeof item.content === 'string') return item.content;
+      return item.content
+        .map((item) => {
+          if (item.type === 'text') return item.text;
+          return '';
+        })
+        .join('');
+    })();
+
+    return sum + countPromptTokens(`${contentPrompt}${functionCallPrompt}`, item.role);
+  }, 0) +
+  countToolsTokens(tools) +
+  countToolsTokens(functionCall);
 
 /* slice messages from top to bottom by maxTokens */
 export function sliceMessagesTB({
@@ -68,7 +131,7 @@ export function sliceMessagesTB({
   messages: ChatItemType[];
   maxTokens: number;
 }) {
-  const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
+  const adaptMessages = chats2GPTMessages({ messages, reserveId: true });
   let reduceTokens = maxTokens;
   let result: ChatItemType[] = [];
 

packages/global/core/ai/constant.ts

@@ -1,7 +0,0 @@
-export enum ChatCompletionRequestMessageRoleEnum {
-  'System' = 'system',
-  'User' = 'user',
-  'Assistant' = 'assistant',
-  'Function' = 'function',
-  'Tool' = 'tool'
-}

packages/global/core/ai/constants.ts

@@ -0,0 +1,27 @@
+export enum ChatCompletionRequestMessageRoleEnum {
+  'System' = 'system',
+  'User' = 'user',
+  'Assistant' = 'assistant',
+  'Function' = 'function',
+  'Tool' = 'tool'
+}
+
+export enum ChatMessageTypeEnum {
+  text = 'text',
+  image_url = 'image_url'
+}
+
+export enum LLMModelTypeEnum {
+  all = 'all',
+  classify = 'classify',
+  extractFields = 'extractFields',
+  toolCall = 'toolCall',
+  queryExtension = 'queryExtension'
+}
+export const llmModelTypeFilterMap = {
+  [LLMModelTypeEnum.all]: 'model',
+  [LLMModelTypeEnum.classify]: 'usedInClassify',
+  [LLMModelTypeEnum.extractFields]: 'usedInExtractFields',
+  [LLMModelTypeEnum.toolCall]: 'usedInToolCall',
+  [LLMModelTypeEnum.queryExtension]: 'usedInQueryExtension'
+};

packages/global/core/ai/model.d.ts

@@ -10,7 +10,13 @@ export type LLMModelItemType = {
 
   censor?: boolean;
   vision?: boolean;
-  datasetProcess?: boolean;
+
+  // diff function model
+  datasetProcess?: boolean; // dataset
+  usedInClassify?: boolean; // classify
+  usedInExtractFields?: boolean; // extract fields
+  usedInToolCall?: boolean; // tool call
+  usedInQueryExtension?: boolean; // query extension
 
   functionCall: boolean;
   toolChoice: boolean;

packages/global/core/ai/type.d.ts

@@ -1,20 +1,33 @@
+import openai from 'openai';
 import type {
-  ChatCompletion,
-  ChatCompletionCreateParams,
+  ChatCompletionMessageToolCall,
   ChatCompletionChunk,
   ChatCompletionMessageParam,
-  ChatCompletionContentPart
+  ChatCompletionToolMessageParam,
+  ChatCompletionAssistantMessageParam
 } from 'openai/resources';
+import { ChatMessageTypeEnum } from './constants';
 
-export type ChatCompletionContentPart = ChatCompletionContentPart;
-export type ChatCompletionCreateParams = ChatCompletionCreateParams;
-export type ChatMessageItemType = Omit<ChatCompletionMessageParam, 'name'> & {
-  name?: any;
+export * from 'openai/resources';
+
+export type ChatCompletionMessageParam = ChatCompletionMessageParam & {
   dataId?: string;
-  content: any;
-} & any;
+};
+export type ChatCompletionToolMessageParam = ChatCompletionToolMessageParam & { name: string };
+export type ChatCompletionAssistantToolParam = {
+  role: 'assistant';
+  tool_calls: ChatCompletionMessageToolCall[];
+};
 
-export type ChatCompletion = ChatCompletion;
+export type ChatCompletionMessageToolCall = ChatCompletionMessageToolCall & {
+  toolName?: string;
+  toolAvatar?: string;
+};
+export type ChatCompletionMessageFunctionCall = ChatCompletionAssistantMessageParam.FunctionCall & {
+  id?: string;
+  toolName?: string;
+  toolAvatar?: string;
+};
 export type StreamChatType = Stream<ChatCompletionChunk>;
 
 export type PromptTemplateItem = {
@@ -22,3 +35,6 @@ export type PromptTemplateItem = {
   desc: string;
   value: string;
 };
+
+export default openai;
+export * from 'openai';

packages/global/core/chat/adapt.ts

@@ -1,40 +1,298 @@
-import type { ChatItemType } from '../../core/chat/type.d';
-import { ChatRoleEnum } from '../../core/chat/constants';
-import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constant';
-import type { ChatMessageItemType } from '../../core/ai/type.d';
+import type {
+  ChatItemType,
+  ChatItemValueItemType,
+  RuntimeUserPromptType,
+  UserChatItemType
+} from '../../core/chat/type.d';
+import { ChatFileTypeEnum, ChatItemValueTypeEnum, ChatRoleEnum } from '../../core/chat/constants';
+import type {
+  ChatCompletionContentPart,
+  ChatCompletionFunctionMessageParam,
+  ChatCompletionMessageFunctionCall,
+  ChatCompletionMessageParam,
+  ChatCompletionMessageToolCall,
+  ChatCompletionToolMessageParam
+} from '../../core/ai/type.d';
+import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constants';
 
-const chat2Message = {
-  [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,
-  [ChatRoleEnum.Human]: ChatCompletionRequestMessageRoleEnum.User,
-  [ChatRoleEnum.System]: ChatCompletionRequestMessageRoleEnum.System,
-  [ChatRoleEnum.Function]: ChatCompletionRequestMessageRoleEnum.Function,
-  [ChatRoleEnum.Tool]: ChatCompletionRequestMessageRoleEnum.Tool
-};
-const message2Chat = {
+const GPT2Chat = {
   [ChatCompletionRequestMessageRoleEnum.System]: ChatRoleEnum.System,
   [ChatCompletionRequestMessageRoleEnum.User]: ChatRoleEnum.Human,
   [ChatCompletionRequestMessageRoleEnum.Assistant]: ChatRoleEnum.AI,
-  [ChatCompletionRequestMessageRoleEnum.Function]: ChatRoleEnum.Function,
-  [ChatCompletionRequestMessageRoleEnum.Tool]: ChatRoleEnum.Tool
+  [ChatCompletionRequestMessageRoleEnum.Function]: ChatRoleEnum.AI,
+  [ChatCompletionRequestMessageRoleEnum.Tool]: ChatRoleEnum.AI
 };
 
-export function adaptRole_Chat2Message(role: `${ChatRoleEnum}`) {
-  return chat2Message[role];
-}
 export function adaptRole_Message2Chat(role: `${ChatCompletionRequestMessageRoleEnum}`) {
-  return message2Chat[role];
+  return GPT2Chat[role];
 }
 
-export const adaptChat2GptMessages = ({
+export const simpleUserContentPart = (content: ChatCompletionContentPart[]) => {
+  if (content.length === 1 && content[0].type === 'text') {
+    return content[0].text;
+  }
+  return content;
+};
+
+export const chats2GPTMessages = ({
   messages,
-  reserveId
+  reserveId,
+  reserveTool = false
 }: {
   messages: ChatItemType[];
   reserveId: boolean;
-}): ChatMessageItemType[] => {
-  return messages.map((item) => ({
-    ...(reserveId && { dataId: item.dataId }),
-    role: chat2Message[item.obj],
-    content: item.value || ''
-  }));
+  reserveTool?: boolean;
+}): ChatCompletionMessageParam[] => {
+  let results: ChatCompletionMessageParam[] = [];
+
+  messages.forEach((item) => {
+    const dataId = reserveId ? item.dataId : undefined;
+    if (item.obj === ChatRoleEnum.Human) {
+      const value = item.value
+        .map((item) => {
+          if (item.type === ChatItemValueTypeEnum.text) {
+            return {
+              type: 'text',
+              text: item.text?.content || ''
+            };
+          }
+          if (item.type === 'file' && item.file?.type === ChatFileTypeEnum.image) {
+            return {
+              type: 'image_url',
+              image_url: {
+                url: item.file?.url || ''
+              }
+            };
+          }
+          return;
+        })
+        .filter(Boolean) as ChatCompletionContentPart[];
+
+      results.push({
+        dataId,
+        role: ChatCompletionRequestMessageRoleEnum.User,
+        content: simpleUserContentPart(value)
+      });
+    } else if (item.obj === ChatRoleEnum.System) {
+      const content = item.value?.[0]?.text?.content;
+      if (content) {
+        results.push({
+          dataId,
+          role: ChatCompletionRequestMessageRoleEnum.System,
+          content
+        });
+      }
+    } else {
+      item.value.forEach((value) => {
+        if (value.type === ChatItemValueTypeEnum.tool && value.tools && reserveTool) {
+          const tool_calls: ChatCompletionMessageToolCall[] = [];
+          const toolResponse: ChatCompletionToolMessageParam[] = [];
+          value.tools.forEach((tool) => {
+            tool_calls.push({
+              id: tool.id,
+              type: 'function',
+              function: {
+                name: tool.functionName,
+                arguments: tool.params
+              }
+            });
+            toolResponse.push({
+              tool_call_id: tool.id,
+              role: ChatCompletionRequestMessageRoleEnum.Tool,
+              name: tool.functionName,
+              content: tool.response
+            });
+          });
+          results = results
+            .concat({
+              dataId,
+              role: ChatCompletionRequestMessageRoleEnum.Assistant,
+              tool_calls
+            })
+            .concat(toolResponse);
+        } else if (value.text) {
+          results.push({
+            dataId,
+            role: ChatCompletionRequestMessageRoleEnum.Assistant,
+            content: value.text.content
+          });
+        }
+      });
+    }
+  });
+
+  return results;
+};
+export const GPTMessages2Chats = (
+  messages: ChatCompletionMessageParam[],
+  reserveTool = true
+): ChatItemType[] => {
+  return messages
+    .map((item) => {
+      const value: ChatItemType['value'] = [];
+      const obj = GPT2Chat[item.role];
+
+      if (
+        obj === ChatRoleEnum.System &&
+        item.role === ChatCompletionRequestMessageRoleEnum.System
+      ) {
+        value.push({
+          type: ChatItemValueTypeEnum.text,
+          text: {
+            content: item.content
+          }
+        });
+      } else if (
+        obj === ChatRoleEnum.Human &&
+        item.role === ChatCompletionRequestMessageRoleEnum.User
+      ) {
+        if (typeof item.content === 'string') {
+          value.push({
+            type: ChatItemValueTypeEnum.text,
+            text: {
+              content: item.content
+            }
+          });
+        } else if (Array.isArray(item.content)) {
+          item.content.forEach((item) => {
+            if (item.type === 'text') {
+              value.push({
+                type: ChatItemValueTypeEnum.text,
+                text: {
+                  content: item.text
+                }
+              });
+            } else if (item.type === 'image_url') {
+              value.push({
+                //@ts-ignore
+                type: 'file',
+                file: {
+                  type: ChatFileTypeEnum.image,
+                  name: '',
+                  url: item.image_url.url
+                }
+              });
+            }
+          });
+          // @ts-ignore
+        }
+      } else if (
+        obj === ChatRoleEnum.AI &&
+        item.role === ChatCompletionRequestMessageRoleEnum.Assistant
+      ) {
+        if (item.content && typeof item.content === 'string') {
+          value.push({
+            type: ChatItemValueTypeEnum.text,
+            text: {
+              content: item.content
+            }
+          });
+        } else if (item.tool_calls && reserveTool) {
+          // save tool calls
+          const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[];
+          value.push({
+            //@ts-ignore
+            type: ChatItemValueTypeEnum.tool,
+            tools: toolCalls.map((tool) => {
+              let toolResponse =
+                messages.find(
+                  (msg) =>
+                    msg.role === ChatCompletionRequestMessageRoleEnum.Tool &&
+                    msg.tool_call_id === tool.id
+                )?.content || '';
+              toolResponse =
+                typeof toolResponse === 'string' ? toolResponse : JSON.stringify(toolResponse);
+
+              return {
+                id: tool.id,
+                toolName: tool.toolName || '',
+                toolAvatar: tool.toolAvatar || '',
+                functionName: tool.function.name,
+                params: tool.function.arguments,
+                response: toolResponse as string
+              };
+            })
+          });
+        } else if (item.function_call && reserveTool) {
+          const functionCall = item.function_call as ChatCompletionMessageFunctionCall;
+          const functionResponse = messages.find(
+            (msg) =>
+              msg.role === ChatCompletionRequestMessageRoleEnum.Function &&
+              msg.name === item.function_call?.name
+          ) as ChatCompletionFunctionMessageParam;
+
+          if (functionResponse) {
+            value.push({
+              //@ts-ignore
+              type: ChatItemValueTypeEnum.tool,
+              tools: [
+                {
+                  id: functionCall.id || '',
+                  toolName: functionCall.toolName || '',
+                  toolAvatar: functionCall.toolAvatar || '',
+                  functionName: functionCall.name,
+                  params: functionCall.arguments,
+                  response: functionResponse.content || ''
+                }
+              ]
+            });
+          }
+        }
+      }
+
+      return {
+        dataId: item.dataId,
+        obj,
+        value
+      } as ChatItemType;
+    })
+    .filter((item) => item.value.length > 0);
+};
+
+export const chatValue2RuntimePrompt = (value: ChatItemValueItemType[]): RuntimeUserPromptType => {
+  const prompt: RuntimeUserPromptType = {
+    files: [],
+    text: ''
+  };
+  value.forEach((item) => {
+    if (item.type === 'file' && item.file) {
+      prompt.files?.push(item.file);
+    } else if (item.text) {
+      prompt.text += item.text.content;
+    }
+  });
+  return prompt;
+};
+
+export const runtimePrompt2ChatsValue = (
+  prompt: RuntimeUserPromptType
+): UserChatItemType['value'] => {
+  const value: UserChatItemType['value'] = [];
+  if (prompt.files) {
+    prompt.files.forEach((file) => {
+      value.push({
+        type: ChatItemValueTypeEnum.file,
+        file
+      });
+    });
+  }
+  if (prompt.text) {
+    value.push({
+      type: ChatItemValueTypeEnum.text,
+      text: {
+        content: prompt.text
+      }
+    });
+  }
+  return value;
+};
+
+export const getSystemPrompt = (prompt?: string): ChatItemType[] => {
+  if (!prompt) return [];
+  return [
+    {
+      obj: ChatRoleEnum.System,
+      value: [{ type: ChatItemValueTypeEnum.text, text: { content: prompt } }]
+    }
+  ];
 };

packages/global/core/chat/constants.ts

@@ -1,28 +1,30 @@
 export enum ChatRoleEnum {
   System = 'System',
   Human = 'Human',
-  AI = 'AI',
-  Function = 'Function',
-  Tool = 'Tool'
+  AI = 'AI'
 }
 export const ChatRoleMap = {
   [ChatRoleEnum.System]: {
-    name: '系统提示词'
+    name: '系统'
   },
   [ChatRoleEnum.Human]: {
     name: '用户'
   },
   [ChatRoleEnum.AI]: {
     name: 'AI'
-  },
-  [ChatRoleEnum.Function]: {
-    name: 'Function'
-  },
-  [ChatRoleEnum.Tool]: {
-    name: 'Tool'
   }
 };
 
+export enum ChatFileTypeEnum {
+  image = 'image',
+  file = 'file'
+}
+export enum ChatItemValueTypeEnum {
+  text = 'text',
+  file = 'file',
+  tool = 'tool'
+}
+
 export enum ChatSourceEnum {
   test = 'test',
   online = 'online',

packages/global/core/chat/type.d.ts

@@ -1,11 +1,20 @@
 import { ClassifyQuestionAgentItemType } from '../module/type';
 import { SearchDataResponseItemType } from '../dataset/type';
-import { ChatRoleEnum, ChatSourceEnum, ChatStatusEnum } from './constants';
+import {
+  ChatFileTypeEnum,
+  ChatItemValueTypeEnum,
+  ChatRoleEnum,
+  ChatSourceEnum,
+  ChatStatusEnum
+} from './constants';
 import { FlowNodeTypeEnum } from '../module/node/constant';
 import { ModuleOutputKeyEnum } from '../module/constants';
+import { DispatchNodeResponseKeyEnum } from '../module/runtime/constants';
 import { AppSchema } from '../app/type';
 import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
+import { ChatBoxInputType } from '../../../../projects/app/src/components/ChatBox/type';
+import { DispatchNodeResponseType } from '../module/runtime/type.d';
 
 export type ChatSchema = {
   _id: string;
@@ -30,7 +39,53 @@ export type ChatWithAppSchema = Omit<ChatSchema, 'appId'> & {
   appId: AppSchema;
 };
 
-export type ChatItemSchema = {
+export type UserChatItemValueItemType = {
+  type: ChatItemValueTypeEnum.text | ChatItemValueTypeEnum.file;
+  text?: {
+    content: string;
+  };
+  file?: {
+    type: `${ChatFileTypeEnum}`;
+    name?: string;
+    url: string;
+  };
+};
+export type UserChatItemType = {
+  obj: ChatRoleEnum.Human;
+  value: UserChatItemValueItemType[];
+};
+export type SystemChatItemValueItemType = {
+  type: ChatItemValueTypeEnum.text;
+  text?: {
+    content: string;
+  };
+};
+export type SystemChatItemType = {
+  obj: ChatRoleEnum.System;
+  value: SystemChatItemValueItemType[];
+};
+export type AIChatItemValueItemType = {
+  type: ChatItemValueTypeEnum.text | ChatItemValueTypeEnum.tool;
+  text?: {
+    content: string;
+  };
+  tools?: ToolModuleResponseItemType[];
+};
+export type AIChatItemType = {
+  obj: ChatRoleEnum.AI;
+  value: AIChatItemValueItemType[];
+  userGoodFeedback?: string;
+  userBadFeedback?: string;
+  customFeedbacks?: string[];
+  adminFeedback?: AdminFbkType;
+  [DispatchNodeResponseKeyEnum.nodeResponse]?: ChatHistoryItemResType[];
+};
+export type ChatItemValueItemType =
+  | UserChatItemValueItemType
+  | SystemChatItemValueItemType
+  | AIChatItemValueItemType;
+
+export type ChatItemSchema = (UserChatItemType | SystemChatItemType | AIChatItemType) & {
   dataId: string;
   chatId: string;
   userId: string;
@@ -38,13 +93,6 @@ export type ChatItemSchema = {
   tmbId: string;
   appId: string;
   time: Date;
-  obj: `${ChatRoleEnum}`;
-  value: string;
-  userGoodFeedback?: string;
-  userBadFeedback?: string;
-  customFeedbacks?: string[];
-  adminFeedback?: AdminFbkType;
-  [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
 
 export type AdminFbkType = {
@@ -56,22 +104,16 @@ export type AdminFbkType = {
 };
 
 /* --------- chat item ---------- */
-export type ChatItemType = {
+export type ChatItemType = (UserChatItemType | SystemChatItemType | AIChatItemType) & {
   dataId?: string;
-  obj: ChatItemSchema['obj'];
-  value: any;
-  userGoodFeedback?: string;
-  userBadFeedback?: string;
-  customFeedbacks?: ChatItemSchema['customFeedbacks'];
-  adminFeedback?: ChatItemSchema['feedback'];
-  [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
 
-export type ChatSiteItemType = ChatItemType & {
+export type ChatSiteItemType = (UserChatItemType | SystemChatItemType | AIChatItemType) & {
+  dataId?: string;
   status: `${ChatStatusEnum}`;
   moduleName?: string;
   ttsBuffer?: Uint8Array;
-};
+} & ChatBoxInputType;
 
 /* --------- team chat --------- */
 export type ChatAppListSchema = {
@@ -93,60 +135,25 @@ export type ChatHistoryItemType = HistoryItemType & {
 };
 
 /* ------- response data ------------ */
-export type moduleDispatchResType = {
-  // common
-  moduleLogo?: string;
-  runningTime?: number;
-  query?: string;
-  textOutput?: string;
-
-  // bill
-  tokens?: number;
-  model?: string;
-  contextTotalLen?: number;
-  totalPoints?: number;
-
-  // chat
-  temperature?: number;
-  maxToken?: number;
-  quoteList?: SearchDataResponseItemType[];
-  historyPreview?: ChatItemType[]; // completion context array. history will slice
-
-  // dataset search
-  similarity?: number;
-  limit?: number;
-  searchMode?: `${DatasetSearchModeEnum}`;
-  searchUsingReRank?: boolean;
-  extensionModel?: string;
-  extensionResult?: string;
-  extensionTokens?: number;
-
-  // cq
-  cqList?: ClassifyQuestionAgentItemType[];
-  cqResult?: string;
-
-  // content extract
-  extractDescription?: string;
-  extractResult?: Record<string, any>;
-
-  // http
-  params?: Record<string, any>;
-  body?: Record<string, any>;
-  headers?: Record<string, any>;
-  httpResult?: Record<string, any>;
-
-  // plugin output
-  pluginOutput?: Record<string, any>;
-  pluginDetail?: ChatHistoryItemResType[];
-
-  // tf switch
-  tfSwitchResult?: boolean;
-
-  // abandon
-  tokens?: number;
-};
-
-export type ChatHistoryItemResType = moduleDispatchResType & {
+export type ChatHistoryItemResType = DispatchNodeResponseType & {
   moduleType: `${FlowNodeTypeEnum}`;
   moduleName: string;
 };
+
+/* One tool run response  */
+export type ToolRunResponseItemType = Record<string, any> | Array;
+/* tool module response */
+export type ToolModuleResponseItemType = {
+  id: string;
+  toolName: string; // tool name
+  toolAvatar: string;
+  params: string; // tool params
+  response: string;
+  functionName: string;
+};
+
+/* dispatch run time */
+export type RuntimeUserPromptType = {
+  files?: UserChatItemValueItemType['file'][];
+  text: string;
+};

packages/global/core/chat/utils.ts

@@ -1,6 +1,79 @@
-import { IMG_BLOCK_KEY, FILE_BLOCK_KEY } from './constants';
+import { DispatchNodeResponseType } from '../module/runtime/type';
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../module/node/constant';
+import { ChatItemValueTypeEnum, ChatRoleEnum } from './constants';
+import { ChatHistoryItemResType, ChatItemType } from './type.d';
 
-export function chatContentReplaceBlock(content: string = '') {
-  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
-  return content.replace(regex, '').trim();
-}
+export const getChatTitleFromChatMessage = (message?: ChatItemType, defaultValue = '新对话') => {
+  // @ts-ignore
+  const textMsg = message?.value.find((item) => item.type === ChatItemValueTypeEnum.text);
+
+  if (textMsg?.text?.content) {
+    return textMsg.text.content.slice(0, 20);
+  }
+
+  return defaultValue;
+};
+
+export const getHistoryPreview = (
+  completeMessages: ChatItemType[]
+): {
+  obj: `${ChatRoleEnum}`;
+  value: string;
+}[] => {
+  return completeMessages.map((item, i) => {
+    if (item.obj === ChatRoleEnum.System || i >= completeMessages.length - 2) {
+      return {
+        obj: item.obj,
+        value: item.value?.[0]?.text?.content || ''
+      };
+    }
+
+    const content = item.value
+      .map((item) => {
+        if (item.text?.content) {
+          const content =
+            item.text.content.length > 20
+              ? `${item.text.content.slice(0, 20)}...`
+              : item.text.content;
+          return content;
+        }
+        return '';
+      })
+      .filter(Boolean)
+      .join('\n');
+
+    return {
+      obj: item.obj,
+      value: content
+    };
+  });
+};
+
+export const filterPublicNodeResponseData = ({
+  flowResponses = []
+}: {
+  flowResponses?: ChatHistoryItemResType[];
+}) => {
+  const filedList = ['quoteList', 'moduleType'];
+  const filterModuleTypeList: any[] = [
+    FlowNodeTypeEnum.pluginModule,
+    FlowNodeTypeEnum.datasetSearchNode,
+    FlowNodeTypeEnum.tools
+  ];
+
+  return flowResponses
+    .filter((item) => filterModuleTypeList.includes(item.moduleType))
+    .map((item) => {
+      const obj: DispatchNodeResponseType = {};
+      for (let key in item) {
+        if (key === 'toolDetail' || key === 'pluginDetail') {
+          // @ts-ignore
+          obj[key] = filterPublicNodeResponseData({ flowResponses: item[key] });
+        } else if (filedList.includes(key)) {
+          // @ts-ignore
+          obj[key] = item[key];
+        }
+      }
+      return obj as ChatHistoryItemResType;
+    });
+};

packages/global/core/dataset/constants.ts

@@ -83,17 +83,17 @@ export const TrainingTypeMap = {
   [TrainingModeEnum.chunk]: {
     label: 'core.dataset.training.Chunk mode',
     tooltip: 'core.dataset.import.Chunk Split Tip',
-    isPlus: true
+    openSource: true
   },
   [TrainingModeEnum.auto]: {
     label: 'core.dataset.training.Auto mode',
     tooltip: 'core.dataset.training.Auto mode Tip',
-    isPlus: true
+    openSource: false
   },
   [TrainingModeEnum.qa]: {
     label: 'core.dataset.training.QA mode',
     tooltip: 'core.dataset.import.QA Import Tip',
-    isPlus: true
+    openSource: true
   }
 };
 

packages/global/core/module/constants.ts

@@ -21,7 +21,10 @@ export enum ModuleIOValueTypeEnum {
 
   // plugin special type
   selectApp = 'selectApp',
-  selectDataset = 'selectDataset'
+  selectDataset = 'selectDataset',
+
+  // tool
+  tools = 'tools'
 }
 
 /* reg: modulename key */
@@ -89,12 +92,10 @@ export enum ModuleInputKeyEnum {
 
 export enum ModuleOutputKeyEnum {
   // common
-  responseData = 'responseData',
-  moduleDispatchBills = 'moduleDispatchBills',
   userChatInput = 'userChatInput',
   finish = 'finish',
   history = 'history',
-  answerText = 'answerText', //  answer module text key
+  answerText = 'answerText', // module answer. the value will be show and save to history
   success = 'success',
   failed = 'failed',
   text = 'system_text',
@@ -110,7 +111,13 @@ export enum ModuleOutputKeyEnum {
 
   // tf switch
   resultTrue = 'system_resultTrue',
-  resultFalse = 'system_resultFalse'
+  resultFalse = 'system_resultFalse',
+
+  // tools
+  selectedTools = 'selectedTools',
+
+  // http
+  httpRawResponse = 'httpRawResponse'
 }
 
 export enum VariableInputEnum {

packages/global/core/module/node/constant.ts

@@ -56,7 +56,8 @@ export enum FlowNodeTypeEnum {
   pluginModule = 'pluginModule',
   pluginInput = 'pluginInput',
   pluginOutput = 'pluginOutput',
-  queryExtension = 'cfr'
+  queryExtension = 'cfr',
+  tools = 'tools'
 
   // abandon
 }

packages/global/core/module/node/type.d.ts

@@ -2,6 +2,7 @@ import { FlowNodeInputTypeEnum, FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from
 import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, ModuleOutputKeyEnum } from '../constants';
 import { SelectedDatasetType } from '../api';
 import { EditInputFieldMap, EditOutputFieldMap } from './type';
+import { LLMModelTypeEnum } from '../../ai/constants';
 
 export type FlowNodeChangeProps = {
   moduleId: string;
@@ -28,6 +29,7 @@ export type FlowNodeInputItemType = {
   label: string;
   description?: string;
   required?: boolean;
+  toolDescription?: string; // If this field is not empty, it is entered as a tool
 
   edit?: boolean; // Whether to allow editing
   editField?: EditInputFieldMap;
@@ -49,6 +51,8 @@ export type FlowNodeInputItemType = {
   step?: number; // slider
   max?: number; // slider, number input
   min?: number; // slider, number input
+
+  llmModelType?: `${LLMModelTypeEnum}`;
 };
 
 export type FlowNodeOutputTargetItemType = {
@@ -62,6 +66,8 @@ export type FlowNodeOutputItemType = {
 
   label?: string;
   description?: string;
+  required?: boolean;
+  defaultValue?: any;
 
   edit?: boolean;
   editField?: EditOutputFieldMap;
@@ -74,12 +80,14 @@ export type FlowNodeOutputItemType = {
 export type EditInputFieldMap = EditOutputFieldMap & {
   inputType?: boolean;
   required?: boolean;
+  isToolInput?: boolean;
 };
 export type EditOutputFieldMap = {
   name?: boolean;
   key?: boolean;
   description?: boolean;
   dataType?: boolean;
+  defaultValue?: boolean;
 };
 export type EditNodeFieldType = {
   inputType?: `${FlowNodeInputTypeEnum}`; // input type
@@ -89,6 +97,8 @@ export type EditNodeFieldType = {
   label?: string;
   description?: string;
   valueType?: `${ModuleIOValueTypeEnum}`;
+  isToolInput?: boolean;
+  defaultValue?: string;
 };
 
 /* ------------- item type --------------- */

packages/global/core/module/runtime/constants.ts

@@ -0,0 +1,19 @@
+export enum SseResponseEventEnum {
+  error = 'error',
+  answer = 'answer', // animation stream
+  fastAnswer = 'fastAnswer', // direct answer text, not animation
+  flowNodeStatus = 'flowNodeStatus', // update node status
+
+  toolCall = 'toolCall', // tool start
+  toolParams = 'toolParams', // tool params return
+  toolResponse = 'toolResponse', // tool response return
+  flowResponses = 'flowResponses' // sse response request
+}
+
+export enum DispatchNodeResponseKeyEnum {
+  nodeResponse = 'responseData', // run node response
+  nodeDispatchUsages = 'nodeDispatchUsages', // the node bill.
+  childrenResponses = 'childrenResponses', // Some nodes make recursive calls that need to be returned
+  toolResponses = 'toolResponses', // The result is passed back to the tool node for use
+  assistantResponses = 'assistantResponses' // assistant response
+}

packages/global/core/module/runtime/type.d.ts

@@ -0,0 +1,101 @@
+import { ChatNodeUsageType } from '../../../support/wallet/bill/type';
+import { ChatItemValueItemType, ToolRunResponseItemType } from '../../chat/type';
+import { FlowNodeInputItemType, FlowNodeOutputItemType } from '../node/type';
+import { ModuleItemType } from '../type';
+import { DispatchNodeResponseKeyEnum } from './constants';
+
+export type RunningModuleItemType = {
+  name: ModuleItemType['name'];
+  avatar: ModuleItemType['avatar'];
+  intro?: ModuleItemType['intro'];
+  moduleId: ModuleItemType['moduleId'];
+  flowType: ModuleItemType['flowType'];
+  showStatus?: ModuleItemType['showStatus'];
+  isEntry?: ModuleItemType['isEntry'];
+
+  inputs: {
+    key: string;
+    value?: any;
+    valueType?: FlowNodeInputItemType['valueType'];
+    required?: boolean;
+    toolDescription?: string;
+  }[];
+  outputs: {
+    key: string;
+    required?: boolean;
+    defaultValue?: any;
+    answer?: boolean;
+    response?: boolean;
+    value?: any;
+    valueType?: FlowNodeOutputItemType['valueType'];
+    targets: {
+      moduleId: string;
+      key: string;
+    }[];
+  }[];
+};
+
+export type DispatchNodeResponseType = {
+  // common
+  moduleLogo?: string;
+  runningTime?: number;
+  query?: string;
+  textOutput?: string;
+
+  // bill
+  tokens?: number;
+  model?: string;
+  contextTotalLen?: number;
+  totalPoints?: number;
+
+  // chat
+  temperature?: number;
+  maxToken?: number;
+  quoteList?: SearchDataResponseItemType[];
+  historyPreview?: {
+    obj: `${ChatRoleEnum}`;
+    value: string;
+  }[]; // completion context array. history will slice
+
+  // dataset search
+  similarity?: number;
+  limit?: number;
+  searchMode?: `${DatasetSearchModeEnum}`;
+  searchUsingReRank?: boolean;
+  extensionModel?: string;
+  extensionResult?: string;
+  extensionTokens?: number;
+
+  // cq
+  cqList?: ClassifyQuestionAgentItemType[];
+  cqResult?: string;
+
+  // content extract
+  extractDescription?: string;
+  extractResult?: Record<string, any>;
+
+  // http
+  params?: Record<string, any>;
+  body?: Record<string, any>;
+  headers?: Record<string, any>;
+  httpResult?: Record<string, any>;
+
+  // plugin output
+  pluginOutput?: Record<string, any>;
+  pluginDetail?: ChatHistoryItemResType[];
+
+  // tf switch
+  tfSwitchResult?: boolean;
+
+  // tool
+  toolCallTokens?: number;
+  toolDetail?: ChatHistoryItemResType[];
+};
+
+export type DispatchNodeResultType<T> = {
+  [DispatchNodeResponseKeyEnum.nodeResponse]?: DispatchNodeResponseType; // The node response detail
+  [DispatchNodeResponseKeyEnum.nodeDispatchUsages]?: ChatNodeUsageType[]; //
+  [DispatchNodeResponseKeyEnum.childrenResponses]?: DispatchNodeResultType[];
+  [DispatchNodeResponseKeyEnum.toolResponses]?: ToolRunResponseItemType;
+  [DispatchNodeResponseKeyEnum.assistantResponses]?: ChatItemValueItemType[];
+} & T;

packages/global/core/module/runtime/utils.ts

@@ -0,0 +1,31 @@
+import { ChatCompletionRequestMessageRoleEnum } from '../../ai/constants';
+
+export const textAdaptGptResponse = ({
+  text,
+  model = '',
+  finish_reason = null,
+  extraData = {}
+}: {
+  model?: string;
+  text: string | null;
+  finish_reason?: null | 'stop';
+  extraData?: Object;
+}) => {
+  return JSON.stringify({
+    ...extraData,
+    id: '',
+    object: '',
+    created: 0,
+    model,
+    choices: [
+      {
+        delta:
+          text === null
+            ? {}
+            : { role: ChatCompletionRequestMessageRoleEnum.Assistant, content: text },
+        index: 0,
+        finish_reason
+      }
+    ]
+  });
+};

packages/global/core/module/template/constants.ts

@@ -0,0 +1,115 @@
+import { UserGuideModule } from './system/userGuide';
+import { UserInputModule } from './system/userInput';
+import { AiChatModule } from './system/aiChat';
+import { DatasetSearchModule } from './system/datasetSearch';
+import { DatasetConcatModule } from './system/datasetConcat';
+import { AssignedAnswerModule } from './system/assignedAnswer';
+import { ClassifyQuestionModule } from './system/classifyQuestion';
+import { ContextExtractModule } from './system/contextExtract';
+import { HttpModule468 } from './system/http468';
+import { HttpModule } from './system/abandon/http';
+import { ToolModule } from './system/tools';
+
+import { RunAppModule } from './system/runApp';
+import { PluginInputModule } from './system/pluginInput';
+import { PluginOutputModule } from './system/pluginOutput';
+import { RunPluginModule } from './system/runPlugin';
+import { AiQueryExtension } from './system/queryExtension';
+
+import type { FlowModuleTemplateType, moduleTemplateListType } from '../../module/type.d';
+import { ModuleTemplateTypeEnum } from '../../module/constants';
+
+/* app flow module templates */
+export const appSystemModuleTemplates: FlowModuleTemplateType[] = [
+  UserGuideModule,
+  UserInputModule,
+  AiChatModule,
+  AssignedAnswerModule,
+  DatasetSearchModule,
+  DatasetConcatModule,
+  RunAppModule,
+  ToolModule,
+  ClassifyQuestionModule,
+  ContextExtractModule,
+  HttpModule468,
+  AiQueryExtension
+];
+/* plugin flow module templates */
+export const pluginSystemModuleTemplates: FlowModuleTemplateType[] = [
+  PluginInputModule,
+  PluginOutputModule,
+  AiChatModule,
+  AssignedAnswerModule,
+  DatasetSearchModule,
+  DatasetConcatModule,
+  RunAppModule,
+  ToolModule,
+  ClassifyQuestionModule,
+  ContextExtractModule,
+  HttpModule468,
+  AiQueryExtension
+];
+
+/* all module */
+export const moduleTemplatesFlat: FlowModuleTemplateType[] = [
+  UserGuideModule,
+  UserInputModule,
+  AiChatModule,
+  DatasetSearchModule,
+  DatasetConcatModule,
+  AssignedAnswerModule,
+  ClassifyQuestionModule,
+  ContextExtractModule,
+  HttpModule468,
+  HttpModule,
+  ToolModule,
+  AiChatModule,
+  RunAppModule,
+  PluginInputModule,
+  PluginOutputModule,
+  RunPluginModule,
+  AiQueryExtension
+];
+
+export const moduleTemplatesList: moduleTemplateListType = [
+  {
+    type: ModuleTemplateTypeEnum.userGuide,
+    label: 'core.module.template.Guide module',
+    list: []
+  },
+  {
+    type: ModuleTemplateTypeEnum.systemInput,
+    label: 'core.module.template.System input module',
+    list: []
+  },
+  {
+    type: ModuleTemplateTypeEnum.textAnswer,
+    label: 'core.module.template.Response module',
+    list: []
+  },
+  {
+    type: ModuleTemplateTypeEnum.functionCall,
+    label: 'core.module.template.Function module',
+    list: []
+  },
+  {
+    type: ModuleTemplateTypeEnum.tools,
+    label: 'core.module.template.Tool module',
+    list: []
+  },
+  {
+    type: ModuleTemplateTypeEnum.externalCall,
+    label: 'core.module.template.External module',
+    list: []
+  },
+  {
+    type: ModuleTemplateTypeEnum.personalPlugin,
+    label: 'core.module.template.My plugin module',
+    list: []
+  },
+  {
+    type: ModuleTemplateTypeEnum.other,
+    label: '其他',
+    list: []
+  }
+];

packages/global/core/module/template/input.ts

@@ -2,6 +2,7 @@ import type { FlowNodeInputItemType } from '../node/type.d';
 import { DYNAMIC_INPUT_KEY, ModuleInputKeyEnum } from '../constants';
 import { FlowNodeInputTypeEnum } from '../node/constant';
 import { ModuleIOValueTypeEnum } from '../constants';
+import { chatNodeSystemPromptTip } from './tip';
 
 export const Input_Template_Switch: FlowNodeInputItemType = {
   key: ModuleInputKeyEnum.switch,
@@ -58,6 +59,28 @@ export const Input_Template_DynamicInput: FlowNodeInputItemType = {
   hideInApp: true
 };
 
+export const Input_Template_AiModel: FlowNodeInputItemType = {
+  key: ModuleInputKeyEnum.aiModel,
+  type: FlowNodeInputTypeEnum.selectLLMModel,
+  label: 'core.module.input.label.aiModel',
+  required: true,
+  valueType: ModuleIOValueTypeEnum.string,
+  showTargetInApp: false,
+  showTargetInPlugin: false
+};
+
+export const Input_Template_System_Prompt: FlowNodeInputItemType = {
+  key: ModuleInputKeyEnum.aiSystemPrompt,
+  type: FlowNodeInputTypeEnum.textarea,
+  max: 3000,
+  valueType: ModuleIOValueTypeEnum.string,
+  label: 'core.ai.Prompt',
+  description: chatNodeSystemPromptTip,
+  placeholder: chatNodeSystemPromptTip,
+  showTargetInApp: true,
+  showTargetInPlugin: true
+};
+
 export const Input_Template_Dataset_Quote: FlowNodeInputItemType = {
   key: ModuleInputKeyEnum.aiChatDatasetQuote,
   type: FlowNodeInputTypeEnum.target,

packages/global/core/module/template/system/aiChat.ts

@@ -11,9 +11,11 @@ import {
   ModuleTemplateTypeEnum
 } from '../../constants';
 import {
+  Input_Template_AiModel,
   Input_Template_Dataset_Quote,
   Input_Template_History,
   Input_Template_Switch,
+  Input_Template_System_Prompt,
   Input_Template_UserChatInput
 } from '../input';
 import { chatNodeSystemPromptTip } from '../tip';
@@ -24,20 +26,13 @@ export const AiChatModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.textAnswer,
   flowType: FlowNodeTypeEnum.chatNode,
   avatar: '/imgs/module/AI.png',
-  name: 'core.module.template.Ai chat',
-  intro: 'core.module.template.Ai chat intro',
+  name: 'AI 对话',
+  intro: 'AI 大模型对话',
   showStatus: true,
+  // isTool: true,
   inputs: [
     Input_Template_Switch,
-    {
-      key: ModuleInputKeyEnum.aiModel,
-      type: FlowNodeInputTypeEnum.selectLLMModel,
-      label: 'core.module.input.label.aiModel',
-      required: true,
-      valueType: ModuleIOValueTypeEnum.string,
-      showTargetInApp: false,
-      showTargetInPlugin: false
-    },
+    Input_Template_AiModel,
     // --- settings modal
     {
       key: ModuleInputKeyEnum.aiChatTemperature,
@@ -98,18 +93,13 @@ export const AiChatModule: FlowModuleTemplateType = {
     },
     // settings modal ---
     {
-      key: ModuleInputKeyEnum.aiSystemPrompt,
-      type: FlowNodeInputTypeEnum.textarea,
+      ...Input_Template_System_Prompt,
       label: 'core.ai.Prompt',
-      max: 300,
-      valueType: ModuleIOValueTypeEnum.string,
       description: chatNodeSystemPromptTip,
-      placeholder: chatNodeSystemPromptTip,
-      showTargetInApp: true,
-      showTargetInPlugin: true
+      placeholder: chatNodeSystemPromptTip
     },
     Input_Template_History,
-    Input_Template_UserChatInput,
+    { ...Input_Template_UserChatInput, toolDescription: '用户问题' },
     Input_Template_Dataset_Quote
   ],
   outputs: [

packages/global/core/module/template/system/assignedAnswer.ts

@@ -9,8 +9,9 @@ export const AssignedAnswerModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.textAnswer,
   flowType: FlowNodeTypeEnum.answerNode,
   avatar: '/imgs/module/reply.png',
-  name: 'core.module.template.Assigned reply',
-  intro: 'core.module.template.Assigned reply intro',
+  name: '指定回复',
+  intro:
+    '该模块可以直接回复一段指定的内容。常用于引导、提示。非字符串内容传入时，会转成字符串进行输出。',
   inputs: [
     Input_Template_Switch,
     {

packages/global/core/module/template/system/classifyQuestion.ts

@@ -6,40 +6,34 @@ import {
 import { FlowModuleTemplateType } from '../../type.d';
 import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
 import {
+  Input_Template_AiModel,
   Input_Template_History,
   Input_Template_Switch,
   Input_Template_UserChatInput
 } from '../input';
 import { Output_Template_UserChatInput } from '../output';
+import { Input_Template_System_Prompt } from '../input';
+import { LLMModelTypeEnum } from '../../../ai/constants';
 
 export const ClassifyQuestionModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.classifyQuestion,
   templateType: ModuleTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.classifyQuestion,
   avatar: '/imgs/module/cq.png',
-  name: 'core.module.template.Classify question',
-  intro: `core.module.template.Classify question intro`,
+  name: '问题分类',
+  intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：\n类型1: 打招呼\n类型2: 关于商品“使用”问题\n类型3: 关于商品“购买”问题\n类型4: 其他问题`,
   showStatus: true,
   inputs: [
     Input_Template_Switch,
     {
-      key: ModuleInputKeyEnum.aiModel,
-      type: FlowNodeInputTypeEnum.selectLLMModel,
-      valueType: ModuleIOValueTypeEnum.string,
-      label: 'core.module.input.label.Classify model',
-      required: true,
-      showTargetInApp: false,
-      showTargetInPlugin: false
+      ...Input_Template_AiModel,
+      llmModelType: LLMModelTypeEnum.classify
     },
     {
-      key: ModuleInputKeyEnum.aiSystemPrompt,
-      type: FlowNodeInputTypeEnum.textarea,
-      valueType: ModuleIOValueTypeEnum.string,
+      ...Input_Template_System_Prompt,
       label: 'core.module.input.label.Background',
       description: 'core.module.input.description.Background',
-      placeholder: 'core.module.input.placeholder.Classify background',
-      showTargetInApp: true,
-      showTargetInPlugin: true
+      placeholder: 'core.module.input.placeholder.Classify background'
     },
     Input_Template_History,
     Input_Template_UserChatInput,

packages/global/core/module/template/system/contextExtract.ts

@@ -10,26 +10,23 @@ import {
   ModuleOutputKeyEnum,
   ModuleTemplateTypeEnum
 } from '../../constants';
-import { Input_Template_History, Input_Template_Switch } from '../input';
+import { Input_Template_AiModel, Input_Template_History, Input_Template_Switch } from '../input';
+import { LLMModelTypeEnum } from '../../../ai/constants';
 
 export const ContextExtractModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.contentExtract,
   templateType: ModuleTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.contentExtract,
   avatar: '/imgs/module/extract.png',
-  name: 'core.module.template.Extract field',
-  intro: 'core.module.template.Extract field intro',
+  name: '文本内容提取',
+  intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等',
   showStatus: true,
+  isTool: true,
   inputs: [
     Input_Template_Switch,
     {
-      key: ModuleInputKeyEnum.aiModel,
-      type: FlowNodeInputTypeEnum.selectLLMModel,
-      valueType: ModuleIOValueTypeEnum.string,
-      label: 'core.module.input.label.LLM',
-      required: true,
-      showTargetInApp: false,
-      showTargetInPlugin: false
+      ...Input_Template_AiModel,
+      llmModelType: LLMModelTypeEnum.extractFields
     },
     {
       key: ModuleInputKeyEnum.description,
@@ -52,7 +49,8 @@ export const ContextExtractModule: FlowModuleTemplateType = {
       required: true,
       valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: true,
-      showTargetInPlugin: true
+      showTargetInPlugin: true,
+      toolDescription: '需要检索的内容'
     },
     {
       key: ModuleInputKeyEnum.extractKeys,

packages/global/core/module/template/system/datasetConcat.ts

@@ -26,7 +26,7 @@ export const DatasetConcatModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.tools,
   avatar: '/imgs/module/concat.svg',
   name: '知识库搜索引用合并',
-  intro: 'core.module.template.Dataset search result concat intro',
+  intro: '可以将多个知识库搜索结果进行合并输出。使用 RRF 的合并方式进行最终排序输出。',
   showStatus: false,
   inputs: [
     Input_Template_Switch,

packages/global/core/module/template/system/datasetSearch.ts

@@ -19,9 +19,10 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.datasetSearchNode,
   avatar: '/imgs/module/db.png',
-  name: 'core.module.template.Dataset search',
-  intro: 'core.module.template.Dataset search intro',
+  name: '知识库搜索',
+  intro: '调用知识库搜索能力，查找“有可能”与问题相关的内容',
   showStatus: true,
+  isTool: true,
   inputs: [
     Input_Template_Switch,
     {
@@ -97,7 +98,10 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       showTargetInPlugin: false,
       value: ''
     },
-    Input_Template_UserChatInput
+    {
+      ...Input_Template_UserChatInput,
+      toolDescription: '需要检索的内容'
+    }
   ],
   outputs: [
     Output_Template_UserChatInput,

packages/global/core/module/template/system/http468.ts

@@ -5,9 +5,9 @@ import {
 } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type';
 import {
-  DYNAMIC_INPUT_KEY,
   ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
+  ModuleOutputKeyEnum,
   ModuleTemplateTypeEnum
 } from '../../constants';
 import {
@@ -22,9 +22,10 @@ export const HttpModule468: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.externalCall,
   flowType: FlowNodeTypeEnum.httpRequest468,
   avatar: '/imgs/module/http.png',
-  name: 'core.module.template.Http request',
-  intro: 'core.module.template.Http request intro',
+  name: 'HTTP 请求',
+  intro: '可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）',
   showStatus: true,
+  isTool: true,
   inputs: [
     Input_Template_Switch,
     {
@@ -86,7 +87,6 @@ export const HttpModule468: FlowModuleTemplateType = {
       editField: {
         key: true,
         description: true,
-        required: true,
         dataType: true
       },
       defaultEditField: {
@@ -94,19 +94,27 @@ export const HttpModule468: FlowModuleTemplateType = {
         key: '',
         description: '',
         inputType: FlowNodeInputTypeEnum.target,
-        valueType: ModuleIOValueTypeEnum.string,
-        required: true
+        valueType: ModuleIOValueTypeEnum.string
       }
     }
   ],
   outputs: [
     Output_Template_Finish,
+    {
+      key: ModuleOutputKeyEnum.httpRawResponse,
+      label: '原始响应',
+      description: 'HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。',
+      valueType: ModuleIOValueTypeEnum.any,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    },
     {
       ...Output_Template_AddOutput,
       editField: {
         key: true,
         description: true,
-        dataType: true
+        dataType: true,
+        defaultValue: true
       },
       defaultEditField: {
         label: '',

packages/global/core/module/template/system/queryExtension.ts

@@ -13,28 +13,26 @@ import {
 import {
   Input_Template_History,
   Input_Template_Switch,
-  Input_Template_UserChatInput
+  Input_Template_UserChatInput,
+  Input_Template_AiModel
 } from '../input';
 import { Output_Template_UserChatInput } from '../output';
+import { LLMModelTypeEnum } from '../../../ai/constants';
 
 export const AiQueryExtension: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.chatNode,
   templateType: ModuleTemplateTypeEnum.other,
   flowType: FlowNodeTypeEnum.queryExtension,
   avatar: '/imgs/module/cfr.svg',
-  name: 'core.module.template.Query extension',
-  intro: 'core.module.template.Query extension intro',
+  name: '问题优化',
+  intro:
+    '使用问题优化功能，可以提高知识库连续对话时搜索的精度。使用该功能后，会先利用 AI 根据上下文构建一个或多个新的检索词，这些检索词更利于进行知识库搜索。该模块已内置在知识库搜索模块中，如果您仅进行一次知识库搜索，可直接使用知识库内置的补全功能。',
   showStatus: true,
   inputs: [
     Input_Template_Switch,
     {
-      key: ModuleInputKeyEnum.aiModel,
-      type: FlowNodeInputTypeEnum.selectLLMModel,
-      label: 'core.module.input.label.aiModel',
-      required: true,
-      valueType: ModuleIOValueTypeEnum.string,
-      showTargetInApp: false,
-      showTargetInPlugin: false
+      ...Input_Template_AiModel,
+      llmModelType: LLMModelTypeEnum.queryExtension
     },
     {
       key: ModuleInputKeyEnum.aiSystemPrompt,

packages/global/core/module/template/system/runApp.ts

@@ -22,8 +22,8 @@ export const RunAppModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.externalCall,
   flowType: FlowNodeTypeEnum.runApp,
   avatar: '/imgs/module/app.png',
-  name: 'core.module.template.Running app',
-  intro: 'core.module.template.Running app intro',
+  name: '应用调用',
+  intro: '可以选择一个其他应用进行调用',
   showStatus: true,
   inputs: [
     Input_Template_Switch,
@@ -52,7 +52,7 @@ export const RunAppModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleOutputKeyEnum.answerText,
-      label: 'AI回复',
+      label: '回复的文本',
       description: '将在应用完全结束后触发',
       valueType: ModuleIOValueTypeEnum.string,
       type: FlowNodeOutputTypeEnum.source,

packages/global/core/module/template/system/runPlugin.ts

@@ -9,6 +9,7 @@ export const RunPluginModule: FlowModuleTemplateType = {
   intro: '',
   name: '',
   showStatus: false,
+  isTool: true,
   inputs: [], // [{key:'pluginId'},...]
   outputs: []
 };

packages/global/core/module/template/system/tools.ts

@@ -0,0 +1,52 @@
+import { FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import {
+  ModuleIOValueTypeEnum,
+  ModuleOutputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../constants';
+import {
+  Input_Template_AiModel,
+  Input_Template_History,
+  Input_Template_Switch,
+  Input_Template_System_Prompt,
+  Input_Template_UserChatInput
+} from '../input';
+import { chatNodeSystemPromptTip } from '../tip';
+import { Output_Template_Finish, Output_Template_UserChatInput } from '../output';
+import { LLMModelTypeEnum } from '../../../ai/constants';
+
+export const ToolModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.tools,
+  flowType: FlowNodeTypeEnum.tools,
+  templateType: ModuleTemplateTypeEnum.functionCall,
+  avatar: '/imgs/module/tool.svg',
+  name: '工具调用（实验）',
+  intro: '通过AI模型自动选择一个或多个工具进行调用。工具可以是其他功能块或插件。',
+  showStatus: true,
+  inputs: [
+    Input_Template_Switch,
+    {
+      ...Input_Template_AiModel,
+      llmModelType: LLMModelTypeEnum.toolCall
+    },
+    {
+      ...Input_Template_System_Prompt,
+      label: 'core.ai.Prompt',
+      description: chatNodeSystemPromptTip,
+      placeholder: chatNodeSystemPromptTip
+    },
+    Input_Template_History,
+    Input_Template_UserChatInput
+  ],
+  outputs: [
+    Output_Template_UserChatInput,
+    {
+      key: ModuleOutputKeyEnum.selectedTools,
+      valueType: ModuleIOValueTypeEnum.tools,
+      type: FlowNodeOutputTypeEnum.hidden,
+      targets: []
+    },
+    Output_Template_Finish
+  ]
+};

packages/global/core/module/template/system/userGuide.ts

@@ -8,7 +8,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.userGuide,
   flowType: FlowNodeTypeEnum.userGuide,
   avatar: '/imgs/module/userGuide.png',
-  name: 'core.module.template.User guide',
+  name: '全局配置',
   intro: userGuideTip,
   inputs: [
     {

packages/global/core/module/template/system/userInput.ts

@@ -16,8 +16,8 @@ export const UserInputModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.systemInput,
   flowType: FlowNodeTypeEnum.questionInput,
   avatar: '/imgs/module/userChatInput.svg',
-  name: 'core.module.template.Chat entrance',
-  intro: 'core.module.template.Chat entrance intro',
+  name: '对话入口',
+  intro: '当用户发送一个内容后，流程将会从这个模块开始执行。',
   inputs: [
     {
       key: ModuleInputKeyEnum.userChatInput,

packages/global/core/module/type.d.ts

@@ -5,10 +5,16 @@ import {
   ModuleTemplateTypeEnum,
   VariableInputEnum
 } from './constants';
+import { DispatchNodeResponseKeyEnum } from './runtime/constants';
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 import { UserModelSchema } from 'support/user/type';
-import { moduleDispatchResType } from '..//chat/type';
-import { ChatModuleUsageType } from '../../support/wallet/bill/type';
+import {
+  ChatItemValueItemType,
+  ToolRunResponseItemType,
+  UserChatItemValueItemType
+} from '../chat/type';
+import { ChatNodeUsageType } from '../../support/wallet/bill/type';
+import { RunningModuleItemType } from './runtime/type';
 
 export type FlowModuleTemplateType = {
   id: string; // module id, unique
@@ -17,6 +23,7 @@ export type FlowModuleTemplateType = {
   avatar?: string;
   name: string;
   intro: string; // template list intro
+  isTool?: boolean; // can be connected by tool
   showStatus?: boolean; // chatting response step status
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
@@ -44,6 +51,9 @@ export type ModuleItemType = {
   showStatus?: boolean;
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
+
+  // runTime field
+  isEntry?: boolean;
 };
 
 /* --------------- function type -------------------- */
@@ -85,30 +95,6 @@ export type ContextExtractAgentItemType = {
 };
 
 /* -------------- running module -------------- */
-export type RunningModuleItemType = {
-  name: ModuleItemType['name'];
-  moduleId: ModuleItemType['moduleId'];
-  flowType: ModuleItemType['flowType'];
-  showStatus?: ModuleItemType['showStatus'];
-} & {
-  inputs: {
-    key: string;
-    value?: any;
-    valueType?: `${ModuleIOValueTypeEnum}`;
-  }[];
-  outputs: {
-    key: string;
-    answer?: boolean;
-    response?: boolean;
-    value?: any;
-    valueType?: `${ModuleIOValueTypeEnum}`;
-    targets: {
-      moduleId: string;
-      key: string;
-    }[];
-  }[];
-};
-
 export type ChatDispatchProps = {
   res: NextApiResponse;
   mode: 'test' | 'chat';
@@ -120,15 +106,13 @@ export type ChatDispatchProps = {
   responseChatItemId?: string;
   histories: ChatItemType[];
   variables: Record<string, any>;
+  inputFiles?: UserChatItemValueItemType['file'][];
   stream: boolean;
   detail: boolean; // response detail
 };
 
 export type ModuleDispatchProps<T> = ChatDispatchProps & {
   module: RunningModuleItemType;
+  runtimeModules: RunningModuleItemType[];
   params: T;
 };
-export type ModuleDispatchResponse<T> = T & {
-  [ModuleOutputKeyEnum.responseData]?: moduleDispatchResType;
-  [ModuleOutputKeyEnum.moduleDispatchBills]?: ChatModuleUsageType[];
-};

packages/global/core/module/utils.ts

@@ -10,6 +10,7 @@ import { AppTTSConfigType, ModuleItemType, VariableItemType } from './type';
 import { Input_Template_Switch } from './template/input';
 import { EditorVariablePickerType } from '../../../web/components/common/Textarea/PromptEditor/type';
 
+/* module  */
 export const getGuideModule = (modules: ModuleItemType[]) =>
   modules.find((item) => item.flowType === FlowNodeTypeEnum.userGuide);
 
@@ -57,13 +58,13 @@ export const getModuleInputUiField = (input: FlowNodeInputItemType) => {
   return {};
 };
 
-export function plugin2ModuleIO(
+export const plugin2ModuleIO = (
   pluginId: string,
   modules: ModuleItemType[]
 ): {
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
-} {
+} => {
   const pluginInput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginInput);
   const pluginOutput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginOutput);
 
@@ -99,7 +100,7 @@ export function plugin2ModuleIO(
         }))
       : []
   };
-}
+};
 
 export const formatEditorVariablePickerIcon = (
   variables: { key: string; label: string; type?: `${VariableInputEnum}` }[]

packages/global/package.json

@@ -6,7 +6,7 @@
     "dayjs": "^1.11.7",
     "encoding": "^0.1.13",
     "js-tiktoken": "^1.0.7",
-    "openai": "4.23.0",
+    "openai": "4.28.0",
     "nanoid": "^4.0.1",
     "timezones-list": "^3.0.2"
   },

packages/global/support/outLink/api.d.ts

@@ -1,4 +1,4 @@
-import type { HistoryItemType, ChatSiteItemType } from '../../core/chat/type.d';
+import type { HistoryItemType } from '../../core/chat/type.d';
 import { OutLinkSchema } from './type.d';
 
 export type AuthOutLinkInitProps = {

packages/global/support/wallet/bill/type.d.ts

@@ -22,7 +22,7 @@ export type BillSchemaType = {
   username: string;
 };
 
-export type ChatModuleUsageType = {
+export type ChatNodeUsageType = {
   tokens?: number;
   totalPoints: number;
   moduleName: string;