V4.7-alpha (#985)

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/core/chat/adapt.ts

@@ -1,40 +1,298 @@
-import type { ChatItemType } from '../../core/chat/type.d';
-import { ChatRoleEnum } from '../../core/chat/constants';
-import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constant';
-import type { ChatMessageItemType } from '../../core/ai/type.d';
+import type {
+  ChatItemType,
+  ChatItemValueItemType,
+  RuntimeUserPromptType,
+  UserChatItemType
+} from '../../core/chat/type.d';
+import { ChatFileTypeEnum, ChatItemValueTypeEnum, ChatRoleEnum } from '../../core/chat/constants';
+import type {
+  ChatCompletionContentPart,
+  ChatCompletionFunctionMessageParam,
+  ChatCompletionMessageFunctionCall,
+  ChatCompletionMessageParam,
+  ChatCompletionMessageToolCall,
+  ChatCompletionToolMessageParam
+} from '../../core/ai/type.d';
+import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constants';
 
-const chat2Message = {
-  [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,
-  [ChatRoleEnum.Human]: ChatCompletionRequestMessageRoleEnum.User,
-  [ChatRoleEnum.System]: ChatCompletionRequestMessageRoleEnum.System,
-  [ChatRoleEnum.Function]: ChatCompletionRequestMessageRoleEnum.Function,
-  [ChatRoleEnum.Tool]: ChatCompletionRequestMessageRoleEnum.Tool
-};
-const message2Chat = {
+const GPT2Chat = {
   [ChatCompletionRequestMessageRoleEnum.System]: ChatRoleEnum.System,
   [ChatCompletionRequestMessageRoleEnum.User]: ChatRoleEnum.Human,
   [ChatCompletionRequestMessageRoleEnum.Assistant]: ChatRoleEnum.AI,
-  [ChatCompletionRequestMessageRoleEnum.Function]: ChatRoleEnum.Function,
-  [ChatCompletionRequestMessageRoleEnum.Tool]: ChatRoleEnum.Tool
+  [ChatCompletionRequestMessageRoleEnum.Function]: ChatRoleEnum.AI,
+  [ChatCompletionRequestMessageRoleEnum.Tool]: ChatRoleEnum.AI
 };
 
-export function adaptRole_Chat2Message(role: `${ChatRoleEnum}`) {
-  return chat2Message[role];
-}
 export function adaptRole_Message2Chat(role: `${ChatCompletionRequestMessageRoleEnum}`) {
-  return message2Chat[role];
+  return GPT2Chat[role];
 }
 
-export const adaptChat2GptMessages = ({
+export const simpleUserContentPart = (content: ChatCompletionContentPart[]) => {
+  if (content.length === 1 && content[0].type === 'text') {
+    return content[0].text;
+  }
+  return content;
+};
+
+export const chats2GPTMessages = ({
   messages,
-  reserveId
+  reserveId,
+  reserveTool = false
 }: {
   messages: ChatItemType[];
   reserveId: boolean;
-}): ChatMessageItemType[] => {
-  return messages.map((item) => ({
-    ...(reserveId && { dataId: item.dataId }),
-    role: chat2Message[item.obj],
-    content: item.value || ''
-  }));
+  reserveTool?: boolean;
+}): ChatCompletionMessageParam[] => {
+  let results: ChatCompletionMessageParam[] = [];
+
+  messages.forEach((item) => {
+    const dataId = reserveId ? item.dataId : undefined;
+    if (item.obj === ChatRoleEnum.Human) {
+      const value = item.value
+        .map((item) => {
+          if (item.type === ChatItemValueTypeEnum.text) {
+            return {
+              type: 'text',
+              text: item.text?.content || ''
+            };
+          }
+          if (item.type === 'file' && item.file?.type === ChatFileTypeEnum.image) {
+            return {
+              type: 'image_url',
+              image_url: {
+                url: item.file?.url || ''
+              }
+            };
+          }
+          return;
+        })
+        .filter(Boolean) as ChatCompletionContentPart[];
+
+      results.push({
+        dataId,
+        role: ChatCompletionRequestMessageRoleEnum.User,
+        content: simpleUserContentPart(value)
+      });
+    } else if (item.obj === ChatRoleEnum.System) {
+      const content = item.value?.[0]?.text?.content;
+      if (content) {
+        results.push({
+          dataId,
+          role: ChatCompletionRequestMessageRoleEnum.System,
+          content
+        });
+      }
+    } else {
+      item.value.forEach((value) => {
+        if (value.type === ChatItemValueTypeEnum.tool && value.tools && reserveTool) {
+          const tool_calls: ChatCompletionMessageToolCall[] = [];
+          const toolResponse: ChatCompletionToolMessageParam[] = [];
+          value.tools.forEach((tool) => {
+            tool_calls.push({
+              id: tool.id,
+              type: 'function',
+              function: {
+                name: tool.functionName,
+                arguments: tool.params
+              }
+            });
+            toolResponse.push({
+              tool_call_id: tool.id,
+              role: ChatCompletionRequestMessageRoleEnum.Tool,
+              name: tool.functionName,
+              content: tool.response
+            });
+          });
+          results = results
+            .concat({
+              dataId,
+              role: ChatCompletionRequestMessageRoleEnum.Assistant,
+              tool_calls
+            })
+            .concat(toolResponse);
+        } else if (value.text) {
+          results.push({
+            dataId,
+            role: ChatCompletionRequestMessageRoleEnum.Assistant,
+            content: value.text.content
+          });
+        }
+      });
+    }
+  });
+
+  return results;
+};
+export const GPTMessages2Chats = (
+  messages: ChatCompletionMessageParam[],
+  reserveTool = true
+): ChatItemType[] => {
+  return messages
+    .map((item) => {
+      const value: ChatItemType['value'] = [];
+      const obj = GPT2Chat[item.role];
+
+      if (
+        obj === ChatRoleEnum.System &&
+        item.role === ChatCompletionRequestMessageRoleEnum.System
+      ) {
+        value.push({
+          type: ChatItemValueTypeEnum.text,
+          text: {
+            content: item.content
+          }
+        });
+      } else if (
+        obj === ChatRoleEnum.Human &&
+        item.role === ChatCompletionRequestMessageRoleEnum.User
+      ) {
+        if (typeof item.content === 'string') {
+          value.push({
+            type: ChatItemValueTypeEnum.text,
+            text: {
+              content: item.content
+            }
+          });
+        } else if (Array.isArray(item.content)) {
+          item.content.forEach((item) => {
+            if (item.type === 'text') {
+              value.push({
+                type: ChatItemValueTypeEnum.text,
+                text: {
+                  content: item.text
+                }
+              });
+            } else if (item.type === 'image_url') {
+              value.push({
+                //@ts-ignore
+                type: 'file',
+                file: {
+                  type: ChatFileTypeEnum.image,
+                  name: '',
+                  url: item.image_url.url
+                }
+              });
+            }
+          });
+          // @ts-ignore
+        }
+      } else if (
+        obj === ChatRoleEnum.AI &&
+        item.role === ChatCompletionRequestMessageRoleEnum.Assistant
+      ) {
+        if (item.content && typeof item.content === 'string') {
+          value.push({
+            type: ChatItemValueTypeEnum.text,
+            text: {
+              content: item.content
+            }
+          });
+        } else if (item.tool_calls && reserveTool) {
+          // save tool calls
+          const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[];
+          value.push({
+            //@ts-ignore
+            type: ChatItemValueTypeEnum.tool,
+            tools: toolCalls.map((tool) => {
+              let toolResponse =
+                messages.find(
+                  (msg) =>
+                    msg.role === ChatCompletionRequestMessageRoleEnum.Tool &&
+                    msg.tool_call_id === tool.id
+                )?.content || '';
+              toolResponse =
+                typeof toolResponse === 'string' ? toolResponse : JSON.stringify(toolResponse);
+
+              return {
+                id: tool.id,
+                toolName: tool.toolName || '',
+                toolAvatar: tool.toolAvatar || '',
+                functionName: tool.function.name,
+                params: tool.function.arguments,
+                response: toolResponse as string
+              };
+            })
+          });
+        } else if (item.function_call && reserveTool) {
+          const functionCall = item.function_call as ChatCompletionMessageFunctionCall;
+          const functionResponse = messages.find(
+            (msg) =>
+              msg.role === ChatCompletionRequestMessageRoleEnum.Function &&
+              msg.name === item.function_call?.name
+          ) as ChatCompletionFunctionMessageParam;
+
+          if (functionResponse) {
+            value.push({
+              //@ts-ignore
+              type: ChatItemValueTypeEnum.tool,
+              tools: [
+                {
+                  id: functionCall.id || '',
+                  toolName: functionCall.toolName || '',
+                  toolAvatar: functionCall.toolAvatar || '',
+                  functionName: functionCall.name,
+                  params: functionCall.arguments,
+                  response: functionResponse.content || ''
+                }
+              ]
+            });
+          }
+        }
+      }
+
+      return {
+        dataId: item.dataId,
+        obj,
+        value
+      } as ChatItemType;
+    })
+    .filter((item) => item.value.length > 0);
+};
+
+export const chatValue2RuntimePrompt = (value: ChatItemValueItemType[]): RuntimeUserPromptType => {
+  const prompt: RuntimeUserPromptType = {
+    files: [],
+    text: ''
+  };
+  value.forEach((item) => {
+    if (item.type === 'file' && item.file) {
+      prompt.files?.push(item.file);
+    } else if (item.text) {
+      prompt.text += item.text.content;
+    }
+  });
+  return prompt;
+};
+
+export const runtimePrompt2ChatsValue = (
+  prompt: RuntimeUserPromptType
+): UserChatItemType['value'] => {
+  const value: UserChatItemType['value'] = [];
+  if (prompt.files) {
+    prompt.files.forEach((file) => {
+      value.push({
+        type: ChatItemValueTypeEnum.file,
+        file
+      });
+    });
+  }
+  if (prompt.text) {
+    value.push({
+      type: ChatItemValueTypeEnum.text,
+      text: {
+        content: prompt.text
+      }
+    });
+  }
+  return value;
+};
+
+export const getSystemPrompt = (prompt?: string): ChatItemType[] => {
+  if (!prompt) return [];
+  return [
+    {
+      obj: ChatRoleEnum.System,
+      value: [{ type: ChatItemValueTypeEnum.text, text: { content: prompt } }]
+    }
+  ];
 };

packages/global/core/chat/constants.ts

@@ -1,28 +1,30 @@
 export enum ChatRoleEnum {
   System = 'System',
   Human = 'Human',
-  AI = 'AI',
-  Function = 'Function',
-  Tool = 'Tool'
+  AI = 'AI'
 }
 export const ChatRoleMap = {
   [ChatRoleEnum.System]: {
-    name: '系统提示词'
+    name: '系统'
   },
   [ChatRoleEnum.Human]: {
     name: '用户'
   },
   [ChatRoleEnum.AI]: {
     name: 'AI'
-  },
-  [ChatRoleEnum.Function]: {
-    name: 'Function'
-  },
-  [ChatRoleEnum.Tool]: {
-    name: 'Tool'
   }
 };
 
+export enum ChatFileTypeEnum {
+  image = 'image',
+  file = 'file'
+}
+export enum ChatItemValueTypeEnum {
+  text = 'text',
+  file = 'file',
+  tool = 'tool'
+}
+
 export enum ChatSourceEnum {
   test = 'test',
   online = 'online',

packages/global/core/chat/type.d.ts

@@ -1,11 +1,20 @@
 import { ClassifyQuestionAgentItemType } from '../module/type';
 import { SearchDataResponseItemType } from '../dataset/type';
-import { ChatRoleEnum, ChatSourceEnum, ChatStatusEnum } from './constants';
+import {
+  ChatFileTypeEnum,
+  ChatItemValueTypeEnum,
+  ChatRoleEnum,
+  ChatSourceEnum,
+  ChatStatusEnum
+} from './constants';
 import { FlowNodeTypeEnum } from '../module/node/constant';
 import { ModuleOutputKeyEnum } from '../module/constants';
+import { DispatchNodeResponseKeyEnum } from '../module/runtime/constants';
 import { AppSchema } from '../app/type';
 import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
+import { ChatBoxInputType } from '../../../../projects/app/src/components/ChatBox/type';
+import { DispatchNodeResponseType } from '../module/runtime/type.d';
 
 export type ChatSchema = {
   _id: string;
@@ -30,7 +39,53 @@ export type ChatWithAppSchema = Omit<ChatSchema, 'appId'> & {
   appId: AppSchema;
 };
 
-export type ChatItemSchema = {
+export type UserChatItemValueItemType = {
+  type: ChatItemValueTypeEnum.text | ChatItemValueTypeEnum.file;
+  text?: {
+    content: string;
+  };
+  file?: {
+    type: `${ChatFileTypeEnum}`;
+    name?: string;
+    url: string;
+  };
+};
+export type UserChatItemType = {
+  obj: ChatRoleEnum.Human;
+  value: UserChatItemValueItemType[];
+};
+export type SystemChatItemValueItemType = {
+  type: ChatItemValueTypeEnum.text;
+  text?: {
+    content: string;
+  };
+};
+export type SystemChatItemType = {
+  obj: ChatRoleEnum.System;
+  value: SystemChatItemValueItemType[];
+};
+export type AIChatItemValueItemType = {
+  type: ChatItemValueTypeEnum.text | ChatItemValueTypeEnum.tool;
+  text?: {
+    content: string;
+  };
+  tools?: ToolModuleResponseItemType[];
+};
+export type AIChatItemType = {
+  obj: ChatRoleEnum.AI;
+  value: AIChatItemValueItemType[];
+  userGoodFeedback?: string;
+  userBadFeedback?: string;
+  customFeedbacks?: string[];
+  adminFeedback?: AdminFbkType;
+  [DispatchNodeResponseKeyEnum.nodeResponse]?: ChatHistoryItemResType[];
+};
+export type ChatItemValueItemType =
+  | UserChatItemValueItemType
+  | SystemChatItemValueItemType
+  | AIChatItemValueItemType;
+
+export type ChatItemSchema = (UserChatItemType | SystemChatItemType | AIChatItemType) & {
   dataId: string;
   chatId: string;
   userId: string;
@@ -38,13 +93,6 @@ export type ChatItemSchema = {
   tmbId: string;
   appId: string;
   time: Date;
-  obj: `${ChatRoleEnum}`;
-  value: string;
-  userGoodFeedback?: string;
-  userBadFeedback?: string;
-  customFeedbacks?: string[];
-  adminFeedback?: AdminFbkType;
-  [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
 
 export type AdminFbkType = {
@@ -56,22 +104,16 @@ export type AdminFbkType = {
 };
 
 /* --------- chat item ---------- */
-export type ChatItemType = {
+export type ChatItemType = (UserChatItemType | SystemChatItemType | AIChatItemType) & {
   dataId?: string;
-  obj: ChatItemSchema['obj'];
-  value: any;
-  userGoodFeedback?: string;
-  userBadFeedback?: string;
-  customFeedbacks?: ChatItemSchema['customFeedbacks'];
-  adminFeedback?: ChatItemSchema['feedback'];
-  [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
 
-export type ChatSiteItemType = ChatItemType & {
+export type ChatSiteItemType = (UserChatItemType | SystemChatItemType | AIChatItemType) & {
+  dataId?: string;
   status: `${ChatStatusEnum}`;
   moduleName?: string;
   ttsBuffer?: Uint8Array;
-};
+} & ChatBoxInputType;
 
 /* --------- team chat --------- */
 export type ChatAppListSchema = {
@@ -93,60 +135,25 @@ export type ChatHistoryItemType = HistoryItemType & {
 };
 
 /* ------- response data ------------ */
-export type moduleDispatchResType = {
-  // common
-  moduleLogo?: string;
-  runningTime?: number;
-  query?: string;
-  textOutput?: string;
-
-  // bill
-  tokens?: number;
-  model?: string;
-  contextTotalLen?: number;
-  totalPoints?: number;
-
-  // chat
-  temperature?: number;
-  maxToken?: number;
-  quoteList?: SearchDataResponseItemType[];
-  historyPreview?: ChatItemType[]; // completion context array. history will slice
-
-  // dataset search
-  similarity?: number;
-  limit?: number;
-  searchMode?: `${DatasetSearchModeEnum}`;
-  searchUsingReRank?: boolean;
-  extensionModel?: string;
-  extensionResult?: string;
-  extensionTokens?: number;
-
-  // cq
-  cqList?: ClassifyQuestionAgentItemType[];
-  cqResult?: string;
-
-  // content extract
-  extractDescription?: string;
-  extractResult?: Record<string, any>;
-
-  // http
-  params?: Record<string, any>;
-  body?: Record<string, any>;
-  headers?: Record<string, any>;
-  httpResult?: Record<string, any>;
-
-  // plugin output
-  pluginOutput?: Record<string, any>;
-  pluginDetail?: ChatHistoryItemResType[];
-
-  // tf switch
-  tfSwitchResult?: boolean;
-
-  // abandon
-  tokens?: number;
-};
-
-export type ChatHistoryItemResType = moduleDispatchResType & {
+export type ChatHistoryItemResType = DispatchNodeResponseType & {
   moduleType: `${FlowNodeTypeEnum}`;
   moduleName: string;
 };
+
+/* One tool run response  */
+export type ToolRunResponseItemType = Record<string, any> | Array;
+/* tool module response */
+export type ToolModuleResponseItemType = {
+  id: string;
+  toolName: string; // tool name
+  toolAvatar: string;
+  params: string; // tool params
+  response: string;
+  functionName: string;
+};
+
+/* dispatch run time */
+export type RuntimeUserPromptType = {
+  files?: UserChatItemValueItemType['file'][];
+  text: string;
+};

packages/global/core/chat/utils.ts

@@ -1,6 +1,79 @@
-import { IMG_BLOCK_KEY, FILE_BLOCK_KEY } from './constants';
+import { DispatchNodeResponseType } from '../module/runtime/type';
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../module/node/constant';
+import { ChatItemValueTypeEnum, ChatRoleEnum } from './constants';
+import { ChatHistoryItemResType, ChatItemType } from './type.d';
 
-export function chatContentReplaceBlock(content: string = '') {
-  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
-  return content.replace(regex, '').trim();
-}
+export const getChatTitleFromChatMessage = (message?: ChatItemType, defaultValue = '新对话') => {
+  // @ts-ignore
+  const textMsg = message?.value.find((item) => item.type === ChatItemValueTypeEnum.text);
+
+  if (textMsg?.text?.content) {
+    return textMsg.text.content.slice(0, 20);
+  }
+
+  return defaultValue;
+};
+
+export const getHistoryPreview = (
+  completeMessages: ChatItemType[]
+): {
+  obj: `${ChatRoleEnum}`;
+  value: string;
+}[] => {
+  return completeMessages.map((item, i) => {
+    if (item.obj === ChatRoleEnum.System || i >= completeMessages.length - 2) {
+      return {
+        obj: item.obj,
+        value: item.value?.[0]?.text?.content || ''
+      };
+    }
+
+    const content = item.value
+      .map((item) => {
+        if (item.text?.content) {
+          const content =
+            item.text.content.length > 20
+              ? `${item.text.content.slice(0, 20)}...`
+              : item.text.content;
+          return content;
+        }
+        return '';
+      })
+      .filter(Boolean)
+      .join('\n');
+
+    return {
+      obj: item.obj,
+      value: content
+    };
+  });
+};
+
+export const filterPublicNodeResponseData = ({
+  flowResponses = []
+}: {
+  flowResponses?: ChatHistoryItemResType[];
+}) => {
+  const filedList = ['quoteList', 'moduleType'];
+  const filterModuleTypeList: any[] = [
+    FlowNodeTypeEnum.pluginModule,
+    FlowNodeTypeEnum.datasetSearchNode,
+    FlowNodeTypeEnum.tools
+  ];
+
+  return flowResponses
+    .filter((item) => filterModuleTypeList.includes(item.moduleType))
+    .map((item) => {
+      const obj: DispatchNodeResponseType = {};
+      for (let key in item) {
+        if (key === 'toolDetail' || key === 'pluginDetail') {
+          // @ts-ignore
+          obj[key] = filterPublicNodeResponseData({ flowResponses: item[key] });
+        } else if (filedList.includes(key)) {
+          // @ts-ignore
+          obj[key] = item[key];
+        }
+      }
+      return obj as ChatHistoryItemResType;
+    });
+};