File input (#2270)

* doc * feat: file upload config * perf: chat box file params * feat: markdown show file * feat: chat file store and clear * perf: read file contentType * feat: llm vision config * feat: file url output * perf: plugin error text * perf: image load * feat: ai chat document * perf: file block ui * feat: read file node * feat: file read response field * feat: simple mode support read files * feat: tool call * feat: read file histories * perf: select file * perf: select file config * i18n * i18n * fix: ts; feat: tool response preview result
2025-07-23 05:12:39 +00:00 · 2024-08-06 10:00:22 +08:00
parent 10dcdb5491
commit e36d9d794f
121 changed files with 2600 additions and 1142 deletions
--- a/packages/service/core/chat/controller.ts
+++ b/packages/service/core/chat/controller.ts
@@ -2,6 +2,9 @@ import type { ChatItemType, ChatItemValueItemType } from '@fastgpt/global/core/c
 import { MongoChatItem } from './chatItemSchema';
 import { addLog } from '../../common/system/log';
 import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
+import { delFileByFileIdList, getGFSCollection } from '../../common/file/gridfs/controller';
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { MongoChat } from './chatSchema';

 export async function getChatItems({
  appId,
@@ -75,3 +78,40 @@ export const addCustomFeedbacks = async ({
    addLog.error('addCustomFeedbacks error', error);
  }
 };
+
+/* 
+  Delete chat files
+  1. ChatId: Delete one chat files
+  2. AppId: Delete all the app's chat files
+*/
+export const deleteChatFiles = async ({
+  chatIdList,
+  appId
+}: {
+  chatIdList?: string[];
+  appId?: string;
+}) => {
+  if (!appId && !chatIdList) return Promise.reject('appId or chatIdList is required');
+
+  const appChatIdList = await (async () => {
+    if (appId) {
+      const appChatIdList = await MongoChat.find({ appId }, { chatId: 1 });
+      return appChatIdList.map((item) => String(item.chatId));
+    } else if (chatIdList) {
+      return chatIdList;
+    }
+    return [];
+  })();
+
+  const collection = getGFSCollection(BucketNameEnum.chat);
+  const where = {
+    'metadata.chatId': { $in: appChatIdList }
+  };
+
+  const files = await collection.find(where, { projection: { _id: 1 } }).toArray();
+
+  await delFileByFileIdList({
+    bucketName: BucketNameEnum.chat,
+    fileIdList: files.map((item) => String(item._id))
+  });
+};
--- a/packages/service/core/chat/utils.ts
+++ b/packages/service/core/chat/utils.ts
@@ -1,13 +1,13 @@
 import { countGptMessagesTokens } from '../../common/string/tiktoken/index';
 import type {
  ChatCompletionContentPart,
-  ChatCompletionMessageParam
+  ChatCompletionMessageParam,
+  SdkChatCompletionMessageParam
 } from '@fastgpt/global/core/ai/type.d';
 import axios from 'axios';
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
-import { guessBase64ImageType } from '../../common/file/utils';
+import { getFileContentTypeFromHeader, guessBase64ImageType } from '../../common/file/utils';
 import { serverRequestBaseUrl } from '../../common/api/serverRequest';
-import { cloneDeep } from 'lodash';

 /* slice chat context by tokens */
 const filterEmptyMessages = (messages: ChatCompletionMessageParam[]) => {
@@ -96,89 +96,183 @@ export const filterGPTMessageByMaxTokens = async ({
  return filterEmptyMessages([...systemPrompts, ...chats]);
 };

-export const formatGPTMessagesInRequestBefore = (messages: ChatCompletionMessageParam[]) => {
-  return messages
-    .map((item) => {
-      if (!item.content) return;
-      if (typeof item.content === 'string') {
-        return {
-          ...item,
-          content: item.content.trim()
-        };
-      }
-
-      // array
-      if (item.content.length === 0) return;
-      if (item.content.length === 1 && item.content[0].type === 'text') {
-        return {
-          ...item,
-          content: item.content[0].text
-        };
-      }
-
-      return item;
-    })
-    .filter(Boolean) as ChatCompletionMessageParam[];
-};
-
-/* Load user chat content.
-  Img: to base 64
+/* 
+  Format requested messages
+  1. If not useVision, only retain text.
+  2. Remove file_url
+  3. If useVision, parse url from question, and load image from url(Local url)
 */
-export const loadChatImgToBase64 = async (content: string | ChatCompletionContentPart[]) => {
-  if (typeof content === 'string') {
-    return content;
-  }
+export const loadRequestMessages = async ({
+  messages,
+  useVision = true,
+  origin
+}: {
+  messages: ChatCompletionMessageParam[];
+  useVision?: boolean;
+  origin?: string;
+}) => {
+  // Split question text and image
+  function parseStringWithImages(input: string): ChatCompletionContentPart[] {
+    if (!useVision) {
+      return [{ type: 'text', text: input || '' }];
+    }

-  return Promise.all(
-    content.map(async (item) => {
-      if (item.type === 'text') return item;
+    // 正则表达式匹配图片URL
+    const imageRegex = /(https?:\/\/.*\.(?:png|jpe?g|gif|webp|bmp|tiff?|svg|ico|heic|avif))/i;

-      if (!item.image_url.url) return item;
+    const result: { type: 'text' | 'image'; value: string }[] = [];
+    let lastIndex = 0;
+    let match;

-      /* 
-        1. From db: Get it from db
-        2. From web: Not update
-      */
-      if (item.image_url.url.startsWith('/')) {
-        const response = await axios.get(item.image_url.url, {
-          baseURL: serverRequestBaseUrl,
-          responseType: 'arraybuffer'
-        });
-        const base64 = Buffer.from(response.data).toString('base64');
-        let imageType = response.headers['content-type'];
-        if (imageType === undefined) {
-          imageType = guessBase64ImageType(base64);
-        }
-        return {
-          ...item,
-          image_url: {
-            ...item.image_url,
-            url: `data:${imageType};base64,${base64}`
-          }
-        };
+    // 使用正则表达式查找所有匹配项
+    while ((match = imageRegex.exec(input.slice(lastIndex))) !== null) {
+      const textBefore = input.slice(lastIndex, lastIndex + match.index);
+
+      // 如果图片URL前有文本，添加文本部分
+      if (textBefore) {
+        result.push({ type: 'text', value: textBefore });
      }

-      return item;
-    })
-  );
-};
-export const loadRequestMessages = async (messages: ChatCompletionMessageParam[]) => {
+      // 添加图片URL
+      result.push({ type: 'image', value: match[0] });
+
+      lastIndex += match.index + match[0].length;
+    }
+
+    // 添加剩余的文本（如果有的话）
+    if (lastIndex < input.length) {
+      result.push({ type: 'text', value: input.slice(lastIndex) });
+    }
+
+    return result
+      .map((item) => {
+        if (item.type === 'text') {
+          return { type: 'text', text: item.value };
+        }
+        if (item.type === 'image') {
+          return {
+            type: 'image_url',
+            image_url: {
+              url: item.value
+            }
+          };
+        }
+        return { type: 'text', text: item.value };
+      })
+      .filter(Boolean) as ChatCompletionContentPart[];
+  }
+  // Load image
+  const parseUserContent = async (content: string | ChatCompletionContentPart[]) => {
+    if (typeof content === 'string') {
+      return parseStringWithImages(content);
+    }
+
+    const result = await Promise.all(
+      content.map(async (item) => {
+        if (item.type === 'text') return parseStringWithImages(item.text);
+        if (item.type === 'file_url') return;
+
+        if (!item.image_url.url) return item;
+
+        // Remove url origin
+        const imgUrl = (() => {
+          if (origin && item.image_url.url.startsWith(origin)) {
+            return item.image_url.url.replace(origin, '');
+          }
+          return item.image_url.url;
+        })();
+
+        /* Load local image */
+        if (imgUrl.startsWith('/')) {
+          const response = await axios.get(imgUrl, {
+            baseURL: serverRequestBaseUrl,
+            responseType: 'arraybuffer'
+          });
+          const base64 = Buffer.from(response.data, 'binary').toString('base64');
+          const imageType =
+            getFileContentTypeFromHeader(response.headers['content-type']) ||
+            guessBase64ImageType(base64);
+
+          return {
+            ...item,
+            image_url: {
+              ...item.image_url,
+              url: `data:${imageType};base64,${base64}`
+            }
+          };
+        }
+
+        return item;
+      })
+    );
+
+    return result.flat().filter(Boolean);
+  };
+  // format GPT messages, concat text messages
+  const clearInvalidMessages = (messages: ChatCompletionMessageParam[]) => {
+    return messages
+      .map((item) => {
+        if (item.role === ChatCompletionRequestMessageRoleEnum.System && !item.content) {
+          return;
+        }
+        if (item.role === ChatCompletionRequestMessageRoleEnum.User) {
+          if (!item.content) return;
+
+          if (typeof item.content === 'string') {
+            return {
+              ...item,
+              content: item.content.trim()
+            };
+          }
+
+          // array
+          if (item.content.length === 0) return;
+          if (item.content.length === 1 && item.content[0].type === 'text') {
+            return {
+              ...item,
+              content: item.content[0].text
+            };
+          }
+        }
+
+        return item;
+      })
+      .filter(Boolean) as ChatCompletionMessageParam[];
+  };
+
  if (messages.length === 0) {
    return Promise.reject('core.chat.error.Messages empty');
  }

-  const loadMessages = await Promise.all(
-    messages.map(async (item) => {
+  // filter messages file
+  const filterMessages = messages.map((item) => {
+    // If useVision=false, only retain text.
+    if (
+      item.role === ChatCompletionRequestMessageRoleEnum.User &&
+      Array.isArray(item.content) &&
+      !useVision
+    ) {
+      return {
+        ...item,
+        content: item.content.filter((item) => item.type === 'text')
+      };
+    }
+
+    return item;
+  });
+
+  const loadMessages = (await Promise.all(
+    filterMessages.map(async (item) => {
      if (item.role === ChatCompletionRequestMessageRoleEnum.User) {
        return {
          ...item,
-          content: await loadChatImgToBase64(item.content)
+          content: await parseUserContent(item.content)
        };
      } else {
        return item;
      }
    })
-  );
+  )) as ChatCompletionMessageParam[];

-  return loadMessages;
+  return clearInvalidMessages(loadMessages) as SdkChatCompletionMessageParam[];
 };