lock (#2063)

* lock

* perf: init data

* perf: vision model url

* fix: chat index

packages/service/core/chat/chatItemSchema.ts

@@ -89,7 +89,7 @@ try {
      get chat logs; 
      close custom feedback; 
   */
-  ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true, unique: true });
+  ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true });
   // admin charts
   ChatItemSchema.index({ time: -1, obj: 1 }, { background: true });
   // timer, clear history

packages/service/core/chat/chatSchema.ts

@@ -85,7 +85,7 @@ try {
   // get user history
   ChatSchema.index({ tmbId: 1, appId: 1, top: -1, updateTime: -1 }, { background: true });
   // delete by appid; clear history; init chat; update chat; auth chat; get chat;
-  ChatSchema.index({ appId: 1, chatId: 1 }, { background: true, unique: true });
+  ChatSchema.index({ appId: 1, chatId: 1 }, { background: true });
 
   // get chat logs;
   ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }, { background: true });

packages/service/core/chat/utils.ts

@@ -1,4 +1,3 @@
-import { IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
 import { countGptMessagesTokens } from '../../common/string/tiktoken/index';
 import type {
   ChatCompletionContentPart,
@@ -7,6 +6,8 @@ import type {
 import axios from 'axios';
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
 import { guessBase64ImageType } from '../../common/file/utils';
+import { serverRequestBaseUrl } from '../../common/api/serverRequest';
+import { cloneDeep } from 'lodash';
 
 /* slice chat context by tokens */
 const filterEmptyMessages = (messages: ChatCompletionMessageParam[]) => {
@@ -120,137 +121,64 @@ export const formatGPTMessagesInRequestBefore = (messages: ChatCompletionMessage
     .filter(Boolean) as ChatCompletionMessageParam[];
 };
 
-/**
-    string to vision model. Follow the markdown code block rule for interception:
-
-    @rule:
-    ```img-block
-        {src:""}
-        {src:""}
-    ```
-    ```file-block
-        {name:"",src:""},
-        {name:"",src:""}
-    ```
-    @example:
-        What’s in this image?
-        ```img-block
-            {src:"https://1.png"}
-        ```
-    @return 
-        [
-            { type: 'text', text: 'What’s in this image?' },
-            {
-              type: 'image_url',
-              image_url: {
-                url: 'https://1.png'
-              }
-            }
-        ]
- */
-export async function formatStr2ChatContent(str: string) {
-  const content: ChatCompletionContentPart[] = [];
-  let lastIndex = 0;
-  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
-
-  const imgKey: 'image_url' = 'image_url';
-
-  let match;
-
-  while ((match = regex.exec(str)) !== null) {
-    // add previous text
-    if (match.index > lastIndex) {
-      const text = str.substring(lastIndex, match.index).trim();
-      if (text) {
-        content.push({ type: 'text', text });
-      }
-    }
-
-    const blockType = match[1].trim();
-
-    if (blockType === IMG_BLOCK_KEY) {
-      const blockContentLines = match[2].trim().split('\n');
-      const jsonLines = blockContentLines.map((item) => {
-        try {
-          return JSON.parse(item) as { src: string };
-        } catch (error) {
-          return { src: '' };
-        }
-      });
-
-      for (const item of jsonLines) {
-        if (!item.src) throw new Error("image block's content error");
-      }
-
-      content.push(
-        ...jsonLines.map((item) => ({
-          type: imgKey,
-          image_url: {
-            url: item.src
-          }
-        }))
-      );
-    }
-
-    lastIndex = regex.lastIndex;
-  }
-
-  // add remaining text
-  if (lastIndex < str.length) {
-    const remainingText = str.substring(lastIndex).trim();
-    if (remainingText) {
-      content.push({ type: 'text', text: remainingText });
-    }
-  }
-
-  // Continuous text type content, if type=text, merge them
-  for (let i = 0; i < content.length - 1; i++) {
-    const currentContent = content[i];
-    const nextContent = content[i + 1];
-    if (currentContent.type === 'text' && nextContent.type === 'text') {
-      currentContent.text += nextContent.text;
-      content.splice(i + 1, 1);
-      i--;
-    }
-  }
-
-  if (content.length === 1 && content[0].type === 'text') {
-    return content[0].text;
-  }
-
-  if (!content) return null;
-  // load img to base64
-  for await (const item of content) {
-    if (item.type === imgKey && item[imgKey]?.url) {
-      const response = await axios.get(item[imgKey].url, {
-        responseType: 'arraybuffer'
-      });
-      const base64 = Buffer.from(response.data).toString('base64');
-      item[imgKey].url = `data:${response.headers['content-type']};base64,${base64}`;
-    }
-  }
-
-  return content ? content : null;
-}
-
+/* Load user chat content.
+  Img: to base 64
+*/
 export const loadChatImgToBase64 = async (content: string | ChatCompletionContentPart[]) => {
   if (typeof content === 'string') {
     return content;
   }
+
   return Promise.all(
     content.map(async (item) => {
       if (item.type === 'text') return item;
-      // load image
-      const response = await axios.get(item.image_url.url, {
-        responseType: 'arraybuffer'
-      });
-      const base64 = Buffer.from(response.data).toString('base64');
-      let imageType = response.headers['content-type'];
-      if (imageType === undefined) {
-        imageType = guessBase64ImageType(base64);
+
+      if (!item.image_url.url) return item;
+
+      /* 
+        1. From db: Get it from db
+        2. From web: Not update
+      */
+      if (item.image_url.url.startsWith('/')) {
+        const response = await axios.get(item.image_url.url, {
+          baseURL: serverRequestBaseUrl,
+          responseType: 'arraybuffer'
+        });
+        const base64 = Buffer.from(response.data).toString('base64');
+        let imageType = response.headers['content-type'];
+        if (imageType === undefined) {
+          imageType = guessBase64ImageType(base64);
+        }
+        return {
+          ...item,
+          image_url: {
+            ...item.image_url,
+            url: `data:${imageType};base64,${base64}`
+          }
+        };
       }
-      item.image_url.url = `data:${imageType};base64,${base64}`;
+
       return item;
     })
   );
 };
+export const loadRequestMessages = async (messages: ChatCompletionMessageParam[]) => {
+  if (messages.length === 0) {
+    return Promise.reject('core.chat.error.Messages empty');
+  }
+
+  const loadMessages = await Promise.all(
+    messages.map(async (item) => {
+      if (item.role === ChatCompletionRequestMessageRoleEnum.User) {
+        return {
+          ...item,
+          content: await loadChatImgToBase64(item.content)
+        };
+      } else {
+        return item;
+      }
+    })
+  );
+
+  return loadMessages;
+};