feat: vision model (#489)

* mongo init

* perf: mongo connect

* perf: tts

perf: whisper and tts

peref: tts whisper permission

log

reabase (#488)

* perf: modal

* i18n

* perf: schema lean

* feat: vision model format

* perf: tts loading

* perf: static data

* perf: tts

* feat: image

* perf: image

* perf: upload image and title

* perf: image size

* doc

* perf: color

* doc

* speaking can not select file

* doc

packages/service/core/app/schema.ts

@@ -67,3 +67,5 @@ try {
 
 export const MongoApp: Model<AppType> =
   models[appCollectionName] || model(appCollectionName, AppSchema);
+
+MongoApp.syncIndexes();

packages/service/core/chat/chatItemSchema.ts

@@ -83,3 +83,5 @@ try {
 
 export const MongoChatItem: Model<ChatItemType> =
   models['chatItem'] || model('chatItem', ChatItemSchema);
+
+MongoChatItem.syncIndexes();

packages/service/core/chat/chatSchema.ts

@@ -92,7 +92,7 @@ const ChatSchema = new Schema({
 });
 
 try {
-  ChatSchema.index({ userId: 1 });
+  ChatSchema.index({ tmbId: 1 });
   ChatSchema.index({ updateTime: -1 });
   ChatSchema.index({ appId: 1 });
 } catch (error) {
@@ -101,3 +101,4 @@ try {
 
 export const MongoChat: Model<ChatType> =
   models[chatCollectionName] || model(chatCollectionName, ChatSchema);
+MongoChat.syncIndexes();

packages/service/core/chat/utils.ts

@@ -1,7 +1,8 @@
 import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
-import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
+import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
 import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
+import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
 
 /* slice chat context by tokens */
 export function ChatContextFilter({
@@ -51,3 +52,101 @@ export function ChatContextFilter({
 
   return [...systemPrompts, ...chats];
 }
+
+/**
+    string to vision model. Follow the markdown code block rule for interception:
+
+    @rule:
+    ```img-block
+        {src:""}
+        {src:""}
+    ```
+    ```file-block
+        {name:"",src:""},
+        {name:"",src:""}
+    ```
+    @example:
+        What’s in this image?
+        ```img-block
+            {src:"https://1.png"}
+        ```
+    @return 
+        [
+            { type: 'text', text: 'What’s in this image?' },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'https://1.png'
+              }
+            }
+        ]
+ */
+export function formatStr2ChatContent(str: string) {
+  const content: ChatCompletionContentPart[] = [];
+  let lastIndex = 0;
+  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
+
+  let match;
+
+  while ((match = regex.exec(str)) !== null) {
+    // add previous text
+    if (match.index > lastIndex) {
+      const text = str.substring(lastIndex, match.index).trim();
+      if (text) {
+        content.push({ type: 'text', text });
+      }
+    }
+
+    const blockType = match[1].trim();
+
+    if (blockType === IMG_BLOCK_KEY) {
+      const blockContentLines = match[2].trim().split('\n');
+      const jsonLines = blockContentLines.map((item) => {
+        try {
+          return JSON.parse(item) as { src: string };
+        } catch (error) {
+          return { src: '' };
+        }
+      });
+
+      for (const item of jsonLines) {
+        if (!item.src) throw new Error("image block's content error");
+      }
+
+      content.push(
+        ...jsonLines.map((item) => ({
+          type: 'image_url' as any,
+          image_url: {
+            url: item.src
+          }
+        }))
+      );
+    }
+
+    lastIndex = regex.lastIndex;
+  }
+
+  // add remaining text
+  if (lastIndex < str.length) {
+    const remainingText = str.substring(lastIndex).trim();
+    if (remainingText) {
+      content.push({ type: 'text', text: remainingText });
+    }
+  }
+
+  // Continuous text type content, if type=text, merge them
+  for (let i = 0; i < content.length - 1; i++) {
+    const currentContent = content[i];
+    const nextContent = content[i + 1];
+    if (currentContent.type === 'text' && nextContent.type === 'text') {
+      currentContent.text += nextContent.text;
+      content.splice(i + 1, 1);
+      i--;
+    }
+  }
+
+  if (content.length === 1 && content[0].type === 'text') {
+    return content[0].text;
+  }
+  return content ? content : null;
+}

packages/service/core/dataset/controller.ts

@@ -22,9 +22,9 @@ export async function findDatasetIdTreeByTopDatasetId(
 }
 
 export async function getCollectionWithDataset(collectionId: string) {
-  const data = (
-    await MongoDatasetCollection.findById(collectionId).populate('datasetId')
-  )?.toJSON() as CollectionWithDatasetType;
+  const data = (await MongoDatasetCollection.findById(collectionId)
+    .populate('datasetId')
+    .lean()) as CollectionWithDatasetType;
   if (!data) {
     return Promise.reject('Collection is not exist');
   }

packages/service/core/dataset/data/schema.ts

@@ -76,3 +76,4 @@ try {
 
 export const MongoDatasetData: Model<DatasetDataSchemaType> =
   models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
+MongoDatasetData.syncIndexes();

packages/service/core/dataset/schema.ts

@@ -82,3 +82,4 @@ try {
 
 export const MongoDataset: Model<DatasetSchemaType> =
   models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);
+MongoDataset.syncIndexes();

packages/service/core/dataset/training/schema.ts

@@ -104,3 +104,5 @@ try {
 
 export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> =
   models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema);
+
+MongoDatasetTraining.syncIndexes();

packages/service/core/plugin/schema.ts

@@ -46,10 +46,11 @@ const PluginSchema = new Schema({
 });
 
 try {
-  PluginSchema.index({ userId: 1 });
+  PluginSchema.index({ tmbId: 1 });
 } catch (error) {
   console.log(error);
 }
 
 export const MongoPlugin: Model<PluginItemSchema> =
   models[ModuleCollectionName] || model(ModuleCollectionName, PluginSchema);
+MongoPlugin.syncIndexes();