feat: vision model (#489)

* mongo init

* perf: mongo connect

* perf: tts

perf: whisper and tts

peref: tts whisper permission

log

reabase (#488)

* perf: modal

* i18n

* perf: schema lean

* feat: vision model format

* perf: tts loading

* perf: static data

* perf: tts

* feat: image

* perf: image

* perf: upload image and title

* perf: image size

* doc

* perf: color

* doc

* speaking can not select file

* doc

packages/global/common/string/tools.ts

@@ -24,7 +24,7 @@ export const simpleText = (text: string) => {
 };
 
 /* 
-    replace {{variable}} to value
+  replace {{variable}} to value
 */
 export function replaceVariable(text: string, obj: Record<string, string | number>) {
   for (const key in obj) {

packages/global/core/ai/model.d.ts

@@ -9,6 +9,7 @@ export type ChatModelItemType = LLMModelItemType & {
   quoteMaxToken: number;
   maxTemperature: number;
   censor?: boolean;
+  vision?: boolean;
   defaultSystemChatPrompt?: string;
 };
 

packages/global/core/ai/model.ts

@@ -17,6 +17,7 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 2000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
     defaultSystemChatPrompt: ''
   },
   {
@@ -28,6 +29,7 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 8000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
     defaultSystemChatPrompt: ''
   },
   {
@@ -39,6 +41,19 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 4000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
+    defaultSystemChatPrompt: ''
+  },
+  {
+    model: 'gpt-4-vision-preview',
+    name: 'GPT4-Vision',
+    maxContext: 128000,
+    maxResponse: 4000,
+    price: 0,
+    quoteMaxToken: 100000,
+    maxTemperature: 1.2,
+    censor: false,
+    vision: true,
     defaultSystemChatPrompt: ''
   }
 ];

packages/global/core/ai/type.d.ts

@@ -5,12 +5,14 @@ import type {
   ChatCompletionMessageParam,
   ChatCompletionContentPart
 } from 'openai/resources';
+
 export type ChatCompletionContentPart = ChatCompletionContentPart;
 export type ChatCompletionCreateParams = ChatCompletionCreateParams;
-export type ChatMessageItemType = Omit<ChatCompletionMessageParam> & {
+export type ChatMessageItemType = Omit<ChatCompletionMessageParam, 'name'> & {
+  name?: any;
   dataId?: string;
   content: any;
-};
+} & any;
 
 export type ChatCompletion = ChatCompletion;
 export type StreamChatType = Stream<ChatCompletionChunk>;

packages/global/core/chat/constants.ts

@@ -54,3 +54,6 @@ export const ChatSourceMap = {
 
 export const HUMAN_ICON = `/icon/human.svg`;
 export const LOGO_ICON = `/icon/logo.svg`;
+
+export const IMG_BLOCK_KEY = 'img-block';
+export const FILE_BLOCK_KEY = 'file-block';

packages/global/core/chat/utils.ts

@@ -0,0 +1,6 @@
+import { IMG_BLOCK_KEY, FILE_BLOCK_KEY } from './constants';
+
+export function chatContentReplaceBlock(content: string = '') {
+  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
+  return content.replace(regex, '').trim();
+}

packages/service/common/buffer/tts/schema.ts

@@ -33,3 +33,4 @@ try {
 
 export const MongoTTSBuffer: Model<TTSBufferSchemaType> =
   models[collectionName] || model(collectionName, TTSBufferSchema);
+MongoTTSBuffer.syncIndexes();

packages/service/common/file/image/controller.ts

@@ -5,12 +5,26 @@ export function getMongoImgUrl(id: string) {
   return `${imageBaseUrl}${id}`;
 }
 
-export async function uploadMongoImg({ base64Img, userId }: { base64Img: string; userId: string }) {
+export const maxImgSize = 1024 * 1024 * 12;
+export async function uploadMongoImg({
+  base64Img,
+  teamId,
+  expiredTime
+}: {
+  base64Img: string;
+  teamId: string;
+  expiredTime?: Date;
+}) {
+  if (base64Img.length > maxImgSize) {
+    return Promise.reject('Image too large');
+  }
+
   const base64Data = base64Img.split(',')[1];
 
   const { _id } = await MongoImage.create({
-    userId,
-    binary: Buffer.from(base64Data, 'base64')
+    teamId,
+    binary: Buffer.from(base64Data, 'base64'),
+    expiredTime
   });
 
   return getMongoImgUrl(String(_id));

packages/service/common/file/image/schema.ts

@@ -1,16 +1,27 @@
+import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
 import { connectionMongo, type Model } from '../../mongo';
 const { Schema, model, models } = connectionMongo;
 
 const ImageSchema = new Schema({
-  userId: {
+  teamId: {
     type: Schema.Types.ObjectId,
-    ref: 'user',
-    required: true
+    ref: TeamCollectionName
   },
   binary: {
     type: Buffer
+  },
+  expiredTime: {
+    type: Date
   }
 });
 
-export const MongoImage: Model<{ userId: string; binary: Buffer }> =
+try {
+  ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoImage: Model<{ teamId: string; binary: Buffer }> =
   models['image'] || model('image', ImageSchema);
+
+MongoImage.syncIndexes();

packages/service/core/app/schema.ts

@@ -67,3 +67,5 @@ try {
 
 export const MongoApp: Model<AppType> =
   models[appCollectionName] || model(appCollectionName, AppSchema);
+
+MongoApp.syncIndexes();

packages/service/core/chat/chatItemSchema.ts

@@ -83,3 +83,5 @@ try {
 
 export const MongoChatItem: Model<ChatItemType> =
   models['chatItem'] || model('chatItem', ChatItemSchema);
+
+MongoChatItem.syncIndexes();

packages/service/core/chat/chatSchema.ts

@@ -92,7 +92,7 @@ const ChatSchema = new Schema({
 });
 
 try {
-  ChatSchema.index({ userId: 1 });
+  ChatSchema.index({ tmbId: 1 });
   ChatSchema.index({ updateTime: -1 });
   ChatSchema.index({ appId: 1 });
 } catch (error) {
@@ -101,3 +101,4 @@ try {
 
 export const MongoChat: Model<ChatType> =
   models[chatCollectionName] || model(chatCollectionName, ChatSchema);
+MongoChat.syncIndexes();

packages/service/core/chat/utils.ts

@@ -1,7 +1,8 @@
 import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
-import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
+import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
 import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
+import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
 
 /* slice chat context by tokens */
 export function ChatContextFilter({
@@ -51,3 +52,101 @@ export function ChatContextFilter({
 
   return [...systemPrompts, ...chats];
 }
+
+/**
+    string to vision model. Follow the markdown code block rule for interception:
+
+    @rule:
+    ```img-block
+        {src:""}
+        {src:""}
+    ```
+    ```file-block
+        {name:"",src:""},
+        {name:"",src:""}
+    ```
+    @example:
+        What’s in this image?
+        ```img-block
+            {src:"https://1.png"}
+        ```
+    @return 
+        [
+            { type: 'text', text: 'What’s in this image?' },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'https://1.png'
+              }
+            }
+        ]
+ */
+export function formatStr2ChatContent(str: string) {
+  const content: ChatCompletionContentPart[] = [];
+  let lastIndex = 0;
+  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
+
+  let match;
+
+  while ((match = regex.exec(str)) !== null) {
+    // add previous text
+    if (match.index > lastIndex) {
+      const text = str.substring(lastIndex, match.index).trim();
+      if (text) {
+        content.push({ type: 'text', text });
+      }
+    }
+
+    const blockType = match[1].trim();
+
+    if (blockType === IMG_BLOCK_KEY) {
+      const blockContentLines = match[2].trim().split('\n');
+      const jsonLines = blockContentLines.map((item) => {
+        try {
+          return JSON.parse(item) as { src: string };
+        } catch (error) {
+          return { src: '' };
+        }
+      });
+
+      for (const item of jsonLines) {
+        if (!item.src) throw new Error("image block's content error");
+      }
+
+      content.push(
+        ...jsonLines.map((item) => ({
+          type: 'image_url' as any,
+          image_url: {
+            url: item.src
+          }
+        }))
+      );
+    }
+
+    lastIndex = regex.lastIndex;
+  }
+
+  // add remaining text
+  if (lastIndex < str.length) {
+    const remainingText = str.substring(lastIndex).trim();
+    if (remainingText) {
+      content.push({ type: 'text', text: remainingText });
+    }
+  }
+
+  // Continuous text type content, if type=text, merge them
+  for (let i = 0; i < content.length - 1; i++) {
+    const currentContent = content[i];
+    const nextContent = content[i + 1];
+    if (currentContent.type === 'text' && nextContent.type === 'text') {
+      currentContent.text += nextContent.text;
+      content.splice(i + 1, 1);
+      i--;
+    }
+  }
+
+  if (content.length === 1 && content[0].type === 'text') {
+    return content[0].text;
+  }
+  return content ? content : null;
+}

packages/service/core/dataset/controller.ts

@@ -22,9 +22,9 @@ export async function findDatasetIdTreeByTopDatasetId(
 }
 
 export async function getCollectionWithDataset(collectionId: string) {
-  const data = (
-    await MongoDatasetCollection.findById(collectionId).populate('datasetId')
-  )?.toJSON() as CollectionWithDatasetType;
+  const data = (await MongoDatasetCollection.findById(collectionId)
+    .populate('datasetId')
+    .lean()) as CollectionWithDatasetType;
   if (!data) {
     return Promise.reject('Collection is not exist');
   }

packages/service/core/dataset/data/schema.ts

@@ -76,3 +76,4 @@ try {
 
 export const MongoDatasetData: Model<DatasetDataSchemaType> =
   models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
+MongoDatasetData.syncIndexes();

packages/service/core/dataset/schema.ts

@@ -82,3 +82,4 @@ try {
 
 export const MongoDataset: Model<DatasetSchemaType> =
   models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);
+MongoDataset.syncIndexes();

packages/service/core/dataset/training/schema.ts

@@ -104,3 +104,5 @@ try {
 
 export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> =
   models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema);
+
+MongoDatasetTraining.syncIndexes();

packages/service/core/plugin/schema.ts

@@ -46,10 +46,11 @@ const PluginSchema = new Schema({
 });
 
 try {
-  PluginSchema.index({ userId: 1 });
+  PluginSchema.index({ tmbId: 1 });
 } catch (error) {
   console.log(error);
 }
 
 export const MongoPlugin: Model<PluginItemSchema> =
   models[ModuleCollectionName] || model(ModuleCollectionName, PluginSchema);
+MongoPlugin.syncIndexes();

packages/service/support/activity/promotion/schema.ts

@@ -31,3 +31,4 @@ const PromotionRecordSchema = new Schema({
 
 export const MongoPromotionRecord: Model<PromotionRecordType> =
   models['promotionRecord'] || model('promotionRecord', PromotionRecordSchema);
+MongoPromotionRecord.syncIndexes();

packages/service/support/openapi/schema.ts

@@ -70,3 +70,4 @@ const OpenApiSchema = new Schema(
 
 export const MongoOpenApi: Model<OpenApiSchema> =
   models['openapi'] || model('openapi', OpenApiSchema);
+MongoOpenApi.syncIndexes();

packages/service/support/outLink/schema.ts

@@ -71,3 +71,5 @@ const OutLinkSchema = new Schema({
 
 export const MongoOutLink: Model<SchemaType> =
   models['outlinks'] || model('outlinks', OutLinkSchema);
+
+MongoOutLink.syncIndexes();

packages/service/support/permission/auth/app.ts

@@ -22,12 +22,12 @@ export async function authApp({
   }
 > {
   const result = await parseHeaderCert(props);
-  const { userId, teamId, tmbId } = result;
+  const { teamId, tmbId } = result;
   const { role } = await getTeamInfoByTmbId({ tmbId });
 
   const { app, isOwner, canWrite } = await (async () => {
     // get app
-    const app = (await MongoApp.findOne({ _id: appId, teamId }))?.toJSON();
+    const app = await MongoApp.findOne({ _id: appId, teamId }).lean();
     if (!app) {
       return Promise.reject(AppErrEnum.unAuthApp);
     }

packages/service/support/permission/auth/chat.ts

@@ -24,9 +24,9 @@ export async function authChat({
 
   const { chat, isOwner, canWrite } = await (async () => {
     // get chat
-    const chat = (
-      await MongoChat.findOne({ chatId, teamId }).populate('appId')
-    )?.toJSON() as ChatWithAppSchema;
+    const chat = (await MongoChat.findOne({ chatId, teamId })
+      .populate('appId')
+      .lean()) as ChatWithAppSchema;
 
     if (!chat) {
       return Promise.reject('Chat is not exists');

packages/service/support/permission/auth/dataset.ts

@@ -31,7 +31,7 @@ export async function authDataset({
   const { role } = await getTeamInfoByTmbId({ tmbId });
 
   const { dataset, isOwner, canWrite } = await (async () => {
-    const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toObject();
+    const dataset = await MongoDataset.findOne({ _id: datasetId, teamId }).lean();
 
     if (!dataset) {
       return Promise.reject(DatasetErrEnum.unAuthDataset);

packages/service/support/user/schema.ts

@@ -64,3 +64,4 @@ const UserSchema = new Schema({
 
 export const MongoUser: Model<UserModelSchema> =
   models[userCollectionName] || model(userCollectionName, UserSchema);
+MongoUser.syncIndexes();

packages/service/support/wallet/bill/schema.ts

@@ -59,3 +59,4 @@ try {
 }
 
 export const MongoBill: Model<BillType> = models['bill'] || model('bill', BillSchema);
+MongoBill.syncIndexes();