feat: vision model (#489)

* mongo init

* perf: mongo connect

* perf: tts

perf: whisper and tts

peref: tts whisper permission

log

reabase (#488)

* perf: modal

* i18n

* perf: schema lean

* feat: vision model format

* perf: tts loading

* perf: static data

* perf: tts

* feat: image

* perf: image

* perf: upload image and title

* perf: image size

* doc

* perf: color

* doc

* speaking can not select file

* doc

packages/global/core/ai/model.d.ts

@@ -9,6 +9,7 @@ export type ChatModelItemType = LLMModelItemType & {
   quoteMaxToken: number;
   maxTemperature: number;
   censor?: boolean;
+  vision?: boolean;
   defaultSystemChatPrompt?: string;
 };
 

packages/global/core/ai/model.ts

@@ -17,6 +17,7 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 2000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
     defaultSystemChatPrompt: ''
   },
   {
@@ -28,6 +29,7 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 8000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
     defaultSystemChatPrompt: ''
   },
   {
@@ -39,6 +41,19 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 4000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
+    defaultSystemChatPrompt: ''
+  },
+  {
+    model: 'gpt-4-vision-preview',
+    name: 'GPT4-Vision',
+    maxContext: 128000,
+    maxResponse: 4000,
+    price: 0,
+    quoteMaxToken: 100000,
+    maxTemperature: 1.2,
+    censor: false,
+    vision: true,
     defaultSystemChatPrompt: ''
   }
 ];

packages/global/core/ai/type.d.ts

@@ -5,12 +5,14 @@ import type {
   ChatCompletionMessageParam,
   ChatCompletionContentPart
 } from 'openai/resources';
+
 export type ChatCompletionContentPart = ChatCompletionContentPart;
 export type ChatCompletionCreateParams = ChatCompletionCreateParams;
-export type ChatMessageItemType = Omit<ChatCompletionMessageParam> & {
+export type ChatMessageItemType = Omit<ChatCompletionMessageParam, 'name'> & {
+  name?: any;
   dataId?: string;
   content: any;
-};
+} & any;
 
 export type ChatCompletion = ChatCompletion;
 export type StreamChatType = Stream<ChatCompletionChunk>;

packages/global/core/chat/constants.ts

@@ -54,3 +54,6 @@ export const ChatSourceMap = {
 
 export const HUMAN_ICON = `/icon/human.svg`;
 export const LOGO_ICON = `/icon/logo.svg`;
+
+export const IMG_BLOCK_KEY = 'img-block';
+export const FILE_BLOCK_KEY = 'file-block';

packages/global/core/chat/utils.ts

@@ -0,0 +1,6 @@
+import { IMG_BLOCK_KEY, FILE_BLOCK_KEY } from './constants';
+
+export function chatContentReplaceBlock(content: string = '') {
+  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
+  return content.replace(regex, '').trim();
+}