diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md index c6766157d..1f8a8acd9 100644 --- a/docSite/content/docs/development/configuration.md +++ b/docSite/content/docs/development/configuration.md @@ -36,6 +36,7 @@ weight: 520 "quoteMaxToken": 2000, // 最大引用内容长度 "maxTemperature": 1.2, // 最大温度值 "censor": false, // 是否开启敏感词过滤(商业版) + "vision": false, // 支持图片输入 "defaultSystemChatPrompt": "" }, { @@ -47,6 +48,7 @@ weight: 520 "quoteMaxToken": 8000, "maxTemperature": 1.2, "censor": false, + "vision": false, "defaultSystemChatPrompt": "" }, { @@ -58,6 +60,19 @@ weight: 520 "quoteMaxToken": 4000, "maxTemperature": 1.2, "censor": false, + "vision": false, + "defaultSystemChatPrompt": "" + }, + { + "model": "gpt-4-vision-preview", + "name": "GPT4-Vision", + "maxContext": 128000, + "maxResponse": 4000, + "price": 0, + "quoteMaxToken": 100000, + "maxTemperature": 1.2, + "censor": false, + "vision": true, "defaultSystemChatPrompt": "" } ], diff --git a/docSite/content/docs/installation/upgrading/461.md b/docSite/content/docs/installation/upgrading/461.md new file mode 100644 index 000000000..44515e34b --- /dev/null +++ b/docSite/content/docs/installation/upgrading/461.md @@ -0,0 +1,16 @@ +--- +title: 'V4.6.1' +description: 'FastGPT V4.6 .1' +icon: 'upgrade' +draft: false +toc: true +weight: 835 +--- + + +## V4.6.1 功能介绍 + +1. 新增 - GPT4-v 模型支持 +2. 新增 - whisper 语音输入 +3. 优化 - TTS 流传输 +4. 优化 - TTS 缓存 diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts index 61cdf44eb..5e4e7487c 100644 --- a/packages/global/common/string/tools.ts +++ b/packages/global/common/string/tools.ts @@ -24,7 +24,7 @@ export const simpleText = (text: string) => { }; /* - replace {{variable}} to value + replace {{variable}} to value */ export function replaceVariable(text: string, obj: Record) { for (const key in obj) { diff --git a/packages/global/core/ai/model.d.ts b/packages/global/core/ai/model.d.ts index 2d40c32a9..fd316c035 100644 --- a/packages/global/core/ai/model.d.ts +++ b/packages/global/core/ai/model.d.ts @@ -9,6 +9,7 @@ export type ChatModelItemType = LLMModelItemType & { quoteMaxToken: number; maxTemperature: number; censor?: boolean; + vision?: boolean; defaultSystemChatPrompt?: string; }; diff --git a/packages/global/core/ai/model.ts b/packages/global/core/ai/model.ts index 021ee1a7f..f074be10c 100644 --- a/packages/global/core/ai/model.ts +++ b/packages/global/core/ai/model.ts @@ -17,6 +17,7 @@ export const defaultChatModels: ChatModelItemType[] = [ quoteMaxToken: 2000, maxTemperature: 1.2, censor: false, + vision: false, defaultSystemChatPrompt: '' }, { @@ -28,6 +29,7 @@ export const defaultChatModels: ChatModelItemType[] = [ quoteMaxToken: 8000, maxTemperature: 1.2, censor: false, + vision: false, defaultSystemChatPrompt: '' }, { @@ -39,6 +41,19 @@ export const defaultChatModels: ChatModelItemType[] = [ quoteMaxToken: 4000, maxTemperature: 1.2, censor: false, + vision: false, + defaultSystemChatPrompt: '' + }, + { + model: 'gpt-4-vision-preview', + name: 'GPT4-Vision', + maxContext: 128000, + maxResponse: 4000, + price: 0, + quoteMaxToken: 100000, + maxTemperature: 1.2, + censor: false, + vision: true, defaultSystemChatPrompt: '' } ]; diff --git a/packages/global/core/ai/type.d.ts b/packages/global/core/ai/type.d.ts index 1689ee044..1aaefb121 100644 --- a/packages/global/core/ai/type.d.ts +++ b/packages/global/core/ai/type.d.ts @@ -5,12 +5,14 @@ import type { ChatCompletionMessageParam, ChatCompletionContentPart } from 'openai/resources'; + export type ChatCompletionContentPart = ChatCompletionContentPart; export type ChatCompletionCreateParams = ChatCompletionCreateParams; -export type ChatMessageItemType = Omit & { +export type ChatMessageItemType = Omit & { + name?: any; dataId?: string; content: any; -}; +} & any; export type ChatCompletion = ChatCompletion; export type StreamChatType = Stream; diff --git a/packages/global/core/chat/constants.ts b/packages/global/core/chat/constants.ts index 2313fb445..26e2ead32 100644 --- a/packages/global/core/chat/constants.ts +++ b/packages/global/core/chat/constants.ts @@ -54,3 +54,6 @@ export const ChatSourceMap = { export const HUMAN_ICON = `/icon/human.svg`; export const LOGO_ICON = `/icon/logo.svg`; + +export const IMG_BLOCK_KEY = 'img-block'; +export const FILE_BLOCK_KEY = 'file-block'; diff --git a/packages/global/core/chat/utils.ts b/packages/global/core/chat/utils.ts new file mode 100644 index 000000000..19b6759a9 --- /dev/null +++ b/packages/global/core/chat/utils.ts @@ -0,0 +1,6 @@ +import { IMG_BLOCK_KEY, FILE_BLOCK_KEY } from './constants'; + +export function chatContentReplaceBlock(content: string = '') { + const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g'); + return content.replace(regex, '').trim(); +} diff --git a/packages/service/common/buffer/tts/schema.ts b/packages/service/common/buffer/tts/schema.ts index 9e7cf6b83..3004325d2 100644 --- a/packages/service/common/buffer/tts/schema.ts +++ b/packages/service/common/buffer/tts/schema.ts @@ -33,3 +33,4 @@ try { export const MongoTTSBuffer: Model = models[collectionName] || model(collectionName, TTSBufferSchema); +MongoTTSBuffer.syncIndexes(); diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index fc51430d0..e8c1c4b97 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -5,12 +5,26 @@ export function getMongoImgUrl(id: string) { return `${imageBaseUrl}${id}`; } -export async function uploadMongoImg({ base64Img, userId }: { base64Img: string; userId: string }) { +export const maxImgSize = 1024 * 1024 * 12; +export async function uploadMongoImg({ + base64Img, + teamId, + expiredTime +}: { + base64Img: string; + teamId: string; + expiredTime?: Date; +}) { + if (base64Img.length > maxImgSize) { + return Promise.reject('Image too large'); + } + const base64Data = base64Img.split(',')[1]; const { _id } = await MongoImage.create({ - userId, - binary: Buffer.from(base64Data, 'base64') + teamId, + binary: Buffer.from(base64Data, 'base64'), + expiredTime }); return getMongoImgUrl(String(_id)); diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index fb7ea7e69..fbb484c00 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -1,16 +1,27 @@ +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; import { connectionMongo, type Model } from '../../mongo'; const { Schema, model, models } = connectionMongo; const ImageSchema = new Schema({ - userId: { + teamId: { type: Schema.Types.ObjectId, - ref: 'user', - required: true + ref: TeamCollectionName }, binary: { type: Buffer + }, + expiredTime: { + type: Date } }); -export const MongoImage: Model<{ userId: string; binary: Buffer }> = +try { + ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 }); +} catch (error) { + console.log(error); +} + +export const MongoImage: Model<{ teamId: string; binary: Buffer }> = models['image'] || model('image', ImageSchema); + +MongoImage.syncIndexes(); diff --git a/packages/service/core/app/schema.ts b/packages/service/core/app/schema.ts index 356f1412c..5240a9b70 100644 --- a/packages/service/core/app/schema.ts +++ b/packages/service/core/app/schema.ts @@ -67,3 +67,5 @@ try { export const MongoApp: Model = models[appCollectionName] || model(appCollectionName, AppSchema); + +MongoApp.syncIndexes(); diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index 6161f975c..ea7a4f5fd 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -83,3 +83,5 @@ try { export const MongoChatItem: Model = models['chatItem'] || model('chatItem', ChatItemSchema); + +MongoChatItem.syncIndexes(); diff --git a/packages/service/core/chat/chatSchema.ts b/packages/service/core/chat/chatSchema.ts index 71cd37729..66bacf8e7 100644 --- a/packages/service/core/chat/chatSchema.ts +++ b/packages/service/core/chat/chatSchema.ts @@ -92,7 +92,7 @@ const ChatSchema = new Schema({ }); try { - ChatSchema.index({ userId: 1 }); + ChatSchema.index({ tmbId: 1 }); ChatSchema.index({ updateTime: -1 }); ChatSchema.index({ appId: 1 }); } catch (error) { @@ -101,3 +101,4 @@ try { export const MongoChat: Model = models[chatCollectionName] || model(chatCollectionName, ChatSchema); +MongoChat.syncIndexes(); diff --git a/packages/service/core/chat/utils.ts b/packages/service/core/chat/utils.ts index 6272b7c56..3a5056dc7 100644 --- a/packages/service/core/chat/utils.ts +++ b/packages/service/core/chat/utils.ts @@ -1,7 +1,8 @@ import type { ChatItemType } from '@fastgpt/global/core/chat/type.d'; -import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; +import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants'; import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt'; +import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d'; /* slice chat context by tokens */ export function ChatContextFilter({ @@ -51,3 +52,101 @@ export function ChatContextFilter({ return [...systemPrompts, ...chats]; } + +/** + string to vision model. Follow the markdown code block rule for interception: + + @rule: + ```img-block + {src:""} + {src:""} + ``` + ```file-block + {name:"",src:""}, + {name:"",src:""} + ``` + @example: + What’s in this image? + ```img-block + {src:"https://1.png"} + ``` + @return + [ + { type: 'text', text: 'What’s in this image?' }, + { + type: 'image_url', + image_url: { + url: 'https://1.png' + } + } + ] + */ +export function formatStr2ChatContent(str: string) { + const content: ChatCompletionContentPart[] = []; + let lastIndex = 0; + const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g'); + + let match; + + while ((match = regex.exec(str)) !== null) { + // add previous text + if (match.index > lastIndex) { + const text = str.substring(lastIndex, match.index).trim(); + if (text) { + content.push({ type: 'text', text }); + } + } + + const blockType = match[1].trim(); + + if (blockType === IMG_BLOCK_KEY) { + const blockContentLines = match[2].trim().split('\n'); + const jsonLines = blockContentLines.map((item) => { + try { + return JSON.parse(item) as { src: string }; + } catch (error) { + return { src: '' }; + } + }); + + for (const item of jsonLines) { + if (!item.src) throw new Error("image block's content error"); + } + + content.push( + ...jsonLines.map((item) => ({ + type: 'image_url' as any, + image_url: { + url: item.src + } + })) + ); + } + + lastIndex = regex.lastIndex; + } + + // add remaining text + if (lastIndex < str.length) { + const remainingText = str.substring(lastIndex).trim(); + if (remainingText) { + content.push({ type: 'text', text: remainingText }); + } + } + + // Continuous text type content, if type=text, merge them + for (let i = 0; i < content.length - 1; i++) { + const currentContent = content[i]; + const nextContent = content[i + 1]; + if (currentContent.type === 'text' && nextContent.type === 'text') { + currentContent.text += nextContent.text; + content.splice(i + 1, 1); + i--; + } + } + + if (content.length === 1 && content[0].type === 'text') { + return content[0].text; + } + return content ? content : null; +} diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index baf81aa4a..20faa55ae 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -22,9 +22,9 @@ export async function findDatasetIdTreeByTopDatasetId( } export async function getCollectionWithDataset(collectionId: string) { - const data = ( - await MongoDatasetCollection.findById(collectionId).populate('datasetId') - )?.toJSON() as CollectionWithDatasetType; + const data = (await MongoDatasetCollection.findById(collectionId) + .populate('datasetId') + .lean()) as CollectionWithDatasetType; if (!data) { return Promise.reject('Collection is not exist'); } diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index 17bfe01ae..e1993c1a3 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -76,3 +76,4 @@ try { export const MongoDatasetData: Model = models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema); +MongoDatasetData.syncIndexes(); diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 6f61eb90a..5e024c800 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -82,3 +82,4 @@ try { export const MongoDataset: Model = models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema); +MongoDataset.syncIndexes(); diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index b7ce90969..d68f570da 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -104,3 +104,5 @@ try { export const MongoDatasetTraining: Model = models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema); + +MongoDatasetTraining.syncIndexes(); diff --git a/packages/service/core/plugin/schema.ts b/packages/service/core/plugin/schema.ts index 2aff509b8..ea933ff07 100644 --- a/packages/service/core/plugin/schema.ts +++ b/packages/service/core/plugin/schema.ts @@ -46,10 +46,11 @@ const PluginSchema = new Schema({ }); try { - PluginSchema.index({ userId: 1 }); + PluginSchema.index({ tmbId: 1 }); } catch (error) { console.log(error); } export const MongoPlugin: Model = models[ModuleCollectionName] || model(ModuleCollectionName, PluginSchema); +MongoPlugin.syncIndexes(); diff --git a/packages/service/support/activity/promotion/schema.ts b/packages/service/support/activity/promotion/schema.ts index 5f9a52a5b..263202d8d 100644 --- a/packages/service/support/activity/promotion/schema.ts +++ b/packages/service/support/activity/promotion/schema.ts @@ -31,3 +31,4 @@ const PromotionRecordSchema = new Schema({ export const MongoPromotionRecord: Model = models['promotionRecord'] || model('promotionRecord', PromotionRecordSchema); +MongoPromotionRecord.syncIndexes(); diff --git a/packages/service/support/openapi/schema.ts b/packages/service/support/openapi/schema.ts index ebedc52dd..d8f51f5a9 100644 --- a/packages/service/support/openapi/schema.ts +++ b/packages/service/support/openapi/schema.ts @@ -70,3 +70,4 @@ const OpenApiSchema = new Schema( export const MongoOpenApi: Model = models['openapi'] || model('openapi', OpenApiSchema); +MongoOpenApi.syncIndexes(); diff --git a/packages/service/support/outLink/schema.ts b/packages/service/support/outLink/schema.ts index 85b15fece..598a82902 100644 --- a/packages/service/support/outLink/schema.ts +++ b/packages/service/support/outLink/schema.ts @@ -71,3 +71,5 @@ const OutLinkSchema = new Schema({ export const MongoOutLink: Model = models['outlinks'] || model('outlinks', OutLinkSchema); + +MongoOutLink.syncIndexes(); diff --git a/packages/service/support/permission/auth/app.ts b/packages/service/support/permission/auth/app.ts index a15211f47..13cd86cd3 100644 --- a/packages/service/support/permission/auth/app.ts +++ b/packages/service/support/permission/auth/app.ts @@ -22,12 +22,12 @@ export async function authApp({ } > { const result = await parseHeaderCert(props); - const { userId, teamId, tmbId } = result; + const { teamId, tmbId } = result; const { role } = await getTeamInfoByTmbId({ tmbId }); const { app, isOwner, canWrite } = await (async () => { // get app - const app = (await MongoApp.findOne({ _id: appId, teamId }))?.toJSON(); + const app = await MongoApp.findOne({ _id: appId, teamId }).lean(); if (!app) { return Promise.reject(AppErrEnum.unAuthApp); } diff --git a/packages/service/support/permission/auth/chat.ts b/packages/service/support/permission/auth/chat.ts index ddad76e61..c0a89042e 100644 --- a/packages/service/support/permission/auth/chat.ts +++ b/packages/service/support/permission/auth/chat.ts @@ -24,9 +24,9 @@ export async function authChat({ const { chat, isOwner, canWrite } = await (async () => { // get chat - const chat = ( - await MongoChat.findOne({ chatId, teamId }).populate('appId') - )?.toJSON() as ChatWithAppSchema; + const chat = (await MongoChat.findOne({ chatId, teamId }) + .populate('appId') + .lean()) as ChatWithAppSchema; if (!chat) { return Promise.reject('Chat is not exists'); diff --git a/packages/service/support/permission/auth/dataset.ts b/packages/service/support/permission/auth/dataset.ts index 86ac29b60..84bca9474 100644 --- a/packages/service/support/permission/auth/dataset.ts +++ b/packages/service/support/permission/auth/dataset.ts @@ -31,7 +31,7 @@ export async function authDataset({ const { role } = await getTeamInfoByTmbId({ tmbId }); const { dataset, isOwner, canWrite } = await (async () => { - const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toObject(); + const dataset = await MongoDataset.findOne({ _id: datasetId, teamId }).lean(); if (!dataset) { return Promise.reject(DatasetErrEnum.unAuthDataset); diff --git a/packages/service/support/user/schema.ts b/packages/service/support/user/schema.ts index 995ff0953..6ad84a803 100644 --- a/packages/service/support/user/schema.ts +++ b/packages/service/support/user/schema.ts @@ -64,3 +64,4 @@ const UserSchema = new Schema({ export const MongoUser: Model = models[userCollectionName] || model(userCollectionName, UserSchema); +MongoUser.syncIndexes(); diff --git a/packages/service/support/wallet/bill/schema.ts b/packages/service/support/wallet/bill/schema.ts index 275577624..bd2e4138e 100644 --- a/packages/service/support/wallet/bill/schema.ts +++ b/packages/service/support/wallet/bill/schema.ts @@ -59,3 +59,4 @@ try { } export const MongoBill: Model = models['bill'] || model('bill', BillSchema); +MongoBill.syncIndexes(); diff --git a/projects/app/data/config.json b/projects/app/data/config.json index f675dc51e..556863d56 100644 --- a/projects/app/data/config.json +++ b/projects/app/data/config.json @@ -15,6 +15,7 @@ "quoteMaxToken": 2000, "maxTemperature": 1.2, "censor": false, + "vision": false, "defaultSystemChatPrompt": "" }, { @@ -26,6 +27,7 @@ "quoteMaxToken": 8000, "maxTemperature": 1.2, "censor": false, + "vision": false, "defaultSystemChatPrompt": "" }, { @@ -37,6 +39,19 @@ "quoteMaxToken": 4000, "maxTemperature": 1.2, "censor": false, + "vision": false, + "defaultSystemChatPrompt": "" + }, + { + "model": "gpt-4-vision-preview", + "name": "GPT4-Vision", + "maxContext": 128000, + "maxResponse": 4000, + "price": 0, + "quoteMaxToken": 100000, + "maxTemperature": 1.2, + "censor": false, + "vision": true, "defaultSystemChatPrompt": "" } ], diff --git a/projects/app/public/icon/speaking.gif b/projects/app/public/icon/speaking.gif new file mode 100644 index 000000000..bfc583841 Binary files /dev/null and b/projects/app/public/icon/speaking.gif differ diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 36b754852..4f9d7ccae 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -191,6 +191,7 @@ "Update Success": "Update Success", "Update Successful": "Update Successful", "Update Time": "Update Time", + "Upload File Failed": "Upload File Failed", "Username": "UserName", "error": { "unKnow": "There was an accident" diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 5340c1523..27007ef59 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -191,6 +191,7 @@ "Update Success": "更新成功", "Update Successful": "更新成功", "Update Time": "更新时间", + "Upload File Failed": "上传文件失败", "Username": "用户名", "error": { "unKnow": "出现了点意外~" diff --git a/projects/app/src/components/ChatBox/MessageInput.tsx b/projects/app/src/components/ChatBox/MessageInput.tsx index 98c5c5ec1..a4053d47c 100644 --- a/projects/app/src/components/ChatBox/MessageInput.tsx +++ b/projects/app/src/components/ChatBox/MessageInput.tsx @@ -1,7 +1,7 @@ import { useSpeech } from '@/web/common/hooks/useSpeech'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { Box, Flex, Image, Spinner, Textarea } from '@chakra-ui/react'; -import React, { useRef, useEffect, useCallback, useState, useMemo } from 'react'; +import React, { useRef, useEffect, useCallback, useState } from 'react'; import { useTranslation } from 'react-i18next'; import MyTooltip from '../MyTooltip'; import MyIcon from '../Icon'; @@ -10,6 +10,23 @@ import { useRouter } from 'next/router'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { compressImgAndUpload } from '@/web/common/file/controller'; import { useToast } from '@/web/common/hooks/useToast'; +import { customAlphabet } from 'nanoid'; +import { IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants'; +import { addDays } from 'date-fns'; +const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6); + +enum FileTypeEnum { + image = 'image', + file = 'file' +} +type FileItemType = { + id: string; + rawFile: File; + type: `${FileTypeEnum}`; + name: string; + icon: string; // img is base64 + src?: string; +}; const MessageInput = ({ onChange, @@ -17,16 +34,19 @@ const MessageInput = ({ onStop, isChatting, TextareaDom, + showFileSelector = false, resetInputVal }: { onChange: (e: string) => void; onSendMessage: (e: string) => void; onStop: () => void; isChatting: boolean; + showFileSelector?: boolean; TextareaDom: React.MutableRefObject; resetInputVal: (val: string) => void; }) => { const { shareId } = useRouter().query as { shareId?: string }; + const { toast } = useToast(); const { isSpeaking, isTransCription, @@ -37,64 +57,106 @@ const MessageInput = ({ stream } = useSpeech({ shareId }); const { isPc } = useSystemStore(); - const canvasRef = useRef(); + const canvasRef = useRef(null); const { t } = useTranslation(); const textareaMinH = '22px'; - const havInput = !!TextareaDom.current?.value; - const { toast } = useToast(); - const [imgBase64Array, setImgBase64Array] = useState([]); - const [fileList, setFileList] = useState([]); - const [imgSrcArray, setImgSrcArray] = useState([]); + const [fileList, setFileList] = useState([]); + const havInput = !!TextareaDom.current?.value || fileList.length > 0; const { File, onOpen: onOpenSelectFile } = useSelectFile({ - fileType: '.jpg,.png', - multiple: true + fileType: 'image/*', + multiple: true, + maxCount: 10 }); - useEffect(() => { - fileList.forEach((file) => { - const reader = new FileReader(); - reader.readAsDataURL(file); - reader.onload = async () => { - setImgBase64Array((prev) => [...prev, reader.result as string]); - }; - }); - }, [fileList]); - - const onSelectFile = useCallback((e: File[]) => { - if (!e || e.length === 0) { + const uploadFile = async (file: FileItemType) => { + if (file.type === FileTypeEnum.image) { + try { + const src = await compressImgAndUpload({ + file: file.rawFile, + maxW: 1000, + maxH: 1000, + maxSize: 1024 * 1024 * 5, + // 30 day expired. + expiredTime: addDays(new Date(), 30) + }); + setFileList((state) => + state.map((item) => + item.id === file.id + ? { + ...item, + src: `${location.origin}${src}` + } + : item + ) + ); + } catch (error) { + setFileList((state) => state.filter((item) => item.id !== file.id)); + toast({ + status: 'error', + title: t('common.Upload File Failed') + }); + } + } + }; + const onSelectFile = useCallback(async (files: File[]) => { + if (!files || files.length === 0) { return; } - setFileList(e); + const loadFiles = await Promise.all( + files.map( + (file) => + new Promise((resolve, reject) => { + if (file.type.includes('image')) { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => { + const item = { + id: nanoid(), + rawFile: file, + type: FileTypeEnum.image, + name: file.name, + icon: reader.result as string + }; + uploadFile(item); + resolve(item); + }; + reader.onerror = () => { + reject(reader.error); + }; + } else { + resolve({ + id: nanoid(), + rawFile: file, + type: FileTypeEnum.file, + name: file.name, + icon: 'pdf' + }); + } + }) + ) + ); + + setFileList((state) => [...state, ...loadFiles]); }, []); const handleSend = useCallback(async () => { - try { - for (const file of fileList) { - const src = await compressImgAndUpload({ - file, - maxW: 1000, - maxH: 1000, - maxSize: 1024 * 1024 * 2 - }); - imgSrcArray.push(src); - } - } catch (err: any) { - toast({ - title: typeof err === 'string' ? err : '文件上传异常', - status: 'warning' - }); - } - const textareaValue = TextareaDom.current?.value || ''; - const inputMessage = - imgSrcArray.length === 0 - ? textareaValue - : `\`\`\`img-block\n${JSON.stringify(imgSrcArray)}\n\`\`\`\n${textareaValue}`; + + const images = fileList.filter((item) => item.type === FileTypeEnum.image); + const imagesText = + images.length === 0 + ? '' + : `\`\`\`${IMG_BLOCK_KEY} +${images.map((img) => JSON.stringify({ src: img.src })).join('\n')} +\`\`\` +`; + + const inputMessage = `${imagesText}${textareaValue}`; + onSendMessage(inputMessage); - setImgBase64Array([]); - setImgSrcArray([]); - }, [TextareaDom, fileList, imgSrcArray, onSendMessage, toast]); + setFileList([]); + }, [TextareaDom, fileList, onSendMessage]); useEffect(() => { if (!stream) { @@ -107,117 +169,139 @@ const MessageInput = ({ const source = audioContext.createMediaStreamSource(stream); source.connect(analyser); const renderCurve = () => { - renderAudioGraph(analyser, canvasRef.current as HTMLCanvasElement); + if (!canvasRef.current) return; + renderAudioGraph(analyser, canvasRef.current); window.requestAnimationFrame(renderCurve); }; renderCurve(); }, [renderAudioGraph, stream]); return ( - <> - - 0 ? '8px' : '18px'} - position={'relative'} - boxShadow={isSpeaking ? `0 0 10px rgba(54,111,255,0.4)` : `0 0 10px rgba(0,0,0,0.2)`} - {...(isPc - ? { - border: '1px solid', - borderColor: 'rgba(0,0,0,0.12)' - } - : { - borderTop: '1px solid', - borderTopColor: 'rgba(0,0,0,0.15)' - })} - borderRadius={['none', 'md']} - backgroundColor={'white'} + + 0 ? '10px' : ['14px', '18px']} + pb={['14px', '18px']} + position={'relative'} + boxShadow={isSpeaking ? `0 0 10px rgba(54,111,255,0.4)` : `0 0 10px rgba(0,0,0,0.2)`} + borderRadius={['none', 'md']} + bg={'white'} + {...(isPc + ? { + border: '1px solid', + borderColor: 'rgba(0,0,0,0.12)' + } + : { + borderTop: '1px solid', + borderTopColor: 'rgba(0,0,0,0.15)' + })} + > + {/* translate loading */} + - {/* translate loading */} - - - {t('chat.Converting to text')} - - {/* file uploader */} - - - - - - - {/* file preview */} - - {imgBase64Array.length > 0 && - imgBase64Array.map((src, index) => ( - + {t('chat.Converting to text')} + + + {/* file preview */} + + {fileList.map((item) => ( + + {/* uploading */} + {!item.src && ( + - { - setImgBase64Array((prev) => { - prev.splice(index, 1); - return [...prev]; - }); - }} - className="close-icon" - display={['', 'none']} - /> - {'img'} - - ))} - + + + )} + { + setFileList((state) => state.filter((file) => file.id !== item.id)); + }} + className="close-icon" + display={['', 'none']} + /> + {item.type === FileTypeEnum.image && ( + {'img'} + )} + + ))} + + + 0 ? 1 : 0} pl={[2, 4]}> + {/* file selector */} + {showFileSelector && ( + { + if (isSpeaking) return; + onOpenSelectFile; + }} + > + + + + + + )} + {/* input area */}