ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 13:03:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

add image input (#486)

Browse Source 
* add image input

* use json
This commit is contained in:
heheer
2023-11-17 18:22:29 +08:00
committed by GitHub
parent af16817a4a
commit 70f3373246
9 changed files with 181 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
projects/app/public/locales/en/common.json
View File

@@ -233,10 +233,11 @@
},
"chat": {
"Audio Speech Error": "Audio Speech Error",
"Speaking": "I'm listening...",
"Record": "Speech",
"Restart": "Restart",
"Select File": "Select file",
"Send Message": "Send Message",
"Speaking": "I'm listening...",
"Stop Speak": "Stop Speak",
"Type a message": "Input problem",
"tts": {
@@ -589,8 +590,8 @@
"wallet": {
"bill": {
"Audio Speech": "Audio Speech",
"bill username": "User",
"Whisper": "Whisper"
"Whisper": "Whisper",
"bill username": "User"
}
}
}

5
projects/app/public/locales/zh/common.json
View File

@@ -235,6 +235,7 @@
"Audio Speech Error": "语音播报异常",
"Record": "语音输入",
"Restart": "重开对话",
"Select File": "选择文件",
"Send Message": "发送",
"Speaking": "我在听，请说...",
"Stop Speak": "停止录音",
@@ -589,8 +590,8 @@
"wallet": {
"bill": {
"Audio Speech": "语音播报",
"bill username": "用户",
"Whisper": "语音输入"
"Whisper": "语音输入",
"bill username": "用户"
}
}
}

145
projects/app/src/components/ChatBox/MessageInput.tsx
View File

@@ -1,12 +1,15 @@
import { useSpeech } from '@/web/common/hooks/useSpeech';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { Box, Flex, Spinner, Textarea } from '@chakra-ui/react';
import React, { useRef, useEffect } from 'react';
import { Box, Flex, Image, Spinner, Textarea } from '@chakra-ui/react';
import React, { useRef, useEffect, useCallback, useState, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import MyTooltip from '../MyTooltip';
import MyIcon from '../Icon';
import styles from './index.module.scss';
import { useRouter } from 'next/router';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { compressImgAndUpload } from '@/web/common/file/controller';
import { useToast } from '@/web/common/hooks/useToast';
const MessageInput = ({
onChange,
@@ -38,6 +41,60 @@ const MessageInput = ({
const { t } = useTranslation();
const textareaMinH = '22px';
const havInput = !!TextareaDom.current?.value;
const { toast } = useToast();
const [imgBase64Array, setImgBase64Array] = useState<string[]>([]);
const [fileList, setFileList] = useState<File[]>([]);
const [imgSrcArray, setImgSrcArray] = useState<string[]>([]);
const { File, onOpen: onOpenSelectFile } = useSelectFile({
fileType: '.jpg,.png',
multiple: true
});
useEffect(() => {
fileList.forEach((file) => {
const reader = new FileReader();
reader.readAsDataURL(file);
reader.onload = async () => {
setImgBase64Array((prev) => [...prev, reader.result as string]);
};
});
}, [fileList]);
const onSelectFile = useCallback((e: File[]) => {
if (!e || e.length === 0) {
return;
}
setFileList(e);
}, []);
const handleSend = useCallback(async () => {
try {
for (const file of fileList) {
const src = await compressImgAndUpload({
file,
maxW: 1000,
maxH: 1000,
maxSize: 1024 * 1024 * 2
});
imgSrcArray.push(src);
}
} catch (err: any) {
toast({
title: typeof err === 'string' ? err : '文件上传异常',
status: 'warning'
});
}
const textareaValue = TextareaDom.current?.value || '';
const inputMessage =
imgSrcArray.length === 0
? textareaValue
: `\`\`\`img-block\n${JSON.stringify(imgSrcArray)}\n\`\`\`\n${textareaValue}`;
onSendMessage(inputMessage);
setImgBase64Array([]);
setImgSrcArray([]);
}, [TextareaDom, fileList, imgSrcArray, onSendMessage, toast]);
useEffect(() => {
if (!stream) {
@@ -60,7 +117,7 @@ const MessageInput = ({
<>
<Box m={['0 auto', '10px auto']} w={'100%'} maxW={['auto', 'min(800px, 100%)']} px={[0, 5]}>
<Box
py={'18px'}
py={imgBase64Array.length > 0 ? '8px' : '18px'}
position={'relative'}
boxShadow={isSpeaking ? `0 0 10px rgba(54,111,255,0.4)` : `0 0 10px rgba(0,0,0,0.2)`}
{...(isPc
@@ -93,11 +150,74 @@ const MessageInput = ({
<Spinner size={'sm'} mr={4} />
{t('chat.Converting to text')}
</Box>
{/* file uploader */}
<Flex
position={'absolute'}
alignItems={'center'}
left={['12px', '14px']}
bottom={['15px', '13px']}
h={['26px', '32px']}
zIndex={10}
cursor={'pointer'}
onClick={onOpenSelectFile}
>
<MyTooltip label={t('core.chat.Select File')}>
<MyIcon name={'core/chat/fileSelect'} />
</MyTooltip>
<File onSelect={onSelectFile} />
</Flex>
{/* file preview */}
<Flex w={'96%'} wrap={'wrap'} ml={4}>
{imgBase64Array.length > 0 &&
imgBase64Array.map((src, index) => (
<Box
key={index}
border={'1px solid rgba(0,0,0,0.12)'}
mr={2}
mb={2}
rounded={'md'}
position={'relative'}
_hover={{
'.close-icon': { display: 'block' }
}}
>
<MyIcon
name={'closeSolid'}
w={'16px'}
h={'16px'}
color={'myGray.700'}
cursor={'pointer'}
_hover={{ color: 'myBlue.600' }}
position={'absolute'}
right={-2}
top={-2}
onClick={() => {
setImgBase64Array((prev) => {
prev.splice(index, 1);
return [...prev];
});
}}
className="close-icon"
display={['', 'none']}
/>
<Image
alt={'img'}
src={src}
w={'80px'}
h={'80px'}
rounded={'md'}
objectFit={'cover'}
/>
</Box>
))}
</Flex>
{/* input area */}
<Textarea
ref={TextareaDom}
py={0}
pr={['45px', '55px']}
pl={['36px', '40px']}
mt={imgBase64Array.length > 0 ? 4 : 0}
border={'none'}
_focusVisible={{
border: 'none'
@@ -124,13 +244,28 @@ const MessageInput = ({
onKeyDown={(e) => {
// enter send.(pc or iframe && enter and unPress shift)
if ((isPc || window !== parent) && e.keyCode === 13 && !e.shiftKey) {
onSendMessage(TextareaDom.current?.value || '');
handleSend();
e.preventDefault();
}
// 全选内容
// @ts-ignore
e.key === 'a' && e.ctrlKey && e.target?.select();
}}
onPaste={(e) => {
const clipboardData = e.clipboardData;
if (clipboardData) {
const items = clipboardData.items;
const files: File[] = [];
for (let i = 0; i < items.length; i++) {
const item = items[i];
if (item.kind === 'file') {
const file = item.getAsFile();
files.push(file as File);
}
}
setFileList(files);
}
}}
/>
<Flex
position={'absolute'}
@@ -195,7 +330,7 @@ const MessageInput = ({
return onStop();
}
if (havInput) {
onSendMessage(TextareaDom.current?.value || '');
return handleSend();
}
}}
>

15
projects/app/src/components/ChatBox/index.tsx
View File

@@ -17,18 +17,7 @@ import { useToast } from '@/web/common/hooks/useToast';
import { useAudioPlay } from '@/web/common/utils/voice';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useCopyData } from '@/web/common/hooks/useCopyData';
import {
Box,
Card,
Flex,
Input,
Textarea,
Button,
useTheme,
BoxProps,
FlexProps,
Spinner
} from '@chakra-ui/react';
import { Box, Card, Flex, Input, Button, useTheme, BoxProps, FlexProps } from '@chakra-ui/react';
import { feConfigs } from '@/web/common/system/staticData';
import { eventBus } from '@/web/common/utils/eventbus';
import { adaptChat2GptMessages } from '@fastgpt/global/core/chat/adapt';
@@ -633,7 +622,7 @@ const ChatBox = (
borderRadius={'8px 0 8px 8px'}
textAlign={'left'}
>
<Box as={'p'}>{item.value}</Box>
<Markdown source={item.value} isChatting={false} />
</Card>
</Box>
</>

3
projects/app/src/components/Icon/icons/core/chat/fileSelect.svg Normal file
View File

@@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" width="17" height="18" viewBox="0 0 19 20" fill="none">
<path fill-rule="evenodd" clip-rule="evenodd" d="M13.15 1.98982C12.4865 1.98982 11.8501 2.2534 11.3809 2.72259L3.72259 10.3809C2.94066 11.1628 2.50138 12.2234 2.50138 13.3292C2.50138 14.435 2.94066 15.4955 3.72259 16.2774C4.50451 17.0593 5.56502 17.4986 6.67083 17.4986C7.77664 17.4986 8.83715 17.0593 9.61907 16.2774L17.2774 8.61908C17.6028 8.29364 18.1305 8.29364 18.4559 8.61908C18.7814 8.94452 18.7814 9.47216 18.4559 9.79759L10.7976 17.4559C9.7031 18.5504 8.21866 19.1653 6.67083 19.1653C5.123 19.1653 3.63856 18.5504 2.54407 17.4559C1.44959 16.3614 0.834717 14.877 0.834717 13.3292C0.834717 11.7813 1.44959 10.2969 2.54407 9.20242L10.2024 1.54408C10.9842 0.762333 12.0444 0.323151 13.15 0.323151C14.2556 0.323151 15.3158 0.762332 16.0976 1.54408C16.8793 2.32583 17.3185 3.38611 17.3185 4.49167C17.3185 5.59723 16.8793 6.65751 16.0976 7.43926L8.43092 15.0976C7.96191 15.5666 7.32579 15.8301 6.6625 15.8301C5.99921 15.8301 5.36309 15.5666 4.89407 15.0976C4.42506 14.6286 4.16157 13.9925 4.16157 13.3292C4.16157 12.6659 4.42506 12.0298 4.89407 11.5607L11.9694 4.49373C12.2951 4.16849 12.8227 4.1688 13.1479 4.49443C13.4732 4.82006 13.4729 5.34769 13.1472 5.67294L6.07259 12.7393C5.91635 12.8957 5.82824 13.1081 5.82824 13.3292C5.82824 13.5504 5.91613 13.7626 6.07259 13.9191C6.22904 14.0755 6.44124 14.1634 6.6625 14.1634C6.88376 14.1634 7.09595 14.0755 7.25241 13.9191L14.9191 6.26075C15.3881 5.79159 15.6519 5.15505 15.6519 4.49167C15.6519 3.82814 15.3883 3.19178 14.9191 2.72259C14.4499 2.2534 13.8135 1.98982 13.15 1.98982Z" fill="#485058"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

3
projects/app/src/components/Icon/index.tsx
View File

@@ -112,7 +112,8 @@ const iconPaths = {
'core/chat/recordFill': () => import('./icons/core/chat/recordFill.svg'),
'core/chat/stopSpeechFill': () => import('./icons/core/chat/stopSpeechFill.svg'),
'core/chat/stopSpeech': () => import('./icons/core/chat/stopSpeech.svg'),
'core/chat/speaking': () => import('./icons/core/chat/speaking.svg')
'core/chat/speaking': () => import('./icons/core/chat/speaking.svg'),
'core/chat/fileSelect': () => import('./icons/core/chat/fileSelect.svg')
};
export type IconName = keyof typeof iconPaths;

18
projects/app/src/components/Markdown/chat/Image.tsx Normal file
View File

@@ -0,0 +1,18 @@
import { Box, Flex } from '@chakra-ui/react';
import MdImage from '../img/Image';
const ImageBlock = ({ images }: { images: string }) => {
return (
<Flex w={'100%'} wrap={'wrap'}>
{JSON.parse(images).map((src: string) => {
return (
<Box key={src} mr={2} mb={2} rounded={'md'} flex={'0 0 auto'} w={'100px'} h={'100px'}>
<MdImage src={src} />
</Box>
);
})}
</Flex>
);
};
export default ImageBlock;

7
projects/app/src/components/Markdown/index.tsx
View File

@@ -16,12 +16,14 @@ const MdImage = dynamic(() => import('./img/Image'));
const ChatGuide = dynamic(() => import('./chat/Guide'));
const EChartsCodeBlock = dynamic(() => import('./img/EChartsCodeBlock'));
const QuoteBlock = dynamic(() => import('./chat/Quote'));
const ImageBlock = dynamic(() => import('./chat/Image'));
export enum CodeClassName {
guide = 'guide',
mermaid = 'mermaid',
echarts = 'echarts',
quote = 'quote'
quote = 'quote',
img = 'img'
}
function Code({ inline, className, children }: any) {
@@ -41,6 +43,9 @@ function Code({ inline, className, children }: any) {
if (codeType === CodeClassName.quote) {
return <QuoteBlock code={String(children)} />;
}
if (codeType === CodeClassName.img) {
return <ImageBlock images={String(children)} />;
}
return (
<CodeLight className={className} inline={inline} match={match}>
{children}

6
projects/app/src/web/common/hooks/useSpeech.ts
View File

@@ -12,7 +12,7 @@ export const useSpeech = (props?: { shareId?: string }) => {
const { toast } = useToast();
const [isSpeaking, setIsSpeaking] = useState(false);
const [isTransCription, setIsTransCription] = useState(false);
const [audioSecond, setAudioSecone] = useState(0);
const [audioSecond, setAudioSecond] = useState(0);
const intervalRef = useRef<any>();
const startTimestamp = useRef(0);
@@ -59,11 +59,11 @@ export const useSpeech = (props?: { shareId?: string }) => {
mediaRecorder.current.onstart = () => {
startTimestamp.current = Date.now();
setAudioSecone(0);
setAudioSecond(0);
intervalRef.current = setInterval(() => {
const currentTimestamp = Date.now();
const duration = (currentTimestamp - startTimestamp.current) / 1000;
setAudioSecone(duration);
setAudioSecond(duration);
}, 1000);
};