diff --git a/package.json b/package.json index 7a82e8cb5..faa8bf9af 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "format-doc": "zhlint --dir ./docSite *.md --fix" }, "devDependencies": { + "@types/multer": "^1.4.10", "husky": "^8.0.3", "i18next": "^22.5.1", "lint-staged": "^13.2.1", @@ -24,6 +25,7 @@ "node": ">=18.0.0" }, "dependencies": { + "multer": "1.4.5-lts.1", "openai": "4.16.1" } } diff --git a/packages/service/common/file/upload/multer.ts b/packages/service/common/file/upload/multer.ts new file mode 100644 index 000000000..ee0cb8ecd --- /dev/null +++ b/packages/service/common/file/upload/multer.ts @@ -0,0 +1,71 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { customAlphabet } from 'nanoid'; +import multer from 'multer'; +import path from 'path'; +import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; + +const nanoid = customAlphabet('1234567890abcdef', 12); + +type FileType = { + fieldname: string; + originalname: string; + encoding: string; + mimetype: string; + filename: string; + path: string; + size: number; +}; + +export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { + maxSize *= 1024 * 1024; + class UploadModel { + uploader = multer({ + limits: { + fieldSize: maxSize + }, + preservePath: true, + storage: multer.diskStorage({ + filename: (_req, file, cb) => { + const { ext } = path.parse(decodeURIComponent(file.originalname)); + cb(null, nanoid() + ext); + } + }) + }).any(); + + async doUpload(req: NextApiRequest, res: NextApiResponse) { + return new Promise<{ + files: FileType[]; + metadata: Record; + bucketName?: `${BucketNameEnum}`; + }>((resolve, reject) => { + // @ts-ignore + this.uploader(req, res, (error) => { + if (error) { + return reject(error); + } + + resolve({ + ...req.body, + files: + // @ts-ignore + req.files?.map((file) => ({ + ...file, + originalname: decodeURIComponent(file.originalname) + })) || [], + metadata: (() => { + if (!req.body?.metadata) return {}; + try { + return JSON.parse(req.body.metadata); + } catch (error) { + console.log(error); + return {}; + } + })() + }); + }); + }); + } + } + + return new UploadModel(); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b6a1d697..3f2225007 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,10 +8,16 @@ importers: .: dependencies: + multer: + specifier: 1.4.5-lts.1 + version: registry.npmmirror.com/multer@1.4.5-lts.1 openai: specifier: 4.16.1 version: registry.npmmirror.com/openai@4.16.1(encoding@0.1.13) devDependencies: + '@types/multer': + specifier: ^1.4.10 + version: registry.npmmirror.com/@types/multer@1.4.10 husky: specifier: ^8.0.3 version: registry.npmmirror.com/husky@8.0.3 @@ -307,8 +313,8 @@ importers: specifier: ^4.14.191 version: registry.npmmirror.com/@types/lodash@4.14.200 '@types/multer': - specifier: ^1.4.7 - version: registry.npmmirror.com/@types/multer@1.4.9 + specifier: ^1.4.10 + version: registry.npmmirror.com/@types/multer@1.4.10 '@types/node': specifier: ^20.8.5 version: registry.npmmirror.com/@types/node@20.8.7 @@ -4522,10 +4528,10 @@ packages: version: 0.7.33 dev: false - registry.npmmirror.com/@types/multer@1.4.9: - resolution: {integrity: sha512-9NSvPJ2E8bNTc8XtJq1Cimx2Wrn2Ah48F15B2Du/hM8a8CHLhVbJMlF3ZCqhvMdht7Sa+YdP0aKP7N4fxDcrrg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/multer/-/multer-1.4.9.tgz} + registry.npmmirror.com/@types/multer@1.4.10: + resolution: {integrity: sha512-6l9mYMhUe8wbnz/67YIjc7ZJyQNZoKq7fRXVf7nMdgWgalD0KyzJ2ywI7hoATUSXSbTu9q2HBiEwzy0tNN1v2w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/multer/-/multer-1.4.10.tgz} name: '@types/multer' - version: 1.4.9 + version: 1.4.10 dependencies: '@types/express': registry.npmmirror.com/@types/express@4.17.20 dev: true diff --git a/projects/app/package.json b/projects/app/package.json index 38fb44272..5e1565a79 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -72,7 +72,7 @@ "@types/jsdom": "^21.1.1", "@types/jsonwebtoken": "^9.0.3", "@types/lodash": "^4.14.191", - "@types/multer": "^1.4.7", + "@types/multer": "^1.4.10", "@types/node": "^20.8.5", "@types/papaparse": "^5.3.7", "@types/react": "18.0.28", diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 5b640a465..265c59a7d 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -203,6 +203,9 @@ }, "input": { "Repeat Value": "Repeat Value" + }, + "speech": { + "error tip": "Speech Failed" } }, "core": { @@ -226,8 +229,10 @@ }, "chat": { "Audio Speech Error": "Audio Speech Error", + "Record": "Speech", "Restart": "Restart", - "Send Message": "Send Message" + "Send Message": "Send Message", + "Stop Speak": "Stop Speak" }, "dataset": { "Choose Dataset": "Choose Dataset", diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 282b5632e..07adf070e 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -203,6 +203,9 @@ }, "input": { "Repeat Value": "有重复的值" + }, + "speech": { + "error tip": "语音转文字失败" } }, "core": { @@ -226,8 +229,10 @@ }, "chat": { "Audio Speech Error": "语音播报异常", + "Record": "语音输入", "Restart": "重开对话", - "Send Message": "发送" + "Send Message": "发送", + "Stop Speak": "停止录音" }, "dataset": { "Choose Dataset": "关联知识库", diff --git a/projects/app/src/components/ChatBox/index.tsx b/projects/app/src/components/ChatBox/index.tsx index a47905f53..fb326f55a 100644 --- a/projects/app/src/components/ChatBox/index.tsx +++ b/projects/app/src/components/ChatBox/index.tsx @@ -62,6 +62,7 @@ import styles from './index.module.scss'; import { postQuestionGuide } from '@/web/core/ai/api'; import { splitGuideModule } from '@/global/core/app/modules/utils'; import { AppTTSConfigType } from '@/types/app'; +import { useSpeech } from '@/web/common/hooks/useSpeech'; const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24); @@ -149,6 +150,8 @@ const ChatBox = ( const [adminMarkData, setAdminMarkData] = useState(); const [questionGuides, setQuestionGuide] = useState([]); + const { isSpeaking, startSpeak, stopSpeak } = useSpeech(); + const isChatting = useMemo( () => chatHistory[chatHistory.length - 1] && @@ -857,8 +860,22 @@ const ChatBox = ( right={['12px', '14px']} bottom={['15px', '13px']} borderRadius={'md'} - bg={TextareaDom.current?.value ? 'myBlue.600' : ''} + // bg={TextareaDom.current?.value ? 'myBlue.600' : ''} + cursor={'pointer'} lineHeight={1} + onClick={() => { + if (isChatting) { + return chatController.current?.abort('stop'); + } + if (TextareaDom.current?.value) { + return handleSubmit((data) => sendPrompt(data, TextareaDom.current?.value))(); + } + // speech + // if (isSpeaking) { + // return stopSpeak(); + // } + // startSpeak(); + }} > {isChatting ? ( chatController.current?.abort('stop')} /> ) : ( { - handleSubmit((data) => sendPrompt(data, TextareaDom.current?.value))(); - }} + width={['16px', '22px']} + height={['16px', '22px']} + color={TextareaDom.current?.value ? 'myBlue.600' : 'myGray.400'} /> )} diff --git a/projects/app/src/components/Icon/icons/core/chat/recordFill.svg b/projects/app/src/components/Icon/icons/core/chat/recordFill.svg new file mode 100644 index 000000000..959dea3f7 --- /dev/null +++ b/projects/app/src/components/Icon/icons/core/chat/recordFill.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/projects/app/src/components/Icon/icons/core/chat/stopSpeechFill.svg b/projects/app/src/components/Icon/icons/core/chat/stopSpeechFill.svg new file mode 100644 index 000000000..7fa7b69ab --- /dev/null +++ b/projects/app/src/components/Icon/icons/core/chat/stopSpeechFill.svg @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/projects/app/src/components/Icon/index.tsx b/projects/app/src/components/Icon/index.tsx index 1cdde2733..5f24622f7 100644 --- a/projects/app/src/components/Icon/index.tsx +++ b/projects/app/src/components/Icon/index.tsx @@ -106,7 +106,9 @@ const iconPaths = { 'support/permission/publicLight': () => import('./icons/support/permission/publicLight.svg'), 'core/app/ttsFill': () => import('./icons/core/app/ttsFill.svg'), 'common/playLight': () => import('./icons/common/playLight.svg'), - 'core/chat/sendFill': () => import('./icons/core/chat/sendFill.svg') + 'core/chat/sendFill': () => import('./icons/core/chat/sendFill.svg'), + 'core/chat/recordFill': () => import('./icons/core/chat/recordFill.svg'), + 'core/chat/stopSpeechFill': () => import('./icons/core/chat/stopSpeechFill.svg') }; export type IconName = keyof typeof iconPaths; diff --git a/projects/app/src/components/Layout/index.tsx b/projects/app/src/components/Layout/index.tsx index 767bbd3e3..c165359df 100644 --- a/projects/app/src/components/Layout/index.tsx +++ b/projects/app/src/components/Layout/index.tsx @@ -67,7 +67,7 @@ const Layout = ({ children }: { children: JSX.Element }) => { }, [loadGitStar, setScreenWidth]); const { data: unread = 0 } = useQuery(['getUnreadCount'], getUnreadCount, { - enabled: !!userInfo && feConfigs.isPlus, + enabled: !!userInfo && !!feConfigs.isPlus, refetchInterval: 10000 }); diff --git a/projects/app/src/components/common/Textarea/TagTextarea.tsx b/projects/app/src/components/common/Textarea/TagTextarea.tsx index 4313b3de4..5b26a8189 100644 --- a/projects/app/src/components/common/Textarea/TagTextarea.tsx +++ b/projects/app/src/components/common/Textarea/TagTextarea.tsx @@ -29,7 +29,7 @@ const TagTextarea = ({ defaultValues, onUpdate, ...props }: Props) => { return; } if (tags.includes(value)) { - toast({ + return toast({ status: 'warning', title: t('common.input.Repeat Value') }); diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx index 15db35d3d..642eff149 100644 --- a/projects/app/src/pages/_app.tsx +++ b/projects/app/src/pages/_app.tsx @@ -88,11 +88,11 @@ function App({ Component, pageProps }: AppProps) { setLastRoute(router.asPath); }; }, [router.asPath]); - ``; + return ( <> - {feConfigs?.systemTitle || 'AI'} + {feConfigs?.systemTitle || 'FastGPT'} ({ ...file, originalname: decodeURIComponent(file.originalname) })) || [], - bucketName: req.body.bucketName, metadata: (() => { if (!req.body?.metadata) return {}; try { @@ -80,6 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< try { await connectToDatabase(); const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); + console.log(req.body); const { files, bucketName, metadata } = await upload.doUpload(req, res); diff --git a/projects/app/src/pages/api/core/dataset/data/exportAll.ts b/projects/app/src/pages/api/core/dataset/data/exportAll.ts index 9af6d38d2..f8d8ce464 100644 --- a/projects/app/src/pages/api/core/dataset/data/exportAll.ts +++ b/projects/app/src/pages/api/core/dataset/data/exportAll.ts @@ -5,7 +5,7 @@ import { MongoUser } from '@fastgpt/service/support/user/schema'; import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant'; import { findAllChildrenIds } from '../delete'; import QueryStream from 'pg-query-stream'; -import { PgClient, Pg } from '@fastgpt/service/common/pg'; +import { PgClient } from '@fastgpt/service/common/pg'; import { addLog } from '@fastgpt/service/common/mongo/controller'; import { responseWriteController } from '@fastgpt/service/common/response'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; @@ -17,7 +17,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< datasetId: string; }; - if (!datasetId || !Pg) { + if (!datasetId || !global.pgClient) { throw new Error('缺少参数'); } @@ -61,7 +61,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } // connect pg - Pg.connect((err, client, done) => { + global.pgClient.connect((err, client, done) => { if (err) { console.error(err); res.end('Error connecting to database'); diff --git a/projects/app/src/pages/api/v1/audio/transcriptions.ts b/projects/app/src/pages/api/v1/audio/transcriptions.ts new file mode 100644 index 000000000..7294b7e3a --- /dev/null +++ b/projects/app/src/pages/api/v1/audio/transcriptions.ts @@ -0,0 +1,48 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { withNextCors } from '@fastgpt/service/common/middle/cors'; +import { getUploadModel } from '@fastgpt/service/common/file/upload/multer'; +import fs from 'fs'; +import { getAIApi } from '@fastgpt/service/core/ai/config'; + +const upload = getUploadModel({ + maxSize: 2 +}); + +export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { teamId, tmbId } = await authCert({ req, authToken: true }); + + const { files } = await upload.doUpload(req, res); + + const file = files[0]; + + if (!file) { + throw new Error('file not found'); + } + + const ai = getAIApi(); + + const result = await ai.audio.transcriptions.create({ + file: fs.createReadStream(file.path), + model: 'whisper-1' + }); + + jsonRes(res, { + data: result.text + }); + } catch (err) { + console.log(err); + jsonRes(res, { + code: 500, + error: err + }); + } +}); + +export const config = { + api: { + bodyParser: false + } +}; diff --git a/projects/app/src/web/common/hooks/useSpeech.ts b/projects/app/src/web/common/hooks/useSpeech.ts new file mode 100644 index 000000000..a341b29f6 --- /dev/null +++ b/projects/app/src/web/common/hooks/useSpeech.ts @@ -0,0 +1,70 @@ +import { useEffect, useRef, useState } from 'react'; +import { POST } from '../api/request'; +import { useToast } from './useToast'; +import { useTranslation } from 'react-i18next'; +import { getErrText } from '@fastgpt/global/common/error/utils'; + +export const useSpeech = () => { + const { t } = useTranslation(); + const mediaRecorder = useRef(); + const { toast } = useToast(); + const [isSpeaking, setIsSpeaking] = useState(false); + + const startSpeak = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + mediaRecorder.current = new MediaRecorder(stream); + const chunks: Blob[] = []; + + mediaRecorder.current.ondataavailable = (e) => { + chunks.push(e.data); + }; + + mediaRecorder.current.onstop = async () => { + const formData = new FormData(); + const blob = new Blob(chunks, { type: 'audio/webm' }); + formData.append('files', blob, 'recording.webm'); + + const link = document.createElement('a'); + link.href = URL.createObjectURL(blob); + link.download = 'recording.webm'; + document.body.appendChild(link); + link.click(); + link.remove(); + + try { + const result = await POST('/v1/audio/transcriptions', formData, { + timeout: 60000, + headers: { + 'Content-Type': 'multipart/form-data; charset=utf-8' + } + }); + + console.log(result, '==='); + } catch (error) { + toast({ + status: 'warning', + title: getErrText(error, t('common.speech.error tip')) + }); + } + setIsSpeaking(false); + }; + + mediaRecorder.current.start(); + + setIsSpeaking(true); + } catch (error) {} + }; + + const stopSpeak = () => { + if (mediaRecorder.current) { + mediaRecorder.current?.stop(); + } + }; + + return { + startSpeak, + stopSpeak, + isSpeaking + }; +};