Fixed the duplicate data check problem, history filter and add tts stream (#477)

packages/global/core/ai/model.d.ts

@@ -29,4 +29,7 @@ export type AudioSpeechModelType = {
   model: string;
   name: string;
   price: number;
+  baseUrl?: string;
+  key?: string;
+  voices: { label: string; value: string; bufferId: string }[];
 };

packages/global/core/ai/model.ts

@@ -105,11 +105,14 @@ export const defaultAudioSpeechModels: AudioSpeechModelType[] = [
   {
     model: 'tts-1',
     name: 'OpenAI TTS1',
-    price: 0
+    price: 0,
-  },
+    voices: [
-  {
+      { label: 'Alloy', value: 'Alloy', bufferId: 'openai-Alloy' },
-    model: 'tts-1-hd',
+      { label: 'Echo', value: 'Echo', bufferId: 'openai-Echo' },
-    name: 'OpenAI TTS1',
+      { label: 'Fable', value: 'Fable', bufferId: 'openai-Fable' },
-    price: 0
+      { label: 'Onyx', value: 'Onyx', bufferId: 'openai-Onyx' },
+      { label: 'Nova', value: 'Nova', bufferId: 'openai-Nova' },
+      { label: 'Shimmer', value: 'Shimmer', bufferId: 'openai-Shimmer' }
+    ]
   }
 ];

packages/global/core/ai/speech/api.d.ts

@@ -1,8 +0,0 @@
-import { Text2SpeechVoiceEnum } from './constant';
-
-export type Text2SpeechProps = {
-  model?: string;
-  voice?: `${Text2SpeechVoiceEnum}`;
-  input: string;
-  speed?: number;
-};

packages/global/core/ai/speech/constant.ts

@@ -1,17 +0,0 @@
-export enum Text2SpeechVoiceEnum {
-  alloy = 'alloy',
-  echo = 'echo',
-  fable = 'fable',
-  onyx = 'onyx',
-  nova = 'nova',
-  shimmer = 'shimmer'
-}
-export const openaiTTSList = [
-  Text2SpeechVoiceEnum.alloy,
-  Text2SpeechVoiceEnum.echo,
-  Text2SpeechVoiceEnum.fable,
-  Text2SpeechVoiceEnum.onyx,
-  Text2SpeechVoiceEnum.nova,
-  Text2SpeechVoiceEnum.shimmer
-];
-export const openaiTTSModel = 'tts-1';

packages/global/core/app/type.d.ts

@@ -1,7 +1,6 @@
 import { ModuleItemType } from '../module/type';
 import { AppTypeEnum } from './constants';
 import { PermissionTypeEnum } from '../../support/permission/constant';
-import { Text2SpeechVoiceEnum } from '../ai/speech/constant';
 
 export interface AppSchema {
   _id: string;

packages/global/core/chat/type.d.ts

@@ -39,7 +39,6 @@ export type ChatItemSchema = {
   userFeedback?: string;
   adminFeedback?: AdminFbkType;
   [TaskResponseKeyEnum.responseData]?: ChatHistoryItemResType[];
-  tts?: Buffer;
 };
 
 export type AdminFbkType = {
@@ -62,7 +61,7 @@ export type ChatItemType = {
 export type ChatSiteItemType = {
   status: 'loading' | 'running' | 'finish';
   moduleName?: string;
-  ttsBuffer?: Buffer;
+  ttsBuffer?: Uint8Array;
 } & ChatItemType;
 
 export type HistoryItemType = {

packages/service/common/buffer/tts/schema.ts

@@ -0,0 +1,35 @@
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import { TTSBufferSchemaType } from './type.d';
+
+export const collectionName = 'ttsbuffers';
+
+const TTSBufferSchema = new Schema({
+  bufferId: {
+    type: String,
+    required: true
+  },
+  text: {
+    type: String,
+    required: true
+  },
+  buffer: {
+    type: Buffer,
+    required: true
+  },
+  createTime: {
+    type: Date,
+    default: () => new Date()
+  }
+});
+
+try {
+  TTSBufferSchema.index({ bufferId: 1 });
+  //  24 hour
+  TTSBufferSchema.index({ createTime: 1 }, { expireAfterSeconds: 24 * 60 * 60 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoTTSBuffer: Model<TTSBufferSchemaType> =
+  models[collectionName] || model(collectionName, TTSBufferSchema);

packages/service/common/buffer/tts/type.d.ts

@@ -0,0 +1,5 @@
+export type TTSBufferSchemaType = {
+  bufferId: string;
+  text: string;
+  buffer: Buffer;
+};

packages/service/core/ai/audio/speech.ts

@@ -1,26 +1,49 @@
-import { Text2SpeechProps } from '@fastgpt/global/core/ai/speech/api';
+import type { NextApiResponse } from 'next';
 import { getAIApi } from '../config';
 import { defaultAudioSpeechModels } from '../../../../global/core/ai/model';
-import { Text2SpeechVoiceEnum } from '@fastgpt/global/core/ai/speech/constant';
+import { UserModelSchema } from '@fastgpt/global/support/user/type';
 
 export async function text2Speech({
-  model = defaultAudioSpeechModels[0].model,
+  res,
-  voice = Text2SpeechVoiceEnum.alloy,
+  onSuccess,
+  onError,
   input,
-  speed = 1
+  model = defaultAudioSpeechModels[0].model,
-}: Text2SpeechProps) {
+  voice,
-  const ai = getAIApi();
+  speed = 1,
-  const mp3 = await ai.audio.speech.create({
+  props
+}: {
+  res: NextApiResponse;
+  onSuccess: (e: { model: string; buffer: Buffer }) => void;
+  onError: (e: any) => void;
+  input: string;
+  model: string;
+  voice: string;
+  speed?: number;
+  props?: UserModelSchema['openaiAccount'];
+}) {
+  const ai = getAIApi(props);
+  const response = await ai.audio.speech.create({
     model,
+    // @ts-ignore
     voice,
     input,
     response_format: 'mp3',
     speed
   });
-  const buffer = Buffer.from(await mp3.arrayBuffer());
+
-  return {
+  const readableStream = response.body as unknown as NodeJS.ReadableStream;
-    model,
+  readableStream.pipe(res);
-    voice,
+
-    tts: buffer
+  let bufferStore = Buffer.from([]);
-  };
+
+  readableStream.on('data', (chunk) => {
+    bufferStore = Buffer.concat([bufferStore, chunk]);
+  });
+  readableStream.on('end', () => {
+    onSuccess({ model, buffer: bufferStore });
+  });
+  readableStream.on('error', (e) => {
+    onError(e);
+  });
 }

packages/service/core/chat/chatItemSchema.ts

@@ -68,9 +68,6 @@ const ChatItemSchema = new Schema({
   [TaskResponseKeyEnum.responseData]: {
     type: Array,
     default: []
-  },
-  tts: {
-    type: Buffer
   }
 });
 

projects/app/data/config.json

@@ -102,12 +102,17 @@
     {
       "model": "tts-1",
       "name": "OpenAI TTS1",
-      "price": 0
+      "price": 0,
-    },
+      "baseUrl": "https://api.openai.com/v1",
-    {
+      "key": "",
-      "model": "tts-1-hd",
+      "voices": [
-      "name": "OpenAI TTS1HD",
+        { "label": "Alloy", "value": "alloy", "bufferId": "openai-Alloy" },
-      "price": 0
+        { "label": "Echo", "value": "echo", "bufferId": "openai-Echo" },
+        { "label": "Fable", "value": "fable", "bufferId": "openai-Fable" },
+        { "label": "Onyx", "value": "onyx", "bufferId": "openai-Onyx" },
+        { "label": "Nova", "value": "nova", "bufferId": "openai-Nova" },
+        { "label": "Shimmer", "value": "shimmer", "bufferId": "openai-Shimmer" }
+      ]
     }
   ]
 }

projects/app/public/locales/en/common.json

@@ -313,10 +313,12 @@
         "Course": "Document",
         "Delete": "Delete",
         "Index": "Index({{amount}})"
-      }
+      },
+      "input is empty": "The data content cannot be empty"
     },
     "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
     "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!",
+    "import csv tip": "Ensure that the CSV is in UTF-8 format; otherwise, garbled characters will be displayed",
     "recall": {
       "rerank": "Rerank"
     },

projects/app/public/locales/zh/common.json

@@ -313,10 +313,12 @@
         "Course": "说明文档",
         "Delete": "删除数据",
         "Index": "数据索引({{amount}})"
-      }
+      },
+      "input is empty": "数据内容不能为空 "
     },
     "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
     "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！",
+    "import csv tip": "请确保CSV为UTF-8格式，否则会乱码",
     "recall": {
       "rerank": "结果重排"
     },

projects/app/src/components/ChatBox/index.tsx

@@ -1221,18 +1221,19 @@ function ChatController({
               name={'voice'}
               _hover={{ color: '#E74694' }}
               onClick={async () => {
-                const buffer = await playAudio({
+                const response = await playAudio({
                   buffer: chat.ttsBuffer,
                   chatItemId: chat.dataId,
                   text: chat.value
                 });
-                if (!setChatHistory) return;
+
+                if (!setChatHistory || !response.buffer) return;
                 setChatHistory((state) =>
                   state.map((item) =>
                     item.dataId === chat.dataId
                       ? {
                           ...item,
-                          ttsBuffer: buffer
+                          ttsBuffer: response.buffer
                         }
                       : item
                   )

projects/app/src/global/common/api/systemRes.d.ts

@@ -2,7 +2,8 @@ import type {
   ChatModelItemType,
   FunctionModelItemType,
   LLMModelItemType,
-  VectorModelItemType
+  VectorModelItemType,
+  AudioSpeechModels
 } from '@fastgpt/global/core/ai/model.d';
 
 import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
@@ -12,8 +13,8 @@ export type InitDateResponse = {
   qaModels: LLMModelItemType[];
   cqModels: FunctionModelItemType[];
   extractModels: FunctionModelItemType[];
-  qgModels: LLMModelItemType[];
   vectorModels: VectorModelItemType[];
+  audioSpeechModels: AudioSpeechModels[];
   feConfigs: FeConfigsType;
   priceMd: string;
   systemVersion: string;

projects/app/src/global/core/chat/api.d.ts

@@ -1,7 +1,6 @@
 import type { AppTTSConfigType } from '@/types/app';
 
 export type GetChatSpeechProps = {
-  chatItemId?: string;
   ttsConfig: AppTTSConfigType;
   input: string;
 };

projects/app/src/pages/api/core/chat/item/getSpeech.ts

@@ -1,12 +1,13 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
 import { GetChatSpeechProps } from '@/global/core/chat/api.d';
 import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
 import { pushAudioSpeechBill } from '@/service/support/wallet/bill/push';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { authType2BillSource } from '@/service/support/wallet/bill/utils';
+import { getAudioSpeechModel } from '@/service/core/ai/model';
+import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema';
 
 /* 
 1. get tts from chatItem store
@@ -18,50 +19,66 @@ import { authType2BillSource } from '@/service/support/wallet/bill/utils';
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { chatItemId, ttsConfig, input } = req.body as GetChatSpeechProps;
+    const { ttsConfig, input } = req.body as GetChatSpeechProps;
+
+    if (!ttsConfig.model || !ttsConfig.voice) {
+      throw new Error('model or voice not found');
+    }
 
     const { teamId, tmbId, authType } = await authCert({ req, authToken: true });
 
-    const chatItem = await (async () => {
+    const ttsModel = getAudioSpeechModel(ttsConfig.model);
-      if (!chatItemId) return null;
+    const voiceData = ttsModel.voices.find((item) => item.value === ttsConfig.voice);
-      return await MongoChatItem.findOne(
-        {
-          dataId: chatItemId
-        },
-        'tts'
-      );
-    })();
 
-    if (chatItem?.tts) {
+    if (!voiceData) {
-      return jsonRes(res, {
+      throw new Error('voice not found');
-        data: chatItem.tts
-      });
     }
 
-    const { tts, model } = await text2Speech({
+    const ttsBuffer = await MongoTTSBuffer.findOne(
+      {
+        bufferId: voiceData.bufferId,
+        text: input
+      },
+      'buffer'
+    );
+
+    if (ttsBuffer?.buffer) {
+      return res.end(new Uint8Array(ttsBuffer.buffer.buffer));
+    }
+
+    await text2Speech({
+      res,
+      input,
       model: ttsConfig.model,
       voice: ttsConfig.voice,
-      input
+      props: {
-    });
+        // temp code
+        baseUrl: ttsModel.baseUrl || '',
+        key: ttsModel.key || ''
+      },
+      onSuccess: async ({ model, buffer }) => {
+        try {
+          pushAudioSpeechBill({
+            model: model,
+            textLength: input.length,
+            tmbId,
+            teamId,
+            source: authType2BillSource({ authType })
+          });
 
-    (async () => {
+          await MongoTTSBuffer.create({
-      if (!chatItem) return;
+            bufferId: voiceData.bufferId,
-      try {
+            text: input,
-        chatItem.tts = tts;
+            buffer
-        await chatItem.save();
+          });
-      } catch (error) {}
+        } catch (error) {}
-    })();
+      },
-
+      onError: (err) => {
-    jsonRes(res, {
+        jsonRes(res, {
-      data: tts
+          code: 500,
-    });
+          error: err
-
+        });
-    pushAudioSpeechBill({
+      }
-      model: model,
-      textLength: input.length,
-      tmbId,
-      teamId,
-      source: authType2BillSource({ authType })
     });
   } catch (err) {
     jsonRes(res, {

projects/app/src/pages/api/system/getInitData.ts

@@ -34,8 +34,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       qaModels: global.qaModels,
       cqModels: global.cqModels,
       extractModels: global.extractModels,
-      qgModels: global.qgModels,
       vectorModels: global.vectorModels,
+      audioSpeechModels: global.audioSpeechModels.map((item) => ({
+        ...item,
+        baseUrl: undefined,
+        key: undefined
+      })),
       priceMd: global.priceMd,
       systemVersion: global.systemVersion || '0.0.0'
     }

projects/app/src/pages/api/v1/chat/completions.ts

@@ -86,6 +86,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     if (chatMessages[chatMessages.length - 1].obj !== ChatRoleEnum.Human) {
       chatMessages.pop();
     }
+
     // user question
     const question = chatMessages.pop();
     if (!question) {
@@ -173,15 +174,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     const isAppOwner = !shareId && String(user.team.tmbId) === String(app.tmbId);
 
     /* format prompts */
-    const prompts = history.concat(gptMessage2ChatType(messages));
+    const concatHistory = history.concat(chatMessages);
-
-    // set sse response headers
-    if (stream) {
-      res.setHeader('Content-Type', 'text/event-stream;charset=utf-8');
-      res.setHeader('Access-Control-Allow-Origin', '*');
-      res.setHeader('X-Accel-Buffering', 'no');
-      res.setHeader('Cache-Control', 'no-cache, no-transform');
-    }
 
     /* start flow controller */
     const { responseData, answerText } = await dispatchModules({
@@ -193,7 +186,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       tmbId: user.team.tmbId,
       variables,
       params: {
-        history: prompts,
+        history: concatHistory,
         userChatInput: question.value
       },
       stream,

projects/app/src/pages/app/detail/components/TTSSelect.tsx

@@ -6,10 +6,10 @@ import React, { useCallback, useMemo } from 'react';
 import { useTranslation } from 'next-i18next';
 import MySelect from '@/components/Select';
 import { TTSTypeEnum } from '@/constants/app';
-import { Text2SpeechVoiceEnum, openaiTTSModel } from '@fastgpt/global/core/ai/speech/constant';
 import { AppTTSConfigType } from '@/types/app';
 import { useAudioPlay } from '@/web/common/utils/voice';
 import { useLoading } from '@/web/common/hooks/useLoading';
+import { audioSpeechModels } from '@/web/common/system/staticData';
 
 const TTSSelect = ({
   value,
@@ -37,10 +37,16 @@ const TTSSelect = ({
       if (e === TTSTypeEnum.none || e === TTSTypeEnum.web) {
         onChange({ type: e as `${TTSTypeEnum}` });
       } else {
+        const audioModel = audioSpeechModels.find((item) =>
+          item.voices.find((voice) => voice.value === e)
+        );
+        if (!audioModel) {
+          return;
+        }
         onChange({
           type: TTSTypeEnum.model,
-          model: openaiTTSModel,
+          model: audioModel.model,
-          voice: e as `${Text2SpeechVoiceEnum}`,
+          voice: e,
           speed: 1
         });
       }
@@ -77,12 +83,7 @@ const TTSSelect = ({
         list={[
           { label: t('core.app.tts.Close'), value: TTSTypeEnum.none },
           { label: t('core.app.tts.Web'), value: TTSTypeEnum.web },
-          { label: 'Alloy', value: Text2SpeechVoiceEnum.alloy },
+          ...audioSpeechModels.map((item) => item.voices).flat()
-          { label: 'Echo', value: Text2SpeechVoiceEnum.echo },
-          { label: 'Fable', value: Text2SpeechVoiceEnum.fable },
-          { label: 'Onyx', value: Text2SpeechVoiceEnum.onyx },
-          { label: 'Nova', value: Text2SpeechVoiceEnum.nova },
-          { label: 'Shimmer', value: Text2SpeechVoiceEnum.shimmer }
         ]}
         onchange={onclickChange}
       />

projects/app/src/pages/dataset/detail/components/Import/Csv.tsx

@@ -2,11 +2,13 @@ import React from 'react';
 import { Box, Flex, Button } from '@chakra-ui/react';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
+import { useTranslation } from 'next-i18next';
 
 const fileExtension = '.csv';
 const csvTemplate = `index,content\n"被索引的内容","对应的答案。CSV 中请注意内容不能包含双引号，双引号是列分割符号"\n"什么是 laf","laf 是一个云函数开发平台……",""\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
 
 const CsvImport = () => {
+  const { t } = useTranslation();
   const { successChunks, totalChunks, isUnselectedFile, onclickUpload, uploading } =
     useImportStore();
 
@@ -24,6 +26,7 @@ const CsvImport = () => {
           value: csvTemplate,
           type: 'text/csv'
         }}
+        tip={t('dataset.import csv tip')}
       >
         <Flex mt={3}>
           <Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -55,6 +55,7 @@ export interface Props extends BoxProps {
   };
   showUrlFetch?: boolean;
   showCreateFile?: boolean;
+  tip?: string;
 }
 
 const FileSelect = ({
@@ -65,6 +66,7 @@ const FileSelect = ({
   fileTemplate,
   showUrlFetch = true,
   showCreateFile = true,
+  tip,
   ...props
 }: Props) => {
   const { datasetDetail } = useDatasetStore();
@@ -423,6 +425,7 @@ const FileSelect = ({
           {t('file.Click to download file template', { name: fileTemplate.filename })}
         </Box>
       )}
+      {!!tip && <Box color={'myGray.500'}>{tip}</Box>}
       {selectingText !== undefined && (
         <FileSelectLoading loading text={selectingText} fixed={false} />
       )}

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -403,12 +403,14 @@ export const SelectorContainer = ({
   showUrlFetch,
   showCreateFile,
   fileTemplate,
+  tip,
   children
 }: {
   fileExtension: string;
   showUrlFetch?: boolean;
   showCreateFile?: boolean;
   fileTemplate?: FileSelectProps['fileTemplate'];
+  tip?: string;
   children: React.ReactNode;
 }) => {
   const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen } = useImportStore();
@@ -433,6 +435,7 @@ export const SelectorContainer = ({
         showUrlFetch={showUrlFetch}
         showCreateFile={showCreateFile}
         fileTemplate={fileTemplate}
+        tip={tip}
         py={isUnselectedFile ? '100px' : 5}
       />
       {!isUnselectedFile && (

projects/app/src/pages/dataset/detail/components/InputDataModal.tsx

@@ -117,10 +117,8 @@ const InputDataModal = ({
   const { mutate: sureImportData, isLoading: isImporting } = useRequest({
     mutationFn: async (e: InputDataType) => {
       if (!e.q) {
-        return toast({
+        setCurrentTab(TabEnum.content);
-          title: '匹配的知识点不能为空',
+        return Promise.reject(t('dataset.data.input is empty'));
-          status: 'warning'
-        });
       }
       if (countPromptTokens(e.q) >= maxToken) {
         return toast({

projects/app/src/service/core/dataset/data/utils.ts

@@ -1,5 +1,4 @@
-import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { PgClient } from '@fastgpt/service/common/pg';
 
 /**
  * Same value judgment
@@ -13,14 +12,13 @@ export async function hasSameValue({
   q: string;
   a?: string;
 }) {
-  const { rows: existsRows } = await PgClient.query(`
+  const count = await MongoDatasetData.countDocuments({
-  SELECT COUNT(*) > 0 AS exists
+    q,
-  FROM  ${PgDatasetTableName} 
+    a,
-  WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND collection_id='${collectionId}'
+    collectionId
-`);
+  });
-  const exists = existsRows[0]?.exists || false;
 
-  if (exists) {
+  if (count > 0) {
     return Promise.reject('已经存在完全一致的数据');
   }
 }

projects/app/src/service/moduleDispatch/index.ts

@@ -50,6 +50,14 @@ export async function dispatchModules({
   stream?: boolean;
   detail?: boolean;
 }) {
+  // set sse response headers
+  if (stream) {
+    res.setHeader('Content-Type', 'text/event-stream;charset=utf-8');
+    res.setHeader('Access-Control-Allow-Origin', '*');
+    res.setHeader('X-Accel-Buffering', 'no');
+    res.setHeader('Cache-Control', 'no-cache, no-transform');
+  }
+
   variables = {
     ...getSystemVariable({ timezone: user.timezone }),
     ...variables
@@ -167,6 +175,7 @@ export async function dispatchModules({
       user,
       teamId,
       tmbId,
+      chatId,
       inputs: params
     };
 

projects/app/src/types/app.d.ts

@@ -15,7 +15,6 @@ import type { FlowModuleTemplateType, ModuleItemType } from '@fastgpt/global/cor
 import type { ChatSchema } from '@fastgpt/global/core/chat/type';
 import type { AppSchema } from '@fastgpt/global/core/app/type';
 import { ChatModelType } from '@/constants/model';
-import { Text2SpeechVoiceEnum } from '@fastgpt/global/core/ai/speech/constant';
 
 export interface ShareAppItem {
   _id: string;
@@ -40,7 +39,7 @@ export type VariableItemType = {
 export type AppTTSConfigType = {
   type: 'none' | 'web' | 'model';
   model?: string;
-  voice?: `${Text2SpeechVoiceEnum}`;
+  voice?: string;
   speed?: number;
 };
 

projects/app/src/web/common/api/fetch.ts

@@ -37,7 +37,7 @@ export const streamFetch = ({
         })
       });
 
-      if (!response?.body) {
+      if (!response?.body || !response?.ok) {
         throw new Error('Request Error');
       }
 

projects/app/src/web/common/system/staticData.ts

@@ -8,7 +8,8 @@ import {
   defaultCQModels,
   defaultExtractModels,
   defaultQGModels,
-  defaultVectorModels
+  defaultVectorModels,
+  defaultAudioSpeechModels
 } from '@fastgpt/global/core/ai/model';
 
 export let feConfigs: FeConfigsType = {};
@@ -21,6 +22,7 @@ export let qaModelList = defaultQAModels;
 export let cqModelList = defaultCQModels;
 export let extractModelList = defaultExtractModels;
 export let qgModelList = defaultQGModels;
+export let audioSpeechModels = defaultAudioSpeechModels;
 
 let retryTimes = 3;
 
@@ -32,10 +34,11 @@ export const clientInitData = async (): Promise<InitDateResponse> => {
     qaModelList = res.qaModels || [];
     cqModelList = res.cqModels || [];
     extractModelList = res.extractModels || [];
-    qgModelList = res.qgModels || [];
 
     vectorModelList = res.vectorModels || [];
 
+    audioSpeechModels = res.audioSpeechModels || [];
+
     feConfigs = res.feConfigs;
     priceMd = res.priceMd;
     systemVersion = res.systemVersion;

projects/app/src/web/common/utils/voice.ts

@@ -1,7 +1,6 @@
 import { useState, useCallback, useEffect, useMemo } from 'react';
 import { useToast } from '@/web/common/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
-import { getChatItemSpeech } from '@/web/core/chat/api';
 import { AppTTSConfigType } from '@/types/app';
 import { TTSTypeEnum } from '@/constants/app';
 import { useTranslation } from 'next-i18next';
@@ -31,55 +30,85 @@ export const useAudioPlay = (props?: { ttsConfig?: AppTTSConfigType }) => {
   }: {
     text: string;
     chatItemId?: string;
-    buffer?: Buffer;
+    buffer?: Uint8Array;
-  }) => {
+  }) =>
-    text = text.replace(/\\n/g, '\n');
+    new Promise<{ buffer?: Uint8Array }>(async (resolve, reject) => {
-    try {
+      text = text.replace(/\\n/g, '\n');
-      // tts play
+      try {
-      if (audio && ttsConfig && ttsConfig?.type === TTSTypeEnum.model) {
+        // tts play
-        setAudioLoading(true);
+        if (audio && ttsConfig && ttsConfig?.type === TTSTypeEnum.model) {
-        const { data } = buffer
+          setAudioLoading(true);
-          ? { data: buffer }
-          : await getChatItemSpeech({ chatItemId, ttsConfig, input: text });
 
-        const arrayBuffer = new Uint8Array(data).buffer;
+          /* buffer tts */
+          if (buffer) {
+            playAudioBuffer({ audio, buffer });
+            setAudioLoading(false);
+            return resolve({ buffer });
+          }
 
-        const audioUrl = URL.createObjectURL(new Blob([arrayBuffer], { type: 'audio/mp3' }));
+          /* request tts */
+          const response = await fetch('/api/core/chat/item/getSpeech', {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+              chatItemId,
+              ttsConfig,
+              input: text
+            })
+          });
+          setAudioLoading(false);
 
-        audio.src = audioUrl;
+          if (!response.body || !response.ok) {
-        audio.play();
+            const data = await response.json();
-        setAudioLoading(false);
+            toast({
+              status: 'error',
+              title: getErrText(data, t('core.chat.Audio Speech Error'))
+            });
+            return reject(data);
+          }
 
-        return data;
+          const audioBuffer = await readAudioStream({
-      } else {
+            audio,
-        // window speech
+            stream: response.body,
-        window.speechSynthesis?.cancel();
+            contentType: 'audio/mpeg'
-        const msg = new SpeechSynthesisUtterance(text);
+          });
-        const voices = window.speechSynthesis?.getVoices?.() || []; // 获取语言包
+
-        const voice = voices.find((item) => {
+          resolve({
-          return item.lang === 'zh-CN';
+            buffer: audioBuffer
-        });
+          });
-        if (voice) {
+        } else {
-          msg.onstart = () => {
+          // window speech
-            setAudioPlaying(true);
+          window.speechSynthesis?.cancel();
-          };
+          const msg = new SpeechSynthesisUtterance(text);
-          msg.onend = () => {
+          const voices = window.speechSynthesis?.getVoices?.() || []; // 获取语言包
-            setAudioPlaying(false);
+          const voice = voices.find((item) => {
-            msg.onstart = null;
+            return item.lang === 'zh-CN';
-            msg.onend = null;
+          });
-          };
+          if (voice) {
-          msg.voice = voice;
+            msg.onstart = () => {
-          window.speechSynthesis?.speak(msg);
+              setAudioPlaying(true);
+            };
+            msg.onend = () => {
+              setAudioPlaying(false);
+              msg.onstart = null;
+              msg.onend = null;
+            };
+            msg.voice = voice;
+            window.speechSynthesis?.speak(msg);
+          }
+          resolve({});
         }
+      } catch (error) {
+        toast({
+          status: 'error',
+          title: getErrText(error, t('core.chat.Audio Speech Error'))
+        });
+        reject(error);
       }
-    } catch (error) {
+      setAudioLoading(false);
-      toast({
+    });
-        status: 'error',
-        title: getErrText(error, t('core.chat.Audio Speech Error'))
-      });
-    }
-    setAudioLoading(false);
-  };
 
   const cancelAudio = useCallback(() => {
     if (audio) {
@@ -107,6 +136,9 @@ export const useAudioPlay = (props?: { ttsConfig?: AppTTSConfigType }) => {
       audio.onerror = () => {
         setAudioPlaying(false);
       };
+      audio.oncancel = () => {
+        setAudioPlaying(false);
+      };
     }
     const listen = () => {
       cancelAudio();
@@ -137,3 +169,67 @@ export const useAudioPlay = (props?: { ttsConfig?: AppTTSConfigType }) => {
     cancelAudio
   };
 };
+
+export function readAudioStream({
+  audio,
+  stream,
+  contentType = 'audio/mpeg'
+}: {
+  audio: HTMLAudioElement;
+  stream: ReadableStream<Uint8Array>;
+  contentType?: string;
+}): Promise<Uint8Array> {
+  // Create media source and play audio
+  const ms = new MediaSource();
+  const url = URL.createObjectURL(ms);
+  audio.src = url;
+  audio.play();
+
+  let u8Arr: Uint8Array = new Uint8Array();
+  return new Promise<Uint8Array>(async (resolve, reject) => {
+    // Async to read data from ms
+    await new Promise((resolve) => {
+      ms.onsourceopen = resolve;
+    });
+
+    const sourceBuffer = ms.addSourceBuffer(contentType);
+
+    const reader = stream.getReader();
+
+    // read stream
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          resolve(u8Arr);
+          if (sourceBuffer.updating) {
+            await new Promise((resolve) => (sourceBuffer.onupdateend = resolve));
+          }
+          ms.endOfStream();
+          return;
+        }
+
+        u8Arr = new Uint8Array([...u8Arr, ...value]);
+
+        await new Promise((resolve) => {
+          sourceBuffer.onupdateend = resolve;
+          sourceBuffer.appendBuffer(value.buffer);
+        });
+      }
+    } catch (error) {
+      reject(error);
+    }
+  });
+}
+export function playAudioBuffer({
+  audio,
+  buffer
+}: {
+  audio: HTMLAudioElement;
+  buffer: Uint8Array;
+}) {
+  const audioUrl = URL.createObjectURL(new Blob([buffer], { type: 'audio/mpeg' }));
+
+  audio.src = audioUrl;
+  audio.play();
+}

projects/app/src/web/core/chat/api.ts

@@ -43,7 +43,3 @@ export const userUpdateChatFeedback = (data: { chatItemId: string; userFeedback?
 
 export const adminUpdateChatFeedback = (data: AdminUpdateFeedbackParams) =>
   POST('/core/chat/feedback/adminUpdate', data);
-
-/* ------------- function ------------- */
-export const getChatItemSpeech = (data: GetChatSpeechProps) =>
-  POST<{ data: Buffer }>('/core/chat/item/getSpeech', data);