perf: stream response (#2813)

* perf: stream response

* fold code

projects/app/src/web/common/api/fetch.ts

@@ -22,6 +22,16 @@ export type StreamResponseType = {
   responseText: string;
   [DispatchNodeResponseKeyEnum.nodeResponse]: ChatHistoryItemResType[];
 };
+type ResponseQueueItemType =
+  | { event: SseResponseEventEnum.fastAnswer | SseResponseEventEnum.answer; text: string }
+  | { event: SseResponseEventEnum.interactive; [key: string]: any }
+  | {
+      event:
+        | SseResponseEventEnum.toolCall
+        | SseResponseEventEnum.toolParams
+        | SseResponseEventEnum.toolResponse;
+      [key: string]: any;
+    };
 class FatalError extends Error {}
 
 export const streamFetch = ({
@@ -31,22 +41,14 @@ export const streamFetch = ({
   abortCtrl
 }: StreamFetchProps) =>
   new Promise<StreamResponseType>(async (resolve, reject) => {
+    // First res
     const timeoutId = setTimeout(() => {
       abortCtrl.abort('Time out');
     }, 60000);
 
     // response data
     let responseText = '';
-    let responseQueue: (
-      | { event: SseResponseEventEnum.fastAnswer | SseResponseEventEnum.answer; text: string }
-      | {
-          event:
-            | SseResponseEventEnum.toolCall
-            | SseResponseEventEnum.toolParams
-            | SseResponseEventEnum.toolResponse;
-          [key: string]: any;
-        }
-    )[] = [];
+    let responseQueue: ResponseQueueItemType[] = [];
     let errMsg: string | undefined;
     let responseData: ChatHistoryItemResType[] = [];
     let finished = false;
@@ -84,7 +86,7 @@ export const streamFetch = ({
       }
 
       if (responseQueue.length > 0) {
-        const fetchCount = Math.max(1, Math.round(responseQueue.length / 20));
+        const fetchCount = Math.max(1, Math.round(responseQueue.length / 30));
         for (let i = 0; i < fetchCount; i++) {
           const item = responseQueue[i];
           onMessage(item);
@@ -100,13 +102,20 @@ export const streamFetch = ({
         return finish();
       }
 
-      document.hidden
-        ? setTimeout(animateResponseText, 16)
-        : requestAnimationFrame(animateResponseText);
+      requestAnimationFrame(animateResponseText);
     }
     // start animation
     animateResponseText();
 
+    const pushDataToQueue = (data: ResponseQueueItemType) => {
+      // If the document is hidden, the data is directly sent to the front end
+      responseQueue.push(data);
+
+      if (document.hidden) {
+        animateResponseText();
+      }
+    };
+
     try {
       // auto complete variables
       const variables = data?.variables || {};
@@ -171,14 +180,14 @@ export const streamFetch = ({
           if (event === SseResponseEventEnum.answer) {
             const text = parseJson.choices?.[0]?.delta?.content || '';
             for (const item of text) {
-              responseQueue.push({
+              pushDataToQueue({
                 event,
                 text: item
               });
             }
           } else if (event === SseResponseEventEnum.fastAnswer) {
             const text = parseJson.choices?.[0]?.delta?.content || '';
-            responseQueue.push({
+            pushDataToQueue({
               event,
               text
             });
@@ -187,7 +196,7 @@ export const streamFetch = ({
             event === SseResponseEventEnum.toolParams ||
             event === SseResponseEventEnum.toolResponse
           ) {
-            responseQueue.push({
+            pushDataToQueue({
               event,
               ...parseJson
             });
@@ -204,7 +213,7 @@ export const streamFetch = ({
               variables: parseJson
             });
           } else if (event === SseResponseEventEnum.interactive) {
-            responseQueue.push({
+            pushDataToQueue({
               event,
               ...parseJson
             });

projects/app/src/web/core/chat/context/chatContext.tsx

@@ -200,7 +200,7 @@ const ChatContextProvider = ({
         );
       } else {
         // Chat history not exists
-        loadHistories();
+        loadHistories(true);
       }
     },
     [histories, loadHistories, setHistories]