4.8.5 test (#1805)

* perf: revert tip

* feat: create copy app

* perf: file stream read

* perf: read directory over 100 files

* perf: index

* fix: team chat api error

* lock

* fix: i18n file

packages/service/common/file/gridfs/utils.ts

@@ -1,3 +1,6 @@
+import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
+import { PassThrough } from 'stream';
+
 export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
   return new Promise<Buffer>((resolve, reject) => {
     let tmpBuffer: Buffer = Buffer.from([]);
@@ -13,3 +16,38 @@ export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
     });
   });
 };
+
+export const stream2Encoding = async (stream: NodeJS.ReadableStream) => {
+  const start = Date.now();
+  const copyStream = stream.pipe(new PassThrough());
+
+  /* get encoding */
+  const buffer = await (() => {
+    return new Promise<Buffer>((resolve, reject) => {
+      let tmpBuffer: Buffer = Buffer.from([]);
+
+      stream.on('data', (chunk) => {
+        if (tmpBuffer.length < 200) {
+          tmpBuffer = Buffer.concat([tmpBuffer, chunk]);
+
+          if (tmpBuffer.length >= 200) {
+            resolve(tmpBuffer);
+          }
+        }
+      });
+      stream.on('end', () => {
+        resolve(tmpBuffer);
+      });
+      stream.on('error', (err) => {
+        reject(err);
+      });
+    });
+  })();
+
+  const enc = detectFileEncoding(buffer);
+  console.log('Get encoding time', Date.now() - start, enc);
+  return {
+    encoding: enc,
+    stream: copyStream
+  };
+};