fix: chat variable update (#3156)

* perf: file encoding

* fix: chat variable update

packages/service/common/file/gridfs/controller.ts

@@ -36,6 +36,7 @@ export async function uploadFile({
   path,
   filename,
   contentType,
+  encoding,
   metadata = {}
 }: {
   bucketName: `${BucketNameEnum}`;
@@ -44,6 +45,7 @@ export async function uploadFile({
   path: string;
   filename: string;
   contentType?: string;
+  encoding: string;
   metadata?: Record<string, any>;
 }) {
   if (!path) return Promise.reject(`filePath is empty`);
@@ -52,7 +54,7 @@ export async function uploadFile({
   const stats = await fsp.stat(path);
   if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
 
-  const { stream: readStream, encoding } = await stream2Encoding(fs.createReadStream(path));
+  const readStream = fs.createReadStream(path);
 
   // Add default metadata
   metadata.teamId = teamId;

packages/service/common/file/read/utils.ts

@@ -14,6 +14,7 @@ import { addHours } from 'date-fns';
 export type readRawTextByLocalFileParams = {
   teamId: string;
   path: string;
+  encoding: string;
   metadata?: Record<string, any>;
 };
 export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => {
@@ -22,13 +23,12 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
   const extension = path?.split('.')?.pop()?.toLowerCase() || '';
 
   const buffer = fs.readFileSync(path);
-  const encoding = detectFileEncoding(buffer);
 
   const { rawText } = await readRawContentByFileBuffer({
     extension,
     isQAImport: false,
     teamId: params.teamId,
-    encoding,
+    encoding: params.encoding,
     buffer,
     metadata: params.metadata
   });
@@ -53,6 +53,7 @@ export const readRawContentByFileBuffer = async ({
   encoding: string;
   metadata?: Record<string, any>;
 }) => {
+  // Custom read file service
   const customReadfileUrl = process.env.CUSTOM_READ_FILE_URL;
   const customReadFileExtension = process.env.CUSTOM_READ_FILE_EXTENSION || '';
   const ocrParse = process.env.CUSTOM_READ_FILE_OCR || 'false';

packages/service/worker/readFile/extension/rawText.ts

@@ -18,9 +18,17 @@ const rawEncodingList = [
 
 // 加载源文件内容
 export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
-  const content = rawEncodingList.includes(encoding)
-    ? buffer.toString(encoding as BufferEncoding)
-    : iconv.decode(buffer, 'gbk');
+  const content = (() => {
+    try {
+      if (rawEncodingList.includes(encoding)) {
+        return buffer.toString(encoding as BufferEncoding);
+      }
+
+      return iconv.decode(buffer, encoding);
+    } catch (error) {
+      return buffer.toString('utf-8');
+    }
+  })();
 
   return {
     rawText: content