Feat: pptx and xlsx loader (#1118)

* perf: plan tip

* perf: upload size controller

* feat: add image ttl index

* feat: new upload file ux

* remove file

* feat: support read pptx

* feat: support xlsx

* fix: rerank docker flie

projects/app/src/pages/api/common/file/previewContent.ts

@@ -0,0 +1,41 @@
+/* 
+    Read db file content and response 3000 words
+*/
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { readFileContent } from '@fastgpt/service/common/file/gridfs/controller';
+import { authFile } from '@fastgpt/service/support/permission/auth/file';
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+    const { fileId, csvFormat } = req.body as { fileId: string; csvFormat?: boolean };
+
+    if (!fileId) {
+      throw new Error('fileId is empty');
+    }
+
+    const { teamId } = await authFile({ req, authToken: true, fileId });
+
+    const { rawText } = await readFileContent({
+      teamId,
+      bucketName: BucketNameEnum.dataset,
+      fileId,
+      csvFormat
+    });
+
+    jsonRes(res, {
+      data: {
+        previewContent: rawText.slice(0, 3000),
+        totalLength: rawText.length
+      }
+    });
+  } catch (error) {
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}

projects/app/src/pages/api/common/file/read.ts

@@ -2,9 +2,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authFileToken } from '@fastgpt/service/support/permission/controller';
-import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
+import {
+  getDownloadStream,
+  getFileById,
+  readFileEncode
+} from '@fastgpt/service/common/file/gridfs/controller';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
-import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -18,8 +21,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       throw new Error('fileId is empty');
     }
 
-    const [file, encodeStream] = await Promise.all([
+    const [file, encoding, fileStream] = await Promise.all([
       getFileById({ bucketName, fileId }),
+      readFileEncode({ bucketName, fileId }),
       getDownloadStream({ bucketName, fileId })
     ]);
 
@@ -27,24 +31,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       return Promise.reject(CommonErrEnum.fileNotFound);
     }
 
-    // get encoding
-    let buffers: Buffer = Buffer.from([]);
-    for await (const chunk of encodeStream) {
-      buffers = Buffer.concat([buffers, chunk]);
-      if (buffers.length > 10) {
-        encodeStream.abort();
-        break;
-      }
-    }
-
-    const encoding = detectFileEncoding(buffers);
-
     res.setHeader('Content-Type', `${file.contentType}; charset=${encoding}`);
     res.setHeader('Cache-Control', 'public, max-age=3600');
     res.setHeader('Content-Disposition', `inline; filename="${encodeURIComponent(file.filename)}"`);
 
-    const fileStream = await getDownloadStream({ bucketName, fileId });
-
     fileStream.pipe(res);
 
     fileStream.on('error', () => {

projects/app/src/pages/api/common/file/upload.ts

@@ -4,24 +4,22 @@ import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
 import { getUploadModel } from '@fastgpt/service/common/file/multer';
-
-/**
- * Creates the multer uploader
- */
-const upload = getUploadModel({
-  maxSize: 500 * 1024 * 1024
-});
+import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
-  let filePaths: string[] = [];
+  /* Creates the multer uploader */
+  const upload = getUploadModel({
+    maxSize: (global.feConfigs?.uploadFileMaxSize || 500) * 1024 * 1024
+  });
+  const filePaths: string[] = [];
 
   try {
-    const { teamId, tmbId } = await authCert({ req, authToken: true });
-
+    await connectToDatabase();
     const { file, bucketName, metadata } = await upload.doUpload(req, res);
 
-    filePaths = [file.path];
-    await connectToDatabase();
+    filePaths.push(file.path);
+
+    const { teamId, tmbId } = await authCert({ req, authToken: true });
 
     if (!bucketName) {
       throw new Error('bucketName is empty');
@@ -46,6 +44,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       error
     });
   }
+
+  removeFilesByPaths(filePaths);
 }
 
 export const config = {

projects/app/src/pages/api/common/file/uploadImage.ts

@@ -12,12 +12,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
     const { teamId } = await authChatCert({ req, authToken: true });
 
-    const data = await uploadMongoImg({
+    const imgId = await uploadMongoImg({
       teamId,
       ...body
     });
 
-    jsonRes(res, { data });
+    jsonRes(res, { data: imgId });
   } catch (error) {
     jsonRes(res, {
       code: 500,