Optimize base64 storage in files to support concurrent storage (#2856)

* fix: variables check

* remove log

* perf: file img saved

* update doc

packages/global/common/fn/utils.ts

@@ -8,3 +8,24 @@ export const retryRun = <T>(fn: () => T, retry = 2): T => {
     throw error;
   }
 };
+
+export const batchRun = async <T>(arr: T[], fn: (arr: T) => any, batchSize = 10) => {
+  const batchArr = new Array(batchSize).fill(null);
+  const result: any[] = [];
+
+  const batchFn = async () => {
+    const data = arr.shift();
+    if (data) {
+      result.push(await fn(data));
+      return batchFn();
+    }
+  };
+
+  await Promise.all(
+    batchArr.map(async () => {
+      await batchFn();
+    })
+  );
+
+  return result;
+};

packages/global/common/string/markdown.ts

@@ -1,3 +1,4 @@
+import { batchRun } from '../fn/utils';
 import { simpleText } from './tools';
 
 /* Delete redundant text in markdown */
@@ -53,16 +54,19 @@ export const uploadMarkdownBase64 = async ({
     const base64Arr = rawText.match(base64Regex) || [];
 
     // upload base64 and replace it
-    for await (const base64Img of base64Arr) {
-      try {
-        const str = await uploadImgController(base64Img);
-
-        rawText = rawText.replace(base64Img, str);
-      } catch (error) {
-        rawText = rawText.replace(base64Img, '');
-        rawText = rawText.replace(/!\[.*\]\(\)/g, '');
-      }
-    }
+    await batchRun(
+      base64Arr,
+      async (base64Img) => {
+        try {
+          const str = await uploadImgController(base64Img);
+          rawText = rawText.replace(base64Img, str);
+        } catch (error) {
+          rawText = rawText.replace(base64Img, '');
+          rawText = rawText.replace(/!\[.*\]\(\)/g, '');
+        }
+      },
+      20
+    );
   }
 
   // Remove white space on both sides of the picture

packages/service/common/file/image/schema.ts

@@ -31,8 +31,8 @@ const ImageSchema = new Schema({
 });
 
 try {
-  // tts expired
-  ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
+  // tts expired（60 Minutes）
+  ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 * 60 });
   ImageSchema.index({ type: 1 });
   ImageSchema.index({ createTime: 1 });
   // delete related img

packages/service/common/file/multer.ts

@@ -30,9 +30,13 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
         // destination: (_req, _file, cb) => {
         //   cb(null, tmpFileDirPath);
         // },
-        filename: async (req, file, cb) => {
-          const { ext } = path.parse(decodeURIComponent(file.originalname));
-          cb(null, `${getNanoid()}${ext}`);
+        filename: (req, file, cb) => {
+          if (!file?.originalname) {
+            cb(new Error('File not found'), '');
+          } else {
+            const { ext } = path.parse(decodeURIComponent(file.originalname));
+            cb(null, `${getNanoid()}${ext}`);
+          }
         }
       })
     }).single('file');

packages/service/worker/htmlStr2Md/utils.ts

@@ -1,5 +1,4 @@
 import TurndownService from 'turndown';
-const domino = require('domino-ext');
 const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
 
 export const html2md = (html: string): string => {
@@ -15,24 +14,11 @@ export const html2md = (html: string): string => {
   });
 
   try {
-    const window = domino.createWindow(html);
-    const document = window.document;
-
-    turndownService.remove(['i', 'script', 'iframe']);
-    turndownService.addRule('codeBlock', {
-      filter: 'pre',
-      replacement(_, node) {
-        const content = node.textContent?.trim() || '';
-        // @ts-ignore
-        const codeName = node?._attrsByQName?.class?.data?.trim() || '';
-
-        return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
-      }
-    });
+    turndownService.remove(['i', 'script', 'iframe', 'style']);
 
     turndownService.use(turndownPluginGfm.gfm);
 
-    return turndownService.turndown(document);
+    return turndownService.turndown(html);
   } catch (error) {
     console.log('html 2 markdown error', error);
     return '';