Optimize base64 storage in files to support concurrent storage (#2856)

* fix: variables check

* remove log

* perf: file img saved

* update doc
This commit is contained in:
Archer
2024-10-08 12:58:33 +08:00
committed by GitHub
parent dd3a1b910b
commit f6c5695df4
19 changed files with 156 additions and 80 deletions

View File

@@ -8,3 +8,24 @@ export const retryRun = <T>(fn: () => T, retry = 2): T => {
throw error;
}
};
export const batchRun = async <T>(arr: T[], fn: (arr: T) => any, batchSize = 10) => {
const batchArr = new Array(batchSize).fill(null);
const result: any[] = [];
const batchFn = async () => {
const data = arr.shift();
if (data) {
result.push(await fn(data));
return batchFn();
}
};
await Promise.all(
batchArr.map(async () => {
await batchFn();
})
);
return result;
};

View File

@@ -1,3 +1,4 @@
import { batchRun } from '../fn/utils';
import { simpleText } from './tools';
/* Delete redundant text in markdown */
@@ -53,16 +54,19 @@ export const uploadMarkdownBase64 = async ({
const base64Arr = rawText.match(base64Regex) || [];
// upload base64 and replace it
for await (const base64Img of base64Arr) {
try {
const str = await uploadImgController(base64Img);
rawText = rawText.replace(base64Img, str);
} catch (error) {
rawText = rawText.replace(base64Img, '');
rawText = rawText.replace(/!\[.*\]\(\)/g, '');
}
}
await batchRun(
base64Arr,
async (base64Img) => {
try {
const str = await uploadImgController(base64Img);
rawText = rawText.replace(base64Img, str);
} catch (error) {
rawText = rawText.replace(base64Img, '');
rawText = rawText.replace(/!\[.*\]\(\)/g, '');
}
},
20
);
}
// Remove white space on both sides of the picture

View File

@@ -31,8 +31,8 @@ const ImageSchema = new Schema({
});
try {
// tts expired
ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
// tts expired60 Minutes
ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 * 60 });
ImageSchema.index({ type: 1 });
ImageSchema.index({ createTime: 1 });
// delete related img

View File

@@ -30,9 +30,13 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
// destination: (_req, _file, cb) => {
// cb(null, tmpFileDirPath);
// },
filename: async (req, file, cb) => {
const { ext } = path.parse(decodeURIComponent(file.originalname));
cb(null, `${getNanoid()}${ext}`);
filename: (req, file, cb) => {
if (!file?.originalname) {
cb(new Error('File not found'), '');
} else {
const { ext } = path.parse(decodeURIComponent(file.originalname));
cb(null, `${getNanoid()}${ext}`);
}
}
})
}).single('file');

View File

@@ -1,5 +1,4 @@
import TurndownService from 'turndown';
const domino = require('domino-ext');
const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
export const html2md = (html: string): string => {
@@ -15,24 +14,11 @@ export const html2md = (html: string): string => {
});
try {
const window = domino.createWindow(html);
const document = window.document;
turndownService.remove(['i', 'script', 'iframe']);
turndownService.addRule('codeBlock', {
filter: 'pre',
replacement(_, node) {
const content = node.textContent?.trim() || '';
// @ts-ignore
const codeName = node?._attrsByQName?.class?.data?.trim() || '';
return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
}
});
turndownService.remove(['i', 'script', 'iframe', 'style']);
turndownService.use(turndownPluginGfm.gfm);
return turndownService.turndown(document);
return turndownService.turndown(html);
} catch (error) {
console.log('html 2 markdown error', error);
return '';