mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-24 13:53:50 +00:00
Optimize base64 storage in files to support concurrent storage (#2856)
* fix: variables check * remove log * perf: file img saved * update doc
This commit is contained in:
@@ -8,3 +8,24 @@ export const retryRun = <T>(fn: () => T, retry = 2): T => {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const batchRun = async <T>(arr: T[], fn: (arr: T) => any, batchSize = 10) => {
|
||||
const batchArr = new Array(batchSize).fill(null);
|
||||
const result: any[] = [];
|
||||
|
||||
const batchFn = async () => {
|
||||
const data = arr.shift();
|
||||
if (data) {
|
||||
result.push(await fn(data));
|
||||
return batchFn();
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(
|
||||
batchArr.map(async () => {
|
||||
await batchFn();
|
||||
})
|
||||
);
|
||||
|
||||
return result;
|
||||
};
|
||||
|
@@ -1,3 +1,4 @@
|
||||
import { batchRun } from '../fn/utils';
|
||||
import { simpleText } from './tools';
|
||||
|
||||
/* Delete redundant text in markdown */
|
||||
@@ -53,16 +54,19 @@ export const uploadMarkdownBase64 = async ({
|
||||
const base64Arr = rawText.match(base64Regex) || [];
|
||||
|
||||
// upload base64 and replace it
|
||||
for await (const base64Img of base64Arr) {
|
||||
try {
|
||||
const str = await uploadImgController(base64Img);
|
||||
|
||||
rawText = rawText.replace(base64Img, str);
|
||||
} catch (error) {
|
||||
rawText = rawText.replace(base64Img, '');
|
||||
rawText = rawText.replace(/!\[.*\]\(\)/g, '');
|
||||
}
|
||||
}
|
||||
await batchRun(
|
||||
base64Arr,
|
||||
async (base64Img) => {
|
||||
try {
|
||||
const str = await uploadImgController(base64Img);
|
||||
rawText = rawText.replace(base64Img, str);
|
||||
} catch (error) {
|
||||
rawText = rawText.replace(base64Img, '');
|
||||
rawText = rawText.replace(/!\[.*\]\(\)/g, '');
|
||||
}
|
||||
},
|
||||
20
|
||||
);
|
||||
}
|
||||
|
||||
// Remove white space on both sides of the picture
|
||||
|
@@ -31,8 +31,8 @@ const ImageSchema = new Schema({
|
||||
});
|
||||
|
||||
try {
|
||||
// tts expired
|
||||
ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
|
||||
// tts expired(60 Minutes)
|
||||
ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 * 60 });
|
||||
ImageSchema.index({ type: 1 });
|
||||
ImageSchema.index({ createTime: 1 });
|
||||
// delete related img
|
||||
|
@@ -30,9 +30,13 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
|
||||
// destination: (_req, _file, cb) => {
|
||||
// cb(null, tmpFileDirPath);
|
||||
// },
|
||||
filename: async (req, file, cb) => {
|
||||
const { ext } = path.parse(decodeURIComponent(file.originalname));
|
||||
cb(null, `${getNanoid()}${ext}`);
|
||||
filename: (req, file, cb) => {
|
||||
if (!file?.originalname) {
|
||||
cb(new Error('File not found'), '');
|
||||
} else {
|
||||
const { ext } = path.parse(decodeURIComponent(file.originalname));
|
||||
cb(null, `${getNanoid()}${ext}`);
|
||||
}
|
||||
}
|
||||
})
|
||||
}).single('file');
|
||||
|
@@ -1,5 +1,4 @@
|
||||
import TurndownService from 'turndown';
|
||||
const domino = require('domino-ext');
|
||||
const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
|
||||
|
||||
export const html2md = (html: string): string => {
|
||||
@@ -15,24 +14,11 @@ export const html2md = (html: string): string => {
|
||||
});
|
||||
|
||||
try {
|
||||
const window = domino.createWindow(html);
|
||||
const document = window.document;
|
||||
|
||||
turndownService.remove(['i', 'script', 'iframe']);
|
||||
turndownService.addRule('codeBlock', {
|
||||
filter: 'pre',
|
||||
replacement(_, node) {
|
||||
const content = node.textContent?.trim() || '';
|
||||
// @ts-ignore
|
||||
const codeName = node?._attrsByQName?.class?.data?.trim() || '';
|
||||
|
||||
return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
|
||||
}
|
||||
});
|
||||
turndownService.remove(['i', 'script', 'iframe', 'style']);
|
||||
|
||||
turndownService.use(turndownPluginGfm.gfm);
|
||||
|
||||
return turndownService.turndown(document);
|
||||
return turndownService.turndown(html);
|
||||
} catch (error) {
|
||||
console.log('html 2 markdown error', error);
|
||||
return '';
|
||||
|
Reference in New Issue
Block a user