feat: url fetch and create file (#199)

* docs

* docs

* feat: url fetch and create file
This commit is contained in:
Archer
2023-08-19 12:54:24 +08:00
committed by GitHub
parent 4054eb9d18
commit 1fcdd7cb8d
20 changed files with 583 additions and 359 deletions

View File

@@ -151,7 +151,7 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
try {
const splitTexts = text.split(/(?<=[。!?.!?])/g);
const splitTexts = text.split(/(?<=[。!?.!?;])/g);
const chunks: string[] = [];
let preChunk = '';
@@ -268,3 +268,11 @@ export const compressImg = ({
reject('压缩图片异常');
};
});
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\n{2,}/g, '\n');
text = text.replace(/\s{2,}/g, ' ');
return text;
};