mirror of
https://github.com/labring/FastGPT.git
synced 2025-08-02 12:48:30 +00:00
feat: url fetch and create file (#199)
* docs * docs * feat: url fetch and create file
This commit is contained in:
@@ -151,7 +151,7 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
|
||||
const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
|
||||
|
||||
try {
|
||||
const splitTexts = text.split(/(?<=[。!?.!?])/g);
|
||||
const splitTexts = text.split(/(?<=[。!?;.!?;])/g);
|
||||
const chunks: string[] = [];
|
||||
|
||||
let preChunk = '';
|
||||
@@ -268,3 +268,11 @@ export const compressImg = ({
|
||||
reject('压缩图片异常');
|
||||
};
|
||||
});
|
||||
|
||||
/* simple text, remove chinese space and extra \n */
|
||||
export const simpleText = (text: string) => {
|
||||
text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
|
||||
text = text.replace(/\n{2,}/g, '\n');
|
||||
text = text.replace(/\s{2,}/g, ' ');
|
||||
return text;
|
||||
};
|
||||
|
Reference in New Issue
Block a user