ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 05:12:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updae theme and fix some bug (#1711)

Browse Source
This commit is contained in:
Archer
2024-06-07 12:54:30 +08:00
committed by GitHub
parent 19c8a06d51
commit b20d075d35
153 changed files with 1587 additions and 1436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/global/common/string/textSplitter.ts
View File

@@ -102,6 +102,8 @@ const commonSplit = (props: SplitProps): SplitResponse => {
text = text.replace(/(```[\s\S]*?```|~~~[\s\S]*?~~~)/g, function (match) {
return match.replace(/\n/g, codeBlockMarker);
});
// replace invalid \n
text = text.replace(/(\r?\n|\r){3,}/g, '\n\n\n');
// The larger maxLen is, the next sentence is less likely to trigger splitting
const stepReges: { reg: RegExp; maxLen: number }[] = [
@@ -338,7 +340,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
*/
export const splitText2Chunks = (props: SplitProps): SplitResponse => {
let { text = '' } = props;
const start = Date.now();
const splitWithCustomSign = text.split(CUSTOM_SPLIT_SIGN);
const splitResult = splitWithCustomSign.map((item) => {
@@ -348,7 +350,7 @@ export const splitText2Chunks = (props: SplitProps): SplitResponse => {
return commonSplit(props);
});
console.log(Date.now() - start);
return {
chunks: splitResult.map((item) => item.chunks).flat(),
chars: splitResult.reduce((sum, item) => sum + item.chars, 0)