ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 21:13:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

v4.6-4 (#473)

Browse Source
This commit is contained in:
Archer
2023-11-15 21:35:50 +08:00
committed by GitHub
parent bfd8be5df0
commit cd3acb44ab
39 changed files with 457 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/global/common/string/textSplitter.ts
View File

@@ -63,8 +63,8 @@ export const splitText2Chunks = (props: { text: string; maxLen: number; overlapL
let chunks: string[] = [];
for (let i = 0; i < splitTexts.length; i++) {
let text = splitTexts[i];
let chunkToken = countPromptTokens(lastChunk, '');
const textToken = countPromptTokens(text, '');
let chunkToken = lastChunk.length;
const textToken = text.length;
// next chunk is too large / new chunk is too large(The current chunk must be smaller than maxLen)
if (textToken >= maxLen || chunkToken + textToken > maxLen * 1.4) {