This commit is contained in:
Archer
2023-12-18 16:24:50 +08:00
committed by GitHub
parent d33c99f564
commit 703583fff7
130 changed files with 3418 additions and 2579 deletions

View File

@@ -243,8 +243,8 @@ function formatSplitText(text: string, rawText: string) {
// empty result. direct split chunk
if (result.length === 0) {
const splitRes = splitText2Chunks({ text: rawText, chunkLen: 512 });
splitRes.chunks.forEach((chunk) => {
const { chunks } = splitText2Chunks({ text: rawText, chunkLen: 512, countTokens: false });
chunks.forEach((chunk) => {
result.push({
q: chunk,
a: '',