From 6c16fa91663404eb62f345a4cb2369b1c5bfd7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=85=E9=87=8C=E6=A0=88=E6=A0=88?= <37398336+2277419213@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:16:17 +0800 Subject: [PATCH] Fix: Custom delimiter does not take effect when document type is link (#2565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 勤劳上班的卑微小张 --- packages/service/core/dataset/collection/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 5c31c5518..1718a800a 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -159,7 +159,8 @@ export const reloadCollectionChunks = async ({ // split data const { chunks } = splitText2Chunks({ text: newRawText, - chunkLen: col.chunkSize || 512 + chunkLen: col.chunkSize || 512, + customReg: col.chunkSplitter ? [col.chunkSplitter] : [], }); // insert to training queue