perf: password special chars;feat: llm paragraph;perf: chunk setting params;perf: text splitter worker (#4984)

* perf: password special chars * feat: llm paragraph;perf: chunk setting params * perf: text splitter worker * perf: get rawtext buffer * fix: test * fix: test * doc * min chunk size
2025-07-29 09:44:47 +00:00 · 2025-06-10 00:05:54 +08:00
parent 068918a9ee
commit 01ff56b42b
41 changed files with 546 additions and 448 deletions
--- a/packages/global/core/dataset/constants.ts
+++ b/packages/global/core/dataset/constants.ts
@@ -211,7 +211,8 @@ export enum DataChunkSplitModeEnum {
 }
 export enum ParagraphChunkAIModeEnum {
  auto = 'auto',
-  force = 'force'
+  force = 'force',
+  forbid = 'forbid'
 }

 /* ------------ data -------------- */