feat: chunk index independent config (#4271)

* sync collection

* remove lock

* feat: chunk index independent config

* feat: add max chunksize to split chunk function

* remove log

* update doc

* remove

* remove log
This commit is contained in:
Archer
2025-03-21 16:44:25 +08:00
committed by archer
parent 222ff0d49a
commit e812ad6e84
47 changed files with 784 additions and 443 deletions

View File

@@ -573,7 +573,6 @@
"core.dataset.import.Custom process": "自定义规则",
"core.dataset.import.Custom process desc": "自定义设置数据处理规则",
"core.dataset.import.Custom prompt": "自定义提示词",
"core.dataset.import.Custom split char": "自定义分隔符",
"core.dataset.import.Custom text": "自定义文本",
"core.dataset.import.Custom text desc": "手动输入一段文本作为数据集",
"core.dataset.import.Data process params": "数据处理参数",

View File

@@ -27,7 +27,6 @@
"custom_data_process_params": "自定义",
"custom_data_process_params_desc": "自定义设置数据处理规则",
"custom_split_sign_tip": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据,使用特定的分隔符来精确分块。可以使用 | 符号表示多个分割符,例如:“。|.” 表示中英文句号。\n尽量避免使用正则相关特殊符号例如: * () [] {} 等。",
"data.ideal_chunk_length": "理想分块长度",
"data_amount": "{{dataAmount}} 组数据, {{indexAmount}} 组索引",
"data_index_num": "索引 {{index}}",
"data_process_params": "处理参数",
@@ -53,8 +52,6 @@
"file_model_function_tip": "用于增强索引和 QA 生成",
"filename": "文件名",
"folder_dataset": "文件夹",
"ideal_chunk_length": "理想分块长度",
"ideal_chunk_length_tips": "按结束符号进行分段,并将多个分段组成一个分块,该值决定了分块的预估大小,如果会有上下浮动。",
"image_auto_parse": "图片自动索引",
"image_auto_parse_tips": "调用 VLM 自动标注文档里的图片,并生成额外的检索索引",
"image_training_queue": "图片处理排队",
@@ -68,6 +65,8 @@
"import_param_setting": "参数设置",
"import_select_file": "选择文件",
"import_select_link": "输入链接",
"index_size": "索引大小",
"index_size_tips": "向量化时内容的长度,系统会自动按该大小对分块进行进一步的分割。",
"is_open_schedule": "启用定时同步",
"keep_image": "保留图片",
"move.hint": "移动后,所选知识库/文件夹将继承新文件夹的权限设置,原先的权限设置失效。",
@@ -89,6 +88,8 @@
"retain_collection": "调整训练参数",
"retrain_task_submitted": "重新训练任务已提交",
"same_api_collection": "存在相同的 API 集合",
"split_chunk_char": "按指定分割符分块",
"split_chunk_size": "按长度分块",
"split_sign_break": "1 个换行符",
"split_sign_break2": "2 个换行符",
"split_sign_custom": "自定义",