feat: chunk index independent config (#4271)

* sync collection

* remove lock

* feat: chunk index independent config

* feat: add max chunksize to split chunk function

* remove log

* update doc

* remove

* remove log

packages/web/components/common/MyModal/EditFolderModal.tsx

@@ -72,7 +72,7 @@ const EditFolderModal = ({
             {...register('name', { required: true })}
             bg={'myGray.50'}
             autoFocus
-            maxLength={20}
+            maxLength={100}
           />
         </Box>
         <Box mt={4}>

packages/web/components/common/Radio/RadioGroup.tsx

@@ -0,0 +1,67 @@
+import React from 'react';
+import { Box, Flex, Grid, type GridProps, HStack } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+import QuestionTip from '../MyTooltip/QuestionTip';
+
+type Props<T> = Omit<GridProps, 'onChange'> & {
+  list: {
+    title: string;
+    value: T;
+    tooltip?: string;
+  }[];
+  value: T;
+  defaultBg?: string;
+  activeBg?: string;
+  onChange: (e: T) => void;
+};
+
+const RadioGroup = <T = any,>({ list, value, onChange, ...props }: Props<T>) => {
+  const { t } = useTranslation();
+
+  return (
+    <Flex gap={[3, 5]} fontSize={['sm', 'md']} alignItems={'center'} {...props}>
+      {list.map((item) => (
+        <Flex
+          alignItems={'center'}
+          key={item.value as any}
+          cursor={'pointer'}
+          userSelect={'none'}
+          gap={1}
+          onClick={() => onChange(item.value)}
+        >
+          <Box
+            w={'18px'}
+            h={'18px'}
+            borderWidth={'2.4px'}
+            borderColor={value === item.value ? 'primary.015' : 'transparent'}
+            borderRadius={'50%'}
+          >
+            <Flex
+              w={'100%'}
+              h={'100%'}
+              borderWidth={'1px'}
+              borderColor={value === item.value ? 'primary.600' : 'borderColor.high'}
+              bg={value === item.value ? 'primary.1' : 'transparent'}
+              borderRadius={'50%'}
+              alignItems={'center'}
+              justifyContent={'center'}
+            >
+              <Box
+                w={'5px'}
+                h={'5px'}
+                borderRadius={'50%'}
+                bg={value === item.value ? 'primary.600' : 'transparent'}
+              />
+            </Flex>
+          </Box>
+          <HStack spacing={1} color={'myGray.900'} whiteSpace={'nowrap'} fontSize={'sm'}>
+            <Box>{typeof item.title === 'string' ? t(item.title as any) : item.title}</Box>
+            {!!item.tooltip && <QuestionTip label={item.tooltip} color={'myGray.600'} />}
+          </HStack>
+        </Flex>
+      ))}
+    </Flex>
+  );
+};
+
+export default RadioGroup;

packages/web/i18n/en/common.json

@@ -569,7 +569,6 @@
   "core.dataset.import.Custom process": "Custom Rules",
   "core.dataset.import.Custom process desc": "Customize segmentation and preprocessing rules",
   "core.dataset.import.Custom prompt": "Custom Prompt",
-  "core.dataset.import.Custom split char": "Custom Separator",
   "core.dataset.import.Custom text": "Custom Text",
   "core.dataset.import.Custom text desc": "Manually enter a piece of text as a dataset",
   "core.dataset.import.Data process params": "Data Processing Parameters",

packages/web/i18n/en/dataset.json

@@ -27,7 +27,6 @@
   "custom_data_process_params": "Custom",
   "custom_data_process_params_desc": "Customize data processing rules",
   "custom_split_sign_tip": "Allows you to chunk according to custom delimiters. \nUsually used for processed data, using specific separators for precise chunking. \nYou can use the | symbol to represent multiple splitters, such as: \".|.\" to represent a period in Chinese and English.\n\nTry to avoid using special symbols related to regular, such as: * () [] {}, etc.",
-  "data.ideal_chunk_length": "ideal block length",
   "data_amount": "{{dataAmount}} Datas, {{indexAmount}} Indexes",
   "data_index_num": "Index {{index}}",
   "data_process_params": "Params",
@@ -53,8 +52,6 @@
   "file_model_function_tip": "Enhances indexing and QA generation",
   "filename": "Filename",
   "folder_dataset": "Folder",
-  "ideal_chunk_length": "ideal block length",
-  "ideal_chunk_length_tips": "Segment according to the end symbol and combine multiple segments into one block. This value determines the estimated size of the block, if there is any fluctuation.",
   "image_auto_parse": "Automatic image indexing",
   "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
   "image_training_queue": "Queue of image processing",
@@ -68,6 +65,8 @@
   "import_param_setting": "Parameter settings",
   "import_select_file": "Select a file",
   "import_select_link": "Enter link",
+  "index_size": "Index size",
+  "index_size_tips": "When vectorized, the system will automatically further segment the blocks according to this size.",
   "is_open_schedule": "Enable scheduled synchronization",
   "keep_image": "Keep the picture",
   "move.hint": "After moving, the selected knowledge base/folder will inherit the permission settings of the new folder, and the original permission settings will become invalid.",
@@ -89,6 +88,8 @@
   "retain_collection": "Adjust Training Parameters",
   "retrain_task_submitted": "The retraining task has been submitted",
   "same_api_collection": "The same API set exists",
+  "split_chunk_char": "Block by specified splitter",
+  "split_chunk_size": "Block by length",
   "split_sign_break": "1 newline character",
   "split_sign_break2": "2 newline characters",
   "split_sign_custom": "Customize",

packages/web/i18n/zh-CN/common.json

@@ -573,7 +573,6 @@
   "core.dataset.import.Custom process": "自定义规则",
   "core.dataset.import.Custom process desc": "自定义设置数据处理规则",
   "core.dataset.import.Custom prompt": "自定义提示词",
-  "core.dataset.import.Custom split char": "自定义分隔符",
   "core.dataset.import.Custom text": "自定义文本",
   "core.dataset.import.Custom text desc": "手动输入一段文本作为数据集",
   "core.dataset.import.Data process params": "数据处理参数",

packages/web/i18n/zh-CN/dataset.json

@@ -27,7 +27,6 @@
   "custom_data_process_params": "自定义",
   "custom_data_process_params_desc": "自定义设置数据处理规则",
   "custom_split_sign_tip": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据，使用特定的分隔符来精确分块。可以使用 | 符号表示多个分割符，例如：“。|.” 表示中英文句号。\n尽量避免使用正则相关特殊符号，例如: * () [] {} 等。",
-  "data.ideal_chunk_length": "理想分块长度",
   "data_amount": "{{dataAmount}} 组数据, {{indexAmount}} 组索引",
   "data_index_num": "索引 {{index}}",
   "data_process_params": "处理参数",
@@ -53,8 +52,6 @@
   "file_model_function_tip": "用于增强索引和 QA 生成",
   "filename": "文件名",
   "folder_dataset": "文件夹",
-  "ideal_chunk_length": "理想分块长度",
-  "ideal_chunk_length_tips": "按结束符号进行分段，并将多个分段组成一个分块，该值决定了分块的预估大小，如果会有上下浮动。",
   "image_auto_parse": "图片自动索引",
   "image_auto_parse_tips": "调用 VLM 自动标注文档里的图片，并生成额外的检索索引",
   "image_training_queue": "图片处理排队",
@@ -68,6 +65,8 @@
   "import_param_setting": "参数设置",
   "import_select_file": "选择文件",
   "import_select_link": "输入链接",
+  "index_size": "索引大小",
+  "index_size_tips": "向量化时内容的长度，系统会自动按该大小对分块进行进一步的分割。",
   "is_open_schedule": "启用定时同步",
   "keep_image": "保留图片",
   "move.hint": "移动后，所选知识库/文件夹将继承新文件夹的权限设置，原先的权限设置失效。",
@@ -89,6 +88,8 @@
   "retain_collection": "调整训练参数",
   "retrain_task_submitted": "重新训练任务已提交",
   "same_api_collection": "存在相同的 API 集合",
+  "split_chunk_char": "按指定分割符分块",
+  "split_chunk_size": "按长度分块",
   "split_sign_break": "1 个换行符",
   "split_sign_break2": "2 个换行符",
   "split_sign_custom": "自定义",

packages/web/i18n/zh-Hant/common.json

@@ -568,7 +568,6 @@
   "core.dataset.import.Custom process": "自訂規則",
   "core.dataset.import.Custom process desc": "自訂設定資料處理規則",
   "core.dataset.import.Custom prompt": "自訂提示詞",
-  "core.dataset.import.Custom split char": "自訂分隔符",
   "core.dataset.import.Custom text": "自訂文字",
   "core.dataset.import.Custom text desc": "手動輸入一段文字作為資料集",
   "core.dataset.import.Data process params": "資料處理參數",

packages/web/i18n/zh-Hant/dataset.json

@@ -27,7 +27,6 @@
   "custom_data_process_params": "自訂",
   "custom_data_process_params_desc": "自訂資料處理規則",
   "custom_split_sign_tip": "允許你根據自定義的分隔符進行分塊。\n通常用於已處理好的數據，使用特定的分隔符來精確分塊。\n可以使用 | 符號表示多個分割符，例如：“。|.” 表示中英文句號。\n\n盡量避免使用正則相關特殊符號，例如: * () [] {} 等。",
-  "data.ideal_chunk_length": "理想分塊長度",
   "data_amount": "{{dataAmount}} 組數據, {{indexAmount}} 組索引",
   "data_index_num": "索引 {{index}}",
   "data_process_params": "處理參數",
@@ -53,8 +52,6 @@
   "file_model_function_tip": "用於增強索引和問答生成",
   "filename": "檔案名稱",
   "folder_dataset": "資料夾",
-  "ideal_chunk_length": "理想分塊長度",
-  "ideal_chunk_length_tips": "依結束符號進行分段，並將多個分段組成一個分塊，此值決定了分塊的預估大小，可能會有上下浮動。",
   "image_auto_parse": "圖片自動索引",
   "image_auto_parse_tips": "調用 VLM 自動標註文檔裡的圖片，並生成額外的檢索索引",
   "image_training_queue": "圖片處理排隊",
@@ -68,6 +65,8 @@
   "import_param_setting": "參數設置",
   "import_select_file": "選擇文件",
   "import_select_link": "輸入鏈接",
+  "index_size": "索引大小",
+  "index_size_tips": "向量化時內容的長度，系統會自動按該大小對分塊進行進一步的分割。",
   "is_open_schedule": "啟用定時同步",
   "keep_image": "保留圖片",
   "move.hint": "移動後，所選資料集／資料夾將繼承新資料夾的權限設定，原先的權限設定將失效。",
@@ -89,6 +88,8 @@
   "retain_collection": "調整訓練參數",
   "retrain_task_submitted": "重新訓練任務已提交",
   "same_api_collection": "存在相同的 API 集合",
+  "split_chunk_char": "按指定分割符分塊",
+  "split_chunk_size": "按長度分塊",
   "split_sign_break": "1 個換行符",
   "split_sign_break2": "2 個換行符",
   "split_sign_custom": "自定義",