Add image index and pdf parse (#3956)

* feat: think tag parse

* feat: parse think tag test

* feat: pdf parse ux

* feat: doc2x parse

* perf: rewrite training mode setting

* feat: image parse queue

* perf: image index

* feat: image parse process

* feat: add init sh

* fix: ts

packages/web/i18n/zh-Hant/account_usage.json

@@ -2,6 +2,7 @@
   "ai_model": "AI 模型",
   "all": "所有",
   "app_name": "應用程式名",
+  "auto_index": "索引增強",
   "billing_module": "扣費模組",
   "confirm_export": "共篩選出 {{total}} 條數據，是否確認導出？",
   "current_filter_conditions": "當前篩選條件：",
@@ -9,6 +10,7 @@
   "details": "詳情",
   "dingtalk": "釘釘",
   "duration_seconds": "時長（秒）",
+  "embedding_index": "索引生成",
   "every_day": "天",
   "every_month": "月",
   "export_confirm": "導出確認",
@@ -16,6 +18,7 @@
   "export_title": "時間,成員,類型,項目名,AI 積分消耗",
   "feishu": "飛書",
   "generation_time": "生成時間",
+  "image_parse": "圖片標註",
   "input_token_length": "輸入 tokens",
   "member": "成員",
   "member_name": "成員名",
@@ -25,8 +28,12 @@
   "official_account": "公眾號",
   "order_number": "訂單編號",
   "output_token_length": "輸出 tokens",
+  "pages": "頁數",
+  "pdf_enhanced_parse": "PDF 增強解析",
+  "pdf_parse": "PDF 解析",
   "points": "積分",
   "project_name": "專案名",
+  "qa": "問答對提取",
   "select_member_and_source_first": "請先選取成員和類型",
   "share": "分享連結",
   "source": "來源",

packages/web/i18n/zh-Hant/common.json

@@ -561,10 +561,7 @@
   "core.dataset.file": "檔案",
   "core.dataset.folder": "目錄",
   "core.dataset.import.Auto mode Estimated Price Tips": "需要呼叫檔案處理模型，將消耗較多 AI 點數：{{price}} 點數/1K tokens",
-  "core.dataset.import.Auto process": "自動",
-  "core.dataset.import.Auto process desc": "自動設定分割和預處理規則",
   "core.dataset.import.Chunk Range": "範圍：{{min}}~{{max}}",
-  "core.dataset.import.Chunk Split": "直接分段",
   "core.dataset.import.Chunk Split Tip": "將文字依照特定規則進行分段處理後，轉換成可進行語意搜尋的格式，適合大多數場景。不需要呼叫模型額外處理，成本較低。",
   "core.dataset.import.Continue upload": "繼續上傳",
   "core.dataset.import.Custom process": "自訂規則",
@@ -574,7 +571,6 @@
   "core.dataset.import.Custom split char Tips": "允許您根據自訂的分隔符進行分割。通常用於已處理好的資料，使用特定的分隔符來精確分割。",
   "core.dataset.import.Custom text": "自訂文字",
   "core.dataset.import.Custom text desc": "手動輸入一段文字作為資料集",
-  "core.dataset.import.Data Preprocessing": "資料處理",
   "core.dataset.import.Data process params": "資料處理參數",
   "core.dataset.import.Down load csv template": "點選下載 CSV 範本",
   "core.dataset.import.Embedding Estimated Price Tips": "僅使用索引模型，消耗少量 AI 點數：{{price}} 點數/1K tokens",
@@ -596,7 +592,6 @@
   "core.dataset.import.Source name": "來源名稱",
   "core.dataset.import.Sources list": "來源列表",
   "core.dataset.import.Start upload": "開始上傳",
-  "core.dataset.import.Total files": "共 {{total}} 個檔案",
   "core.dataset.import.Upload complete": "上傳完成",
   "core.dataset.import.Upload data": "確認上傳",
   "core.dataset.import.Upload file progress": "檔案上傳進度",
@@ -646,12 +641,12 @@
   "core.dataset.test.test result placeholder": "測試結果將顯示在這裡",
   "core.dataset.test.test result tip": "根據知識庫內容與測試文字的相似度進行排序。您可以根據測試結果調整相應的文字。\n注意：測試記錄中的資料可能已經被修改。點選某筆測試資料後將顯示最新資料。",
   "core.dataset.training.Agent queue": "問答訓練排隊中",
-  "core.dataset.training.Auto mode": "增強處理",
+  "core.dataset.training.Auto mode": "補充索引",
   "core.dataset.training.Auto mode Tip": "透過子索引以及呼叫模型產生相關問題與摘要，來增加資料區塊的語意豐富度，更有利於檢索。需要消耗更多的儲存空間並增加 AI 呼叫次數。",
-  "core.dataset.training.Chunk mode": "直接分段",
+  "core.dataset.training.Chunk mode": "直接分块",
   "core.dataset.training.Full": "預計超過 5 分鐘",
   "core.dataset.training.Leisure": "閒置",
-  "core.dataset.training.QA mode": "問答拆分",
+  "core.dataset.training.QA mode": "問答對提取",
   "core.dataset.training.Vector queue": "索引排隊中",
   "core.dataset.training.Waiting": "預計 5 分鐘",
   "core.dataset.training.Website Sync": "網站同步",
@@ -861,7 +856,6 @@
   "dataset.collections.Select Collection": "選擇檔案",
   "dataset.collections.Select One Collection To Store": "選擇一個檔案進行儲存",
   "dataset.data.Can not edit": "無編輯權限",
-  "dataset.data.Custom Index Number": "自訂索引 {{number}}",
   "dataset.data.Default Index": "預設索引",
   "dataset.data.Delete Tip": "確認刪除此資料？",
   "dataset.data.Index Placeholder": "輸入索引文字內容",
@@ -955,6 +949,7 @@
   "new_create": "建立新項目",
   "no": "否",
   "no_laf_env": "系統未設定 LAF 環境",
+  "not_model_config": "未配置相關模型",
   "not_yet_introduced": "暫無介紹",
   "option": "選項",
   "pay.amount": "金額",
@@ -1120,7 +1115,6 @@
   "support.wallet.invoice_detail": "發票詳細資訊",
   "support.wallet.invoice_info": "發票將在 3-7 個工作天內寄送至電子郵件信箱，請耐心等候",
   "support.wallet.invoicing": "開立發票",
-  "support.wallet.moduleName.index": "產生索引",
   "support.wallet.moduleName.qa": "問答拆分",
   "support.wallet.noBill": "無帳單紀錄",
   "support.wallet.no_invoice": "無發票紀錄",

packages/web/i18n/zh-Hant/dataset.json

@@ -3,11 +3,16 @@
   "add_file": "新增文件",
   "api_file": "API 檔案庫",
   "api_url": "介面位址",
+  "auto_indexes": "自動生成補充索引",
+  "auto_indexes_tips": "通過大模型進行額外索引生成，提高語義豐富度，提高檢索的精度。",
   "chunk_max_tokens": "分塊上限",
   "close_auto_sync": "確認關閉自動同步功能？",
   "collection.Create update time": "建立／更新時間",
   "collection.Training type": "分段模式",
+  "collection.training_type": "處理模式",
   "collection_data_count": "數據量",
+  "collection_metadata_custom_pdf_parse": "PDF增強解析",
+  "collection_metadata_image_parse": "圖片標註",
   "collection_not_support_retraining": "此集合類型不支援重新調整參數",
   "collection_not_support_sync": "該集合不支援同步",
   "collection_sync": "立即同步",
@@ -22,12 +27,21 @@
   "custom_data_process_params_desc": "自訂資料處理規則",
   "data.ideal_chunk_length": "理想分塊長度",
   "data_amount": "{{dataAmount}} 組數據, {{indexAmount}} 組索引",
+  "data_index_custom": "自定義索引",
+  "data_index_default": "默認索引",
+  "data_index_image": "圖片索引",
+  "data_index_num": "索引 {{index}}",
+  "data_index_question": "推測問題索引",
+  "data_index_summary": "摘要索引",
   "data_process_params": "處理參數",
   "data_process_setting": "資料處理設定",
   "dataset.Unsupported operation": "操作不支持",
   "dataset.no_collections": "尚無資料集",
   "dataset.no_tags": "尚無標籤",
+  "default_params": "預設",
+  "default_params_desc": "使用系統默認的參數和規則",
   "edit_dataset_config": "編輯知識庫配置",
+  "enhanced_indexes": "索引增強",
   "error.collectionNotFound": "找不到集合",
   "external_file": "外部檔案庫",
   "external_file_dataset_desc": "可以從外部檔案庫匯入檔案建立資料集，檔案不會進行二次儲存",
@@ -38,19 +52,38 @@
   "feishu_dataset": "飛書知識庫",
   "feishu_dataset_config": "配置飛書知識庫",
   "feishu_dataset_desc": "可通過配置飛書文檔權限，使用飛書文檔構建知識庫，文檔不會進行二次存儲",
+  "file_list": "文件列表",
   "file_model_function_tip": "用於增強索引和問答生成",
   "filename": "檔案名稱",
   "folder_dataset": "資料夾",
   "ideal_chunk_length": "理想分塊長度",
   "ideal_chunk_length_tips": "依結束符號進行分段，並將多個分段組成一個分塊，此值決定了分塊的預估大小，可能會有上下浮動。",
+  "image_auto_parse": "圖片自動索引",
+  "image_auto_parse_tips": "調用 VLM 自動標註文檔裡的圖片，並生成額外的檢索索引",
   "import.Auto mode Estimated Price Tips": "需呼叫文字理解模型，將消耗較多 AI 點數：{{price}} 點數 / 1K tokens",
   "import.Embedding Estimated Price Tips": "僅使用索引模型，消耗少量 AI 點數：{{price}} 點數 / 1K tokens",
+  "import_confirm": "確認上傳",
+  "import_data_preview": "數據預覽",
+  "import_data_process_setting": "數據處理方式設置",
+  "import_file_parse_setting": "文件解析設置",
+  "import_model_config": "模型選擇",
+  "import_param_setting": "參數設置",
+  "import_select_file": "選擇文件",
   "is_open_schedule": "啟用定時同步",
+  "keep_image": "保留圖片",
   "move.hint": "移動後，所選資料集／資料夾將繼承新資料夾的權限設定，原先的權限設定將失效。",
   "open_auto_sync": "開啟定時同步後，系統將每天不定時嘗試同步集合，集合同步期間，會出現無法搜尋到該集合資料現象。",
+  "params_setting": "參數設置",
+  "pdf_enhance_parse": "PDF增強解析",
+  "pdf_enhance_parse_price": "{{price}}積分/頁",
+  "pdf_enhance_parse_tips": "解析 PDF 文件時，調用 PDF 識別模型進行識別，可以將其轉換成 Markdown 並保留文檔中的圖片，同時也可以對掃描件進行識別。",
   "permission.des.manage": "可管理整個資料集的資料和資訊",
   "permission.des.read": "可檢視資料集內容",
   "permission.des.write": "可新增和變更資料集內容",
+  "preview_chunk": "分塊預覽",
+  "preview_chunk_empty": "無法讀取該文件內容",
+  "preview_chunk_intro": "最多展示 10 個分塊",
+  "preview_chunk_not_selected": "點擊左側文件後進行預覽",
   "rebuild_embedding_start_tip": "切換索引模型任務已開始",
   "rebuilding_index_count": "重建中索引數量：{{count}}",
   "request_headers": "請求頭",
@@ -72,8 +105,10 @@
   "tag.tags": "標籤",
   "tag.total_tags": "共 {{total}} 個標籤",
   "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "資料集有索引正在訓練或重建中",
+  "total_num_files": "共 {{total}} 個文件",
   "training_mode": "分段模式",
   "vector_model_max_tokens_tip": "每個分塊數據，最大長度為 3000 tokens",
+  "vllm_model": "圖片理解模型",
   "website_dataset": "網站同步",
   "website_dataset_desc": "網站同步功能讓您可以直接使用網頁連結建立資料集",
   "yuque_dataset": "語雀知識庫",