V4.9.11 feature (#4969)

* Feat: Images dataset collection (#4941) * New pic (#4858) * 更新数据集相关类型，添加图像文件ID和预览URL支持；优化数据集导入功能，新增图像数据集处理组件；修复部分国际化文本；更新文件上传逻辑以支持新功能。 * 与原先代码的差别 * 新增 V4.9.10 更新说明，支持 PG 设置`systemEnv.hnswMaxScanTuples`参数，优化 LLM stream 调用超时，修复全文检索多知识库排序问题。同时更新数据集索引，移除 datasetId 字段以简化查询。 * 更换成fileId_image逻辑，并增加训练队列匹配的逻辑 * 新增图片集合判断逻辑，优化预览URL生成流程，确保仅在数据集为图片集合时生成预览URL，并添加相关日志输出以便调试。 * Refactor Docker Compose configuration to comment out exposed ports for production environments, update image versions for pgvector, fastgpt, and mcp_server, and enhance Redis service with a health check. Additionally, standardize dataset collection labels in constants and improve internationalization strings across multiple languages. * Enhance TrainingStates component by adding internationalization support for the imageParse training mode and update defaultCounts to include imageParse mode in trainingDetail API. * Enhance dataset import context by adding additional steps for image dataset import process and improve internationalization strings for modal buttons in the useEditTitle hook. * Update DatasetImportContext to conditionally render MyStep component based on data source type, improving the import process for non-image datasets. * Refactor image dataset handling by improving internationalization strings, enhancing error messages, and streamlining the preview URL generation process. * 图片上传到新建的 dataset_collection_images 表，逻辑跟随更改 * 修改了除了controller的其他部分问题 * 把图片数据集的逻辑整合到controller里面 * 补充i18n * 补充i18n * resolve评论：主要是上传逻辑的更改和组件复用 * 图片名称的图标显示 * 修改编译报错的命名问题 * 删除不需要的collectionid部分 * 多余文件的处理和改动一个删除按钮 * 除了loading和统一的imageId,其他都resolve掉的 * 处理图标报错 * 复用了MyPhotoView并采用全部替换的方式将imageFileId变成imageId * 去除不必要文件修改 * 报错和字段修改 * 增加上传成功后删除临时文件的逻辑以及回退一些修改 * 删除path字段，将图片保存到gridfs内，并修改增删等操作的代码 * 修正编译错误 --------- Co-authored-by: archer <545436317@qq.com> * perf: image dataset * feat: insert image * perf: image icon * fix: training state --------- Co-authored-by: Zhuangzai fa <143257420+ctrlz526@users.noreply.github.com> * fix: ts (#4948) * Thirddatasetmd (#4942) * add thirddataset.md * fix thirddataset.md * fix * delete wrong png --------- Co-authored-by: dreamer6680 <146868355@qq.com> * perf: api dataset code * perf: log * add secondary.tsx (#4946) * add secondary.tsx * fix --------- Co-authored-by: dreamer6680 <146868355@qq.com> * perf: multiple menu * perf: i18n * feat: parse queue (#4960) * feat: parse queue * feat: sync parse queue * fix thirddataset.md (#4962) * fix thirddataset-4.png (#4963) * feat: Dataset template import (#4934) * 模版导入部分除了文档还没写 * 修复模版导入的 build 错误 * Document production * compress pictures * Change some constants to variables --------- Co-authored-by: Archer <545436317@qq.com> * perf: template import * doc * llm pargraph * bocha tool * fix: del collection --------- Co-authored-by: Zhuangzai fa <143257420+ctrlz526@users.noreply.github.com> Co-authored-by: dreamer6680 <1468683855@qq.com> Co-authored-by: dreamer6680 <146868355@qq.com>
2025-10-15 07:31:19 +00:00 · 2025-06-06 14:48:44 +08:00
parent bb810a43a1
commit c30f069f2f
198 changed files with 4934 additions and 2290 deletions
--- a/packages/web/i18n/en/account_team.json
+++ b/packages/web/i18n/en/account_team.json
@@ -6,6 +6,7 @@
  "accept": "accept",
  "action": "operate",
  "assign_permission": "Permission change",
+  "audit_log": "audit",
  "change_department_name": "Department Editor",
  "change_member_name": "Member name change",
  "change_member_name_self": "Change member name",
@@ -32,6 +33,13 @@
  "create_invoice": "Issuing invoices",
  "create_org": "Create organization",
  "create_sub_org": "Create sub-organization",
+  "dataset.api_file": "API Import",
+  "dataset.common_dataset": "Dataset",
+  "dataset.external_file": "External File",
+  "dataset.feishu_dataset": "Feishu Spreadsheet",
+  "dataset.folder_dataset": "Folder",
+  "dataset.website_dataset": "Website Sync",
+  "dataset.yuque_dataset": "Yuque Knowledge Base",
  "delete": "delete",
  "delete_api_key": "Delete the API key",
  "delete_app": "Delete the workbench application",
@@ -46,6 +54,7 @@
  "delete_from_team": "Move out of the team",
  "delete_group": "Delete a group",
  "delete_org": "Delete organization",
+  "department": "department",
  "edit_info": "Edit information",
  "edit_member": "Edit user",
  "edit_member_tip": "Name",
@@ -136,16 +145,12 @@
  "login": "Log in",
  "manage_member": "Managing members",
  "member": "member",
-  "department": "department",
-  "update": "update",
-  "save_and_publish": "save and publish",
  "member_group": "Belonging to member group",
  "move_app": "App location movement",
  "move_dataset": "Mobile Knowledge Base",
  "move_member": "Move member",
  "move_org": "Move organization",
  "notification_recieve": "Team notification reception",
-  "operation_log": "log",
  "org": "organization",
  "org_description": "Organization description",
  "org_name": "Organization name",
@@ -169,6 +174,7 @@
  "restore_tip_title": "Recovery confirmation",
  "retain_admin_permissions": "Keep administrator rights",
  "retrain_collection": "Retrain the set",
+  "save_and_publish": "save and publish",
  "search_log": "Search log",
  "search_member": "Search for members",
  "search_member_group_name": "Search member/group name",
@@ -190,14 +196,8 @@
  "type.Tool": "Tool",
  "type.Tool set": "Toolset",
  "type.Workflow bot": "Workflow",
-  "dataset.folder_dataset": "Folder",
-  "dataset.common_dataset": "Dataset",
-  "dataset.website_dataset": "Website Sync",
-  "dataset.external_file": "External File",
-  "dataset.api_file": "API Import",
-  "dataset.feishu_dataset": "Feishu Spreadsheet",
-  "dataset.yuque_dataset": "Yuque Knowledge Base",
  "unlimited": "Unlimited",
+  "update": "update",
  "update_api_key": "Update API key",
  "update_app_collaborator": "Apply permission changes",
  "update_app_info": "Application information modification",
@@ -213,4 +213,4 @@
  "user_team_leave_team": "Leave the team",
  "user_team_leave_team_failed": "Failure to leave the team",
  "waiting": "To be accepted"
-}
+}
--- a/packages/web/i18n/en/chat.json
+++ b/packages/web/i18n/en/chat.json
@@ -71,13 +71,13 @@
  "response_embedding_model_tokens": "Vector Model Tokens",
  "response_hybrid_weight": "Embedding : Full text = {{emb}} : {{text}}",
  "response_rerank_tokens": "Rearrange Model Tokens",
+  "search_results": "Search results",
  "select": "Select",
  "select_file": "Upload File",
  "select_file_img": "Upload file / image",
  "select_img": "Upload Image",
  "source_cronJob": "Scheduled execution",
  "stream_output": "Stream Output",
-  "to_dataset": "Go to the Knowledge Base",
  "unsupported_file_type": "Unsupported file types",
  "upload": "Upload",
  "variable_invisable_in_share": "Custom variables are not visible in login-free links",
--- a/packages/web/i18n/en/common.json
+++ b/packages/web/i18n/en/common.json
@@ -180,7 +180,7 @@
  "code_error.user_error.balance_not_enough": "Insufficient Account Balance",
  "code_error.user_error.bin_visitor_guest": "You Are Currently a Guest, Unauthorized to Operate",
  "code_error.user_error.un_auth_user": "User Not Found",
-  "comfirm_import": "comfirm_import",
+  "comfirm_import": "Confirm import",
  "comfirm_leave_page": "Confirm to Leave This Page?",
  "comfirn_create": "Confirm Creation",
  "commercial_function_tip": "Please Upgrade to the Commercial Version to Use This Feature: https://doc.fastgpt.cn/docs/commercial/intro/",
@@ -403,7 +403,6 @@
  "core.chat.response.module model": "Model",
  "core.chat.response.module name": "Model Name",
  "core.chat.response.module query": "Question/Search Term",
-  "core.chat.response.module quoteList": "Quote Content",
  "core.chat.response.module similarity": "Similarity",
  "core.chat.response.module temperature": "Temperature",
  "core.chat.response.module time": "Run Time",
@@ -424,7 +423,6 @@
  "core.dataset.Empty Dataset Tips": "No Dataset Yet, Create One Now!",
  "core.dataset.Folder placeholder": "This is a Directory",
  "core.dataset.Intro Placeholder": "This Dataset Has No Introduction Yet",
-  "core.dataset.Manual collection": "Manual Dataset",
  "core.dataset.My Dataset": "My Dataset",
  "core.dataset.Query extension intro": "Enabling the question optimization function can improve the accuracy of Dataset searches during continuous conversations. After enabling this function, when performing Dataset searches, the AI will complete the missing information of the question based on the conversation history.",
  "core.dataset.Quote Length": "Quote Content Length",
@@ -434,7 +432,6 @@
  "core.dataset.Text collection": "Text Dataset",
  "core.dataset.apiFile": "API File",
  "core.dataset.collection.Click top config website": "Click to Configure Website",
-  "core.dataset.collection.Collection name": "Dataset Name",
  "core.dataset.collection.Collection raw text": "Dataset Content",
  "core.dataset.collection.Empty Tip": "The Dataset is Empty",
  "core.dataset.collection.QA Prompt": "QA Split Prompt",
@@ -451,7 +448,6 @@
  "core.dataset.collection.metadata.metadata": "Metadata",
  "core.dataset.collection.metadata.read source": "View Original Content",
  "core.dataset.collection.metadata.source": "Data Source",
-  "core.dataset.collection.metadata.source name": "Source Name",
  "core.dataset.collection.metadata.source size": "Source Size",
  "core.dataset.collection.status.active": "Ready",
  "core.dataset.collection.status.error": "Error",
@@ -743,7 +739,7 @@
  "core.workflow.value": "Value",
  "core.workflow.variable": "Variable",
  "create": "Create",
-  "create_failed": "Creation Failed",
+  "create_failed": "Create failed",
  "create_success": "Created Successfully",
  "create_time": "Creation Time",
  "cron_job_run_app": "Scheduled Task",
@@ -788,7 +784,6 @@
  "dataset.dataset_name": "Dataset Name",
  "dataset.deleteFolderTips": "Confirm to Delete This Folder and All Its Contained Datasets? Data Cannot Be Recovered After Deletion, Please Confirm!",
  "dataset.test.noResult": "No Search Results",
-  "dataset_data_import_q_placeholder": "Up to {{maxToken}} words.",
  "dataset_data_input_a": "Answer",
  "dataset_data_input_chunk": "Chunk",
  "dataset_data_input_chunk_content": "Chunk",
@@ -802,7 +797,6 @@
  "delete_success": "Deleted Successfully",
  "delete_warning": "Deletion Warning",
  "embedding_model_not_config": "No index model is detected",
-  "error.Create failed": "Create failed",
  "error.code_error": "Verification code error",
  "error.fileNotFound": "File not found~",
  "error.inheritPermissionError": "Inherit permission Error",
@@ -1208,6 +1202,7 @@
  "templateTags.Writing": "Writing",
  "template_market": "Template Market",
  "textarea_variable_picker_tip": "Enter \"/\" to select a variable",
+  "to_dataset": "To dataset",
  "ui.textarea.Magnifying": "Magnifying",
  "un_used": "Unused",
  "unauth_token": "The certificate has expired, please log in again",
@@ -1306,4 +1301,4 @@
  "zoomin_tip_mac": "Zoom Out  ⌘  -",
  "zoomout_tip": "Zoom In  ctrl  +",
  "zoomout_tip_mac": "Zoom In  ⌘  +"
-}
+}
--- a/packages/web/i18n/en/dataset.json
+++ b/packages/web/i18n/en/dataset.json
@@ -8,12 +8,11 @@
  "auto_indexes_tips": "Additional index generation is performed through large models to improve semantic richness and improve retrieval accuracy.",
  "auto_training_queue": "Enhanced index queueing",
  "backup_collection": "Backup data",
-  "backup_data_parse": "Backup data is being parsed",
-  "backup_data_uploading": "Backup data is being uploaded: {{num}}%",
  "backup_dataset": "Backup import",
  "backup_dataset_success": "The backup was created successfully",
  "backup_dataset_tip": "You can reimport the downloaded csv file when exporting the knowledge base.",
  "backup_mode": "Backup import",
+  "backup_template_invalid": "The backup file format is incorrect, it should be the csv file with the first column as q,a,indexes",
  "chunk_max_tokens": "max_tokens",
  "chunk_process_params": "Block processing parameters",
  "chunk_size": "Block size",
@@ -28,16 +27,21 @@
  "collection.training_type": "Chunk type",
  "collection_data_count": "Data amount",
  "collection_metadata_custom_pdf_parse": "PDF enhancement analysis",
+  "collection_name": "Collection name",
  "collection_not_support_retraining": "This collection type does not support retuning parameters",
  "collection_not_support_sync": "This collection does not support synchronization",
  "collection_sync": "Sync data",
  "collection_sync_confirm_tip": "Confirm to start synchronizing data? \nThe system will pull the latest data for comparison. If the contents are different, a new collection will be created and the old collection will be deleted. Please confirm!",
  "collection_tags": "Collection Tags",
+  "common.dataset.data.Input Error Tip": "[Image Dataset] Process error:",
+  "common.error.unKnow": "Unknown error",
  "common_dataset": "General Dataset",
  "common_dataset_desc": "Building a knowledge base by importing files, web page links, or manual entry",
  "condition": "condition",
  "config_sync_schedule": "Configure scheduled synchronization",
+  "confirm_import_images": "Total {{num}} | Confirm create",
  "confirm_to_rebuild_embedding_tip": "Are you sure you want to switch the index for the Dataset?\nSwitching the index is a significant operation that requires re-indexing all data in your Dataset, which may take a long time. Please ensure your account has sufficient remaining points.\n\nAdditionally, you need to update the applications that use this Dataset to avoid conflicts with other indexed model Datasets.",
+  "core.dataset.Image collection": "Image dataset",
  "core.dataset.import.Adjust parameters": "Adjust parameters",
  "custom_data_process_params": "Custom",
  "custom_data_process_params_desc": "Customize data processing rules",
@@ -46,8 +50,10 @@
  "data_error_amount": "{{errorAmount}} Group training exception",
  "data_index_image": "Image index",
  "data_index_num": "Index {{index}}",
+  "data_parsing": "Data analysis",
  "data_process_params": "Params",
  "data_process_setting": "Processing config",
+  "data_uploading": "Data is being uploaded: {{num}}%",
  "dataset.Chunk_Number": "Block number",
  "dataset.Completed": "Finish",
  "dataset.Delete_Chunk": "delete",
@@ -67,7 +73,9 @@
  "dataset.no_tags": "No tags available",
  "default_params": "default",
  "default_params_desc": "Use system default parameters and rules",
+  "download_csv_template": "Click to download the CSV template",
  "edit_dataset_config": "Edit knowledge base configuration",
+  "empty_collection": "Blank dataset",
  "enhanced_indexes": "Index enhancement",
  "error.collectionNotFound": "Collection not found~",
  "external_file": "External File Library",
@@ -90,6 +98,7 @@
  "image_auto_parse": "Automatic image indexing",
  "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
  "image_training_queue": "Queue of image processing",
+  "images_creating": "Creating",
  "immediate_sync": "Immediate Synchronization",
  "import.Auto mode Estimated Price Tips": "The text understanding model needs to be called, which requires more points: {{price}} points/1K tokens",
  "import.Embedding Estimated Price Tips": "Only use the index model and consume a small amount of AI points: {{price}} points/1K tokens",
@@ -104,6 +113,8 @@
  "index_size": "Index size",
  "index_size_tips": "When vectorized, the system will automatically further segment the blocks according to this size.",
  "input_required_field_to_select_baseurl": "Please enter the required information first",
+  "insert_images": "Added pictures",
+  "insert_images_success": "The new picture is successfully added, and you need to wait for the training to be completed before it will be displayed.",
  "is_open_schedule": "Enable scheduled synchronization",
  "keep_image": "Keep the picture",
  "loading": "Loading...",
@@ -135,6 +146,7 @@
  "process.Image_Index": "Image index generation",
  "process.Is_Ready": "Ready",
  "process.Is_Ready_Count": "{{count}} Group is ready",
+  "process.Parse_Image": "Image analysis",
  "process.Parsing": "Parsing",
  "process.Vectorizing": "Index vectorization",
  "process.Waiting": "Queue",
@@ -174,18 +186,28 @@
  "tag.searchOrAddTag": "Search or Add Tag",
  "tag.tags": "Tags",
  "tag.total_tags": "Total {{total}} tags",
+  "template_dataset": "Template import",
+  "template_file_invalid": "The template file format is incorrect, it should be the csv file with the first column as q,a,indexes",
+  "template_mode": "Template import",
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "The Dataset has indexes that are being trained or rebuilt",
  "total_num_files": "Total {{total}} files",
  "training.Error": "{{count}} Group exception",
  "training.Normal": "Normal",
  "training_mode": "Chunk mode",
+  "training_queue_tip": "Training queue status",
  "training_ready": "{{count}} Group",
+  "upload_by_template_format": "Upload by template file",
+  "uploading_progress": "Uploading: {{num}}%",
  "vector_model_max_tokens_tip": "Each chunk of data has a maximum length of 3000 tokens",
+  "vector_training_queue": "Vector training queue",
  "vllm_model": "Image understanding model",
+  "vlm_model_required_tooltip": "A Vision Language Model is required to create image collections",
+  "vlm_model_required_warning": "Image datasets require a Vision Language Model (VLM) to be configured. Please add a model that supports image understanding in the model configuration first.",
+  "waiting_for_training": "Waiting for training",
  "website_dataset": "Website Sync",
  "website_dataset_desc": "Build knowledge base by crawling web page data in batches",
  "website_info": "Website Information",
-  "yuque_dataset": "Yuque Dataset",
-  "yuque_dataset_config": "Yuque Dataset Config",
-  "yuque_dataset_desc": "Can build a dataset using Yuque documents by configuring permissions, without secondary storage"
+  "yuque_dataset": "Yuque Knowledge Base",
+  "yuque_dataset_config": "Configure Yuque Knowledge Base",
+  "yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice"
 }
--- a/packages/web/i18n/en/file.json
+++ b/packages/web/i18n/en/file.json
@@ -1,9 +1,32 @@
 {
+  "Action": "Please select the image to upload",
+  "All images import failed": "All pictures failed to import",
+  "Dataset_ID_not_found": "The dataset ID does not exist",
+  "Failed_to_get_token": "Failed to obtain the token",
+  "Image_ID_copied": "Copy ID",
+  "Image_Preview": "Picture preview",
+  "Image_dataset_requires_VLM_model_to_be_configured": "The image dataset needs to be configured with the image understanding model (VLM) to be used. Please add a model that supports image understanding in the model configuration first.",
+  "Image_does_not_belong_to_current_team": "The picture does not belong to the current team",
+  "Image_file_does_not_exist": "The picture does not exist",
+  "Loading_image": "Loading the picture...",
+  "Loading_image failed": "Preview loading failed",
+  "Only_support_uploading_one_image": "Only support uploading one image",
+  "Please select the image to upload": "Please select the image to upload",
+  "Please select the image to upload select the image to upload": "",
+  "Please wait for all files to upload": "Please wait for all files to be uploaded to complete",
  "bucket_chat": "Conversation Files",
  "bucket_file": "Dataset Documents",
  "click_to_view_raw_source": "Click to View Original Source",
+  "common.dataset_data_input_image_support_format": "Support .jpg, .jpeg, .png, .gif, .webp formats",
+  "delete_image": "Delete pictures",
  "file_name": "Filename",
  "file_size": "Filesize",
+  "image": "picture",
+  "image_collection": "Picture collection",
+  "image_description": "Image description",
+  "image_description_tip": "Please enter the description of the picture",
+  "please_upload_image_first": "Please upload the picture first",
+  "reached_max_file_count": "Maximum file count reached",
  "release_the_mouse_to_upload_the_file": "Release Mouse to Upload File",
  "select_and_drag_file_tip": "Click or Drag Files Here to Upload",
  "select_file_amount_limit": "You can select up to {{max}} files",
@@ -12,7 +35,11 @@
  "support_file_type": "Supports {{fileType}} file types",
  "support_max_count": "Supports up to {{maxCount}} files",
  "support_max_size": "Maximum file size is {{maxSize}}",
+  "template_csv_file_select_tip": "Only support {{fileType}} files that are strictly in accordance with template format</highlight>",
+  "template_strict_highlight": "Strictly follow the template",
+  "total_files": "Total {{selectFiles.length}} files",
+  "upload_error_description": "Only multiple files or a single folder can be uploaded at a time",
  "upload_failed": "Upload Failed",
-  "reached_max_file_count": "Maximum file count reached",
-  "upload_error_description": "Only multiple files or a single folder can be uploaded at a time"
-}
+  "upload_file_error": "Please upload pictures",
+  "uploading": "Uploading..."
+}