Add image index and pdf parse (#3956)

* feat: think tag parse

* feat: parse think tag test

* feat: pdf parse ux

* feat: doc2x parse

* perf: rewrite training mode setting

* feat: image parse queue

* perf: image index

* feat: image parse process

* feat: add init sh

* fix: ts
This commit is contained in:
Archer
2025-03-03 23:08:29 +08:00
committed by archer
parent 08b6f594df
commit adf5377ebe
106 changed files with 2337 additions and 1454 deletions

View File

@@ -2,6 +2,7 @@
"ai_model": "AI 模型",
"all": "所有",
"app_name": "应用名",
"auto_index": "索引增强",
"billing_module": "扣费模块",
"confirm_export": "共筛选出 {{total}} 条数据,是否确认导出?",
"current_filter_conditions": "当前筛选条件:",
@@ -9,6 +10,7 @@
"details": "详情",
"dingtalk": "钉钉",
"duration_seconds": "时长(秒)",
"embedding_index": "索引生成",
"every_day": "天",
"every_month": "月",
"every_week": "每周",
@@ -18,6 +20,7 @@
"export_title": "时间,成员,类型,项目名,AI 积分消耗",
"feishu": "飞书",
"generation_time": "生成时间",
"image_parse": "图片标注",
"input_token_length": "输入 tokens",
"member": "成员",
"member_name": "成员名",
@@ -27,8 +30,12 @@
"official_account": "公众号",
"order_number": "订单号",
"output_token_length": "输出 tokens",
"pages": "页数",
"pdf_enhanced_parse": "PDF 增强解析",
"pdf_parse": "PDF 解析",
"points": "积分",
"project_name": "项目名",
"qa": "问答对提取",
"select_member_and_source_first": "请先选中成员和类型",
"share": "分享链接",
"source": "来源",

View File

@@ -565,10 +565,7 @@
"core.dataset.file": "文件",
"core.dataset.folder": "目录",
"core.dataset.import.Auto mode Estimated Price Tips": "需调用文本理解模型需要消耗较多AI 积分:{{price}} 积分/1K tokens",
"core.dataset.import.Auto process": "自动",
"core.dataset.import.Auto process desc": "自动设置分割和预处理规则",
"core.dataset.import.Chunk Range": "范围:{{min}}~{{max}}",
"core.dataset.import.Chunk Split": "直接分段",
"core.dataset.import.Chunk Split Tip": "将文本按一定的规则进行分段处理后,转成可进行语义搜索的格式,适合绝大多数场景。不需要调用模型额外处理,成本低。",
"core.dataset.import.Continue upload": "继续上传",
"core.dataset.import.Custom process": "自定义规则",
@@ -578,7 +575,6 @@
"core.dataset.import.Custom split char Tips": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据,使用特定的分隔符来精确分块。",
"core.dataset.import.Custom text": "自定义文本",
"core.dataset.import.Custom text desc": "手动输入一段文本作为数据集",
"core.dataset.import.Data Preprocessing": "数据处理",
"core.dataset.import.Data process params": "数据处理参数",
"core.dataset.import.Down load csv template": "点击下载 CSV 模板",
"core.dataset.import.Embedding Estimated Price Tips": "仅使用索引模型,消耗少量 AI 积分:{{price}} 积分/1K tokens",
@@ -600,7 +596,6 @@
"core.dataset.import.Source name": "来源名",
"core.dataset.import.Sources list": "来源列表",
"core.dataset.import.Start upload": "开始上传",
"core.dataset.import.Total files": "共 {{total}} 个文件",
"core.dataset.import.Upload complete": "完成上传",
"core.dataset.import.Upload data": "确认上传",
"core.dataset.import.Upload file progress": "文件上传进度",
@@ -650,12 +645,12 @@
"core.dataset.test.test result placeholder": "测试结果将在这里展示",
"core.dataset.test.test result tip": "根据知识库内容与测试文本的相似度进行排序,你可以根据测试结果调整对应的文本。\n注意测试记录中的数据可能已经被修改过点击某条测试数据后将展示最新的数据。",
"core.dataset.training.Agent queue": "QA 训练排队",
"core.dataset.training.Auto mode": "增强处理",
"core.dataset.training.Auto mode": "补充索引",
"core.dataset.training.Auto mode Tip": "通过子索引以及调用模型生成相关问题与摘要,来增加数据块的语义丰富度,更利于检索。需要消耗更多的存储空间和增加 AI 调用次数。",
"core.dataset.training.Chunk mode": "直接分",
"core.dataset.training.Chunk mode": "直接分",
"core.dataset.training.Full": "预计 5 分钟以上",
"core.dataset.training.Leisure": "空闲",
"core.dataset.training.QA mode": "问答拆分",
"core.dataset.training.QA mode": "问答对提取",
"core.dataset.training.Vector queue": "索引排队",
"core.dataset.training.Waiting": "预计 5 分钟",
"core.dataset.training.Website Sync": "Web 站点同步",
@@ -864,7 +859,6 @@
"dataset.collections.Select Collection": "选择文件",
"dataset.collections.Select One Collection To Store": "选择一个文件进行存储",
"dataset.data.Can not edit": "无编辑权限",
"dataset.data.Custom Index Number": "自定义索引{{number}}",
"dataset.data.Default Index": "默认索引",
"dataset.data.Delete Tip": "确认删除该条数据?",
"dataset.data.Index Placeholder": "输入索引文本内容",
@@ -959,6 +953,7 @@
"new_create": "新建",
"no": "否",
"no_laf_env": "系统未配置Laf环境",
"not_model_config": "未配置相关模型",
"not_yet_introduced": "暂无介绍",
"option": "选项",
"pay.amount": "金额",
@@ -1124,7 +1119,6 @@
"support.wallet.invoice_detail": "发票详情",
"support.wallet.invoice_info": "发票将在 3-7 个工作日内发送至邮箱,请耐心等待",
"support.wallet.invoicing": "开票",
"support.wallet.moduleName.index": "索引生成",
"support.wallet.moduleName.qa": "QA 拆分",
"support.wallet.noBill": "无账单记录~",
"support.wallet.no_invoice": "暂无开票记录",

View File

@@ -3,11 +3,16 @@
"add_file": "添加文件",
"api_file": "API 文件库",
"api_url": "接口地址",
"auto_indexes": "自动生成补充索引",
"auto_indexes_tips": "通过大模型进行额外索引生成,提高语义丰富度,提高检索的精度。",
"chunk_max_tokens": "分块上限",
"close_auto_sync": "确认关闭自动同步功能?",
"collection.Create update time": "创建/更新时间",
"collection.Training type": "训练模式",
"collection.training_type": "处理模式",
"collection_data_count": "数据量",
"collection_metadata_custom_pdf_parse": "PDF增强解析",
"collection_metadata_image_parse": "图片标注",
"collection_not_support_retraining": "该集合类型不支持重新调整参数",
"collection_not_support_sync": "该集合不支持同步",
"collection_sync": "立即同步",
@@ -22,12 +27,21 @@
"custom_data_process_params_desc": "自定义设置数据处理规则",
"data.ideal_chunk_length": "理想分块长度",
"data_amount": "{{dataAmount}} 组数据, {{indexAmount}} 组索引",
"data_index_custom": "自定义索引",
"data_index_default": "默认索引",
"data_index_image": "图片索引",
"data_index_num": "索引 {{index}}",
"data_index_question": "推测问题索引",
"data_index_summary": "摘要索引",
"data_process_params": "处理参数",
"data_process_setting": "数据处理配置",
"dataset.Unsupported operation": "操作不支持",
"dataset.no_collections": "暂无数据集",
"dataset.no_tags": "暂无标签",
"default_params": "默认",
"default_params_desc": "使用系统默认的参数和规则",
"edit_dataset_config": "编辑知识库配置",
"enhanced_indexes": "索引增强",
"error.collectionNotFound": "集合找不到了~",
"external_file": "外部文件库",
"external_file_dataset_desc": "可以从外部文件库导入文件构建知识库,文件不会进行二次存储",
@@ -38,19 +52,38 @@
"feishu_dataset": "飞书知识库",
"feishu_dataset_config": "配置飞书知识库",
"feishu_dataset_desc": "可通过配置飞书文档权限,使用飞书文档构建知识库,文档不会进行二次存储",
"file_list": "文件列表",
"file_model_function_tip": "用于增强索引和 QA 生成",
"filename": "文件名",
"folder_dataset": "文件夹",
"ideal_chunk_length": "理想分块长度",
"ideal_chunk_length_tips": "按结束符号进行分段,并将多个分段组成一个分块,该值决定了分块的预估大小,如果会有上下浮动。",
"image_auto_parse": "图片自动索引",
"image_auto_parse_tips": "调用 VLM 自动标注文档里的图片,并生成额外的检索索引",
"import.Auto mode Estimated Price Tips": "需调用文本理解模型需要消耗较多AI 积分:{{price}} 积分/1K tokens",
"import.Embedding Estimated Price Tips": "仅使用索引模型,消耗少量 AI 积分:{{price}} 积分/1K tokens",
"import_confirm": "确认上传",
"import_data_preview": "数据预览",
"import_data_process_setting": "数据处理方式设置",
"import_file_parse_setting": "文件解析设置",
"import_model_config": "模型选择",
"import_param_setting": "参数设置",
"import_select_file": "选择文件",
"is_open_schedule": "启用定时同步",
"keep_image": "保留图片",
"move.hint": "移动后,所选知识库/文件夹将继承新文件夹的权限设置,原先的权限设置失效。",
"open_auto_sync": "开启定时同步后,系统将会每天不定时尝试同步集合,集合同步期间,会出现无法搜索到该集合数据现象。",
"params_setting": "参数设置",
"pdf_enhance_parse": "PDF增强解析",
"pdf_enhance_parse_price": "{{price}}积分/页",
"pdf_enhance_parse_tips": "解析 PDF 文件时,调用 PDF 识别模型进行识别,可以将其转换成 Markdown 并保留文档中的图片,同时也可以对扫描件进行识别。",
"permission.des.manage": "可管理整个知识库数据和信息",
"permission.des.read": "可查看知识库内容",
"permission.des.write": "可增加和变更知识库内容",
"preview_chunk": "分块预览",
"preview_chunk_empty": "无法读取该文件内容",
"preview_chunk_intro": "最多展示 10 个分块",
"preview_chunk_not_selected": "点击左侧文件后进行预览",
"rebuild_embedding_start_tip": "切换索引模型任务已开始",
"rebuilding_index_count": "重建中索引数量:{{count}}",
"request_headers": "请求头参数,会自动补充 Bearer",
@@ -72,8 +105,10 @@
"tag.tags": "标签",
"tag.total_tags": "共{{total}}个标签",
"the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引",
"total_num_files": "共 {{total}} 个文件",
"training_mode": "处理方式",
"vector_model_max_tokens_tip": "每个分块数据,最大长度为 3000 tokens",
"vllm_model": "图片理解模型",
"website_dataset": "Web 站点同步",
"website_dataset_desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
"yuque_dataset": "语雀知识库",