From 88bd3aaa9e0c05bec56a96205e927582f08e22f2 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Thu, 22 May 2025 15:53:51 +0800 Subject: [PATCH] perf: backup import (#4866) * i18n * remove invalid code * perf: backup import * backup tip * fix: indexsize invalid --- .vscode/settings.json | 2 +- .../zh-cn/docs/development/upgrading/4910.md | 4 +- packages/global/common/error/code/dataset.ts | 2 +- packages/global/core/dataset/api.d.ts | 1 + packages/global/core/dataset/constants.ts | 7 +- .../global/core/dataset/training/utils.ts | 2 +- packages/global/core/dataset/type.d.ts | 1 + .../service/common/file/gridfs/controller.ts | 8 +- packages/service/common/file/read/utils.ts | 22 +- .../service/core/dataset/apiDataset/api.ts | 3 +- .../core/dataset/collection/controller.ts | 11 +- packages/service/core/dataset/read.ts | 53 +++-- .../core/dataset/training/constants.ts | 3 +- .../service/core/dataset/training/utils.ts | 16 -- .../core/workflow/dispatch/agent/extract.ts | 29 +-- .../core/workflow/dispatch/tools/readFiles.ts | 4 +- .../web/components/common/Icon/constants.ts | 1 + .../components/common/Icon/icons/backup.svg | 4 + .../web/components/common/LightTip/index.tsx | 4 +- packages/web/i18n/en/account.json | 1 + packages/web/i18n/en/account_info.json | 7 +- packages/web/i18n/en/account_team.json | 18 +- packages/web/i18n/en/account_thirdParty.json | 4 + packages/web/i18n/en/account_usage.json | 2 + packages/web/i18n/en/app.json | 9 + packages/web/i18n/en/common.json | 9 +- packages/web/i18n/en/dataset.json | 9 + packages/web/i18n/en/login.json | 3 +- packages/web/i18n/en/workflow.json | 1 + packages/web/i18n/zh-CN/common.json | 3 +- packages/web/i18n/zh-CN/dataset.json | 7 + packages/web/i18n/zh-CN/user.json | 7 +- packages/web/i18n/zh-Hant/account.json | 2 + packages/web/i18n/zh-Hant/account_info.json | 7 +- packages/web/i18n/zh-Hant/account_model.json | 2 + packages/web/i18n/zh-Hant/account_team.json | 18 +- .../web/i18n/zh-Hant/account_thirdParty.json | 6 +- packages/web/i18n/zh-Hant/account_usage.json | 2 + packages/web/i18n/zh-Hant/app.json | 9 + packages/web/i18n/zh-Hant/chat.json | 5 +- packages/web/i18n/zh-Hant/common.json | 12 +- packages/web/i18n/zh-Hant/dataset.json | 7 + packages/web/i18n/zh-Hant/login.json | 3 +- packages/web/i18n/zh-Hant/workflow.json | 1 + .../CollectionCard/BackupImportModal.tsx | 94 ++++++++ .../dataset/detail/CollectionCard/Header.tsx | 46 ++-- .../dataset/detail/CollectionCard/index.tsx | 18 +- .../dataset/detail/DataCard.tsx | 5 +- .../detail/Form/CollectionChunkForm.tsx | 20 +- .../dataset/detail/Import/Context.tsx | 16 +- .../Import/commonProgress/PreviewData.tsx | 1 - .../detail/Import/commonProgress/Upload.tsx | 6 - .../detail/Import/diffSource/TableLocal.tsx | 101 -------- .../dataset/detail/Import/index.tsx | 2 - .../dataset/detail/MetaDataCard.tsx | 24 +- .../detail/components/FileSelector.tsx | 218 ++++++++++++++++++ .../core/dataset/collection/create/backup.ts | 86 +++++++ .../dataset/collection/create/csvTable.ts | 61 ----- .../core/dataset/collection/create/link.ts | 6 +- .../pages/api/core/dataset/data/pushData.ts | 7 +- .../src/pages/api/core/dataset/exportAll.ts | 28 ++- .../api/core/dataset/file/getPreviewChunks.ts | 15 +- projects/app/src/pages/login/index.tsx | 2 - .../service/common/system/volumnMongoWatch.ts | 6 +- .../service/core/dataset/data/controller.ts | 12 +- .../app/src/service/events/generateVector.ts | 1 + projects/app/src/web/core/dataset/api.ts | 33 ++- 67 files changed, 751 insertions(+), 388 deletions(-) delete mode 100644 packages/service/core/dataset/training/utils.ts create mode 100644 packages/web/components/common/Icon/icons/backup.svg create mode 100644 projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx delete mode 100644 projects/app/src/pageComponents/dataset/detail/Import/diffSource/TableLocal.tsx create mode 100644 projects/app/src/pageComponents/dataset/detail/components/FileSelector.tsx create mode 100644 projects/app/src/pages/api/core/dataset/collection/create/backup.ts delete mode 100644 projects/app/src/pages/api/core/dataset/collection/create/csvTable.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index bba595f12..82794cc44 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,7 +21,7 @@ "i18n-ally.namespace": true, "i18n-ally.pathMatcher": "{locale}/{namespaces}.json", "i18n-ally.extract.targetPickingStrategy": "most-similar-by-key", - "i18n-ally.translate.engines": ["google"], + "i18n-ally.translate.engines": ["deepl","google"], "[typescript]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, diff --git a/docSite/content/zh-cn/docs/development/upgrading/4910.md b/docSite/content/zh-cn/docs/development/upgrading/4910.md index 902088de3..c5122e77b 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4910.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4910.md @@ -16,9 +16,11 @@ weight: 790 1. LLM stream调用，默认超时调大。 2. 部分确认交互优化。 +3. 纠正原先知识库的“表格数据集”名称，改成“备份导入”。同时支持知识库索引的导出和导入。 ## 🐛 修复 1. 全文检索多知识库时排序得分排序不正确。 2. 流响应捕获 finish_reason 可能不正确。 -3. 工具调用模式，未保存思考输出。 \ No newline at end of file +3. 工具调用模式，未保存思考输出。 +4. 知识库 indexSize 参数未生效。 \ No newline at end of file diff --git a/packages/global/common/error/code/dataset.ts b/packages/global/common/error/code/dataset.ts index 6b1180c96..0798bc450 100644 --- a/packages/global/common/error/code/dataset.ts +++ b/packages/global/common/error/code/dataset.ts @@ -27,7 +27,7 @@ const datasetErr = [ }, { statusText: DatasetErrEnum.unExist, - message: 'core.dataset.error.unExistDataset' + message: i18nT('common:core.dataset.error.unExistDataset') }, { statusText: DatasetErrEnum.unExistCollection, diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index a0d2721f7..a4df84df9 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -147,6 +147,7 @@ export type PushDatasetDataProps = { collectionId: string; data: PushDatasetDataChunkProps[]; trainingType?: DatasetCollectionDataProcessModeEnum; + indexSize?: number; autoIndexes?: boolean; imageIndex?: boolean; prompt?: string; diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 1f50aadf5..1afed622e 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -120,6 +120,8 @@ export const DatasetCollectionSyncResultMap = { export enum DatasetCollectionDataProcessModeEnum { chunk = 'chunk', qa = 'qa', + backup = 'backup', + auto = 'auto' // abandon } export const DatasetCollectionDataProcessModeMap = { @@ -131,6 +133,10 @@ export const DatasetCollectionDataProcessModeMap = { label: i18nT('common:core.dataset.training.QA mode'), tooltip: i18nT('common:core.dataset.import.QA Import Tip') }, + [DatasetCollectionDataProcessModeEnum.backup]: { + label: i18nT('dataset:backup_mode'), + tooltip: i18nT('dataset:backup_mode') + }, [DatasetCollectionDataProcessModeEnum.auto]: { label: i18nT('common:core.dataset.training.Auto mode'), tooltip: i18nT('common:core.dataset.training.Auto mode Tip') @@ -154,7 +160,6 @@ export enum ImportDataSourceEnum { fileLocal = 'fileLocal', fileLink = 'fileLink', fileCustom = 'fileCustom', - csvTable = 'csvTable', externalFile = 'externalFile', apiDataset = 'apiDataset', reTraining = 'reTraining' diff --git a/packages/global/core/dataset/training/utils.ts b/packages/global/core/dataset/training/utils.ts index 633ee9982..faf725e3b 100644 --- a/packages/global/core/dataset/training/utils.ts +++ b/packages/global/core/dataset/training/utils.ts @@ -118,7 +118,7 @@ export const computeChunkSize = (params: { return getLLMMaxChunkSize(params.llmModel); } - return Math.min(params.chunkSize || chunkAutoChunkSize, getLLMMaxChunkSize(params.llmModel)); + return Math.min(params.chunkSize ?? chunkAutoChunkSize, getLLMMaxChunkSize(params.llmModel)); }; export const computeChunkSplitter = (params: { diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index a76a39b26..9463c239a 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -175,6 +175,7 @@ export type DatasetTrainingSchemaType = { q: string; a: string; chunkIndex: number; + indexSize?: number; weight: number; indexes: Omit[]; retryCount: number; diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index 07f33b795..c1d183e68 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -210,15 +210,15 @@ export const readFileContentFromMongo = async ({ tmbId, bucketName, fileId, - isQAImport = false, - customPdfParse = false + customPdfParse = false, + getFormatText }: { teamId: string; tmbId: string; bucketName: `${BucketNameEnum}`; fileId: string; - isQAImport?: boolean; customPdfParse?: boolean; + getFormatText?: boolean; // 数据类型都尽可能转化成 markdown 格式 }): Promise<{ rawText: string; filename: string; @@ -254,8 +254,8 @@ export const readFileContentFromMongo = async ({ // Get raw text const { rawText } = await readRawContentByFileBuffer({ customPdfParse, + getFormatText, extension, - isQAImport, teamId, tmbId, buffer: fileBuffers, diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index 3b263879a..6a5ce5f86 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -16,6 +16,7 @@ export type readRawTextByLocalFileParams = { path: string; encoding: string; customPdfParse?: boolean; + getFormatText?: boolean; metadata?: Record; }; export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => { @@ -27,8 +28,8 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam return readRawContentByFileBuffer({ extension, - isQAImport: false, customPdfParse: params.customPdfParse, + getFormatText: params.getFormatText, teamId: params.teamId, tmbId: params.tmbId, encoding: params.encoding, @@ -46,7 +47,7 @@ export const readRawContentByFileBuffer = async ({ encoding, metadata, customPdfParse = false, - isQAImport = false + getFormatText = true }: { teamId: string; tmbId: string; @@ -57,8 +58,10 @@ export const readRawContentByFileBuffer = async ({ metadata?: Record; customPdfParse?: boolean; - isQAImport: boolean; -}): Promise => { + getFormatText?: boolean; +}): Promise<{ + rawText: string; +}> => { const systemParse = () => runWorker(WorkerNameEnum.readFile, { extension, @@ -176,16 +179,7 @@ export const readRawContentByFileBuffer = async ({ }); } - if (['csv', 'xlsx'].includes(extension)) { - // qa data - if (isQAImport) { - rawText = rawText || ''; - } else { - rawText = formatText || rawText; - } - } - addLog.debug(`Upload file success, time: ${Date.now() - start}ms`); - return { rawText, formatText, imageList }; + return { rawText: getFormatText ? formatText || rawText : rawText }; }; diff --git a/packages/service/core/dataset/apiDataset/api.ts b/packages/service/core/dataset/apiDataset/api.ts index 86625ae4e..e9441db43 100644 --- a/packages/service/core/dataset/apiDataset/api.ts +++ b/packages/service/core/dataset/apiDataset/api.ts @@ -146,7 +146,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer } tmbId, url: previewUrl, relatedId: apiFileId, - customPdfParse + customPdfParse, + getFormatText: true }); return { title, diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index c45b83254..46ad1b8d1 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -36,13 +36,14 @@ import { computeChunkSplitter, getLLMMaxChunkSize } from '@fastgpt/global/core/dataset/training/utils'; +import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants'; export const createCollectionAndInsertData = async ({ dataset, rawText, relatedId, createCollectionParams, - isQAImport = false, + backupParse = false, billId, session }: { @@ -50,8 +51,8 @@ export const createCollectionAndInsertData = async ({ rawText: string; relatedId?: string; createCollectionParams: CreateOneCollectionParams; + backupParse?: boolean; - isQAImport?: boolean; billId?: string; session?: ClientSession; }) => { @@ -81,7 +82,7 @@ export const createCollectionAndInsertData = async ({ maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)), overlapRatio: trainingType === DatasetCollectionDataProcessModeEnum.chunk ? 0.2 : 0, customReg: chunkSplitter ? [chunkSplitter] : [], - isQAImport + backupParse }); // 2. auth limit @@ -157,6 +158,10 @@ export const createCollectionAndInsertData = async ({ billId: traingBillId, data: chunks.map((item, index) => ({ ...item, + indexes: item.indexes?.map((text) => ({ + type: DatasetDataIndexTypeEnum.custom, + text + })), chunkIndex: index })), session diff --git a/packages/service/core/dataset/read.ts b/packages/service/core/dataset/read.ts index d1df601b6..6c88fee98 100644 --- a/packages/service/core/dataset/read.ts +++ b/packages/service/core/dataset/read.ts @@ -2,7 +2,6 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { readFileContentFromMongo } from '../../common/file/gridfs/controller'; import { urlsFetch } from '../../common/string/cheerio'; -import { parseCsvTable2Chunks } from './training/utils'; import { type TextSplitProps, splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import axios from 'axios'; import { readRawContentByFileBuffer } from '../../common/file/read/utils'; @@ -13,18 +12,21 @@ import { type YuqueServer } from '@fastgpt/global/core/dataset/apiDataset'; import { useApiDatasetRequest } from './apiDataset/api'; +import Papa from 'papaparse'; export const readFileRawTextByUrl = async ({ teamId, tmbId, url, customPdfParse, + getFormatText, relatedId }: { teamId: string; tmbId: string; url: string; customPdfParse?: boolean; + getFormatText?: boolean; relatedId: string; // externalFileId / apiFileId }) => { const response = await axios({ @@ -38,7 +40,7 @@ export const readFileRawTextByUrl = async ({ const { rawText } = await readRawContentByFileBuffer({ customPdfParse, - isQAImport: false, + getFormatText, extension, teamId, tmbId, @@ -62,21 +64,21 @@ export const readDatasetSourceRawText = async ({ tmbId, type, sourceId, - isQAImport, selector, externalFileId, apiServer, feishuServer, yuqueServer, - customPdfParse + customPdfParse, + getFormatText }: { teamId: string; tmbId: string; type: DatasetSourceReadTypeEnum; sourceId: string; customPdfParse?: boolean; + getFormatText?: boolean; - isQAImport?: boolean; // csv data selector?: string; // link selector externalFileId?: string; // external file dataset apiServer?: APIFileServer; // api dataset @@ -92,8 +94,8 @@ export const readDatasetSourceRawText = async ({ tmbId, bucketName: BucketNameEnum.dataset, fileId: sourceId, - isQAImport, - customPdfParse + customPdfParse, + getFormatText }); return { title: filename, @@ -183,16 +185,38 @@ export const readApiServerFileContent = async ({ export const rawText2Chunks = ({ rawText, - isQAImport, + backupParse, chunkSize = 512, ...splitProps }: { rawText: string; - isQAImport?: boolean; -} & TextSplitProps) => { - if (isQAImport) { - const { chunks } = parseCsvTable2Chunks(rawText); - return chunks; + backupParse?: boolean; + tableParse?: boolean; +} & TextSplitProps): { + q: string; + a: string; + indexes?: string[]; +}[] => { + const parseDatasetBackup2Chunks = (rawText: string) => { + const csvArr = Papa.parse(rawText).data as string[][]; + console.log(rawText, csvArr); + + const chunks = csvArr + .slice(1) + .map((item) => ({ + q: item[0] || '', + a: item[1] || '', + indexes: item.slice(2) + })) + .filter((item) => item.q || item.a); + + return { + chunks + }; + }; + + if (backupParse) { + return parseDatasetBackup2Chunks(rawText).chunks; } const { chunks } = splitText2Chunks({ @@ -203,6 +227,7 @@ export const rawText2Chunks = ({ return chunks.map((item) => ({ q: item, - a: '' + a: '', + indexes: [] })); }; diff --git a/packages/service/core/dataset/training/constants.ts b/packages/service/core/dataset/training/constants.ts index fb5b6be07..732a62c1a 100644 --- a/packages/service/core/dataset/training/constants.ts +++ b/packages/service/core/dataset/training/constants.ts @@ -1,6 +1,5 @@ export enum ImportDataSourceEnum { fileLocal = 'fileLocal', fileLink = 'fileLink', - fileCustom = 'fileCustom', - tableLocal = 'tableLocal' + fileCustom = 'fileCustom' } diff --git a/packages/service/core/dataset/training/utils.ts b/packages/service/core/dataset/training/utils.ts deleted file mode 100644 index 484eac0b6..000000000 --- a/packages/service/core/dataset/training/utils.ts +++ /dev/null @@ -1,16 +0,0 @@ -import Papa from 'papaparse'; - -export const parseCsvTable2Chunks = (rawText: string) => { - const csvArr = Papa.parse(rawText).data as string[][]; - - const chunks = csvArr - .map((item) => ({ - q: item[0] || '', - a: item[1] || '' - })) - .filter((item) => item.q || item.a); - - return { - chunks - }; -}; diff --git a/packages/service/core/workflow/dispatch/agent/extract.ts b/packages/service/core/workflow/dispatch/agent/extract.ts index 69269266f..6f437adb4 100644 --- a/packages/service/core/workflow/dispatch/agent/extract.ts +++ b/packages/service/core/workflow/dispatch/agent/extract.ts @@ -223,28 +223,29 @@ const toolChoice = async (props: ActionProps) => { } ]; + const body = llmCompletionsBodyFormat( + { + stream: true, + model: extractModel.model, + temperature: 0.01, + messages: filterMessages, + tools, + tool_choice: { type: 'function', function: { name: agentFunName } } + }, + extractModel + ); const { response } = await createChatCompletion({ - body: llmCompletionsBodyFormat( - { - stream: true, - model: extractModel.model, - temperature: 0.01, - messages: filterMessages, - tools, - tool_choice: { type: 'function', function: { name: agentFunName } } - }, - extractModel - ), + body, userKey: externalProvider.openaiAccount }); - const { toolCalls, usage } = await formatLLMResponse(response); + const { text, toolCalls, usage } = await formatLLMResponse(response); const arg: Record = (() => { try { return json5.parse(toolCalls?.[0]?.function?.arguments || ''); } catch (error) { - console.log(agentFunction.parameters); - console.log(toolCalls?.[0]?.function); + console.log('body', body); + console.log('AI response', text, toolCalls?.[0]?.function); console.log('Your model may not support tool_call', error); return {}; } diff --git a/packages/service/core/workflow/dispatch/tools/readFiles.ts b/packages/service/core/workflow/dispatch/tools/readFiles.ts index 44baa6dc9..f8a80efa7 100644 --- a/packages/service/core/workflow/dispatch/tools/readFiles.ts +++ b/packages/service/core/workflow/dispatch/tools/readFiles.ts @@ -211,12 +211,12 @@ export const getFileContentFromLinks = async ({ // Read file const { rawText } = await readRawContentByFileBuffer({ extension, - isQAImport: false, teamId, tmbId, buffer, encoding, - customPdfParse + customPdfParse, + getFormatText: true }); // Add to buffer diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 0c8b75715..3280100f1 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -2,6 +2,7 @@ export const iconPaths = { alignLeft: () => import('./icons/alignLeft.svg'), + backup: () => import('./icons/backup.svg'), book: () => import('./icons/book.svg'), change: () => import('./icons/change.svg'), chatSend: () => import('./icons/chatSend.svg'), diff --git a/packages/web/components/common/Icon/icons/backup.svg b/packages/web/components/common/Icon/icons/backup.svg new file mode 100644 index 000000000..71ca6a826 --- /dev/null +++ b/packages/web/components/common/Icon/icons/backup.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/packages/web/components/common/LightTip/index.tsx b/packages/web/components/common/LightTip/index.tsx index 6d211fd35..edfedb3d6 100644 --- a/packages/web/components/common/LightTip/index.tsx +++ b/packages/web/components/common/LightTip/index.tsx @@ -3,8 +3,10 @@ import { Box, HStack, Icon, type StackProps } from '@chakra-ui/react'; const LightTip = ({ text, + icon = 'common/info', ...props }: { + icon?: string; text: string; } & StackProps) => { return ( @@ -17,7 +19,7 @@ const LightTip = ({ fontSize={'sm'} {...props} > - + {text} ); diff --git a/packages/web/i18n/en/account.json b/packages/web/i18n/en/account.json index 4d6afa51b..b57baec0a 100644 --- a/packages/web/i18n/en/account.json +++ b/packages/web/i18n/en/account.json @@ -66,6 +66,7 @@ "model.tool_choice_tip": "If the model supports tool calling, turn on this switch", "model.used_in_classify": "Used for problem classification", "model.used_in_extract_fields": "for text extraction", + "model.used_in_query_extension": "For problem optimization", "model.used_in_tool_call": "Used for tool call nodes", "model.vision": "Vision model", "model.vision_tag": "Vision", diff --git a/packages/web/i18n/en/account_info.json b/packages/web/i18n/en/account_info.json index 9bf43d09e..a1e3a83dc 100644 --- a/packages/web/i18n/en/account_info.json +++ b/packages/web/i18n/en/account_info.json @@ -39,8 +39,6 @@ "new_password": "New Password", "notification_receiving": "Notify", "old_password": "Old Password", - "openai_account_configuration": "OpenAI account configuration", - "openai_account_setting_exception": "Setting OpenAI account exception", "package_and_usage": "Plans", "package_details": "Details", "package_expiry_time": "Expired", @@ -52,8 +50,10 @@ "password_update_success": "Password changed successfully", "pending_usage": "To be used", "phone_label": "Phone number", + "please_bind_contact": "Please bind the contact information", "please_bind_notification_receiving_path": "Please bind the notification receiving method first", "purchase_extra_package": "Upgrade", + "redeem_coupon": "Redeem coupon", "reminder_create_bound_notification_account": "Remind the creator to bind the notification account", "reset_password": "reset password", "resource_usage": "Usages", @@ -75,6 +75,5 @@ "user_team_team_name": "Team", "verification_code": "Verification code", "you_can_convert": "you can redeem", - "yuan": "Yuan", - "redeem_coupon": "Redeem coupon" + "yuan": "Yuan" } diff --git a/packages/web/i18n/en/account_team.json b/packages/web/i18n/en/account_team.json index c58ded16f..9cf79d45b 100644 --- a/packages/web/i18n/en/account_team.json +++ b/packages/web/i18n/en/account_team.json @@ -8,8 +8,9 @@ "assign_permission": "Permission change", "change_department_name": "Department Editor", "change_member_name": "Member name change", + "confirm_delete_from_org": "Confirm to move {{username}} out of the department?", + "confirm_delete_from_team": "Confirm to move {{username}} out of the team?", "confirm_delete_group": "Confirm to delete group?", - "confirm_delete_member": "Confirm to delete member?", "confirm_delete_org": "Confirm to delete organization?", "confirm_forbidden": "Confirm forbidden", "confirm_leave_team": "Confirmed to leave the team? \nAfter exiting, all your resources in the team are transferred to the team owner.", @@ -21,6 +22,8 @@ "create_sub_org": "Create sub-organization", "delete": "delete", "delete_department": "Delete sub-department", + "delete_from_org": "Move out of department", + "delete_from_team": "Move out of the team", "delete_group": "Delete a group", "delete_org": "Delete organization", "edit_info": "Edit information", @@ -28,6 +31,7 @@ "edit_member_tip": "Name", "edit_org_info": "Edit organization information", "expires": "Expiration time", + "export_members": "Export members", "forbid_hint": "After forbidden, this invitation link will become invalid. This action is irreversible. Are you sure you want to deactivate?", "forbid_success": "Forbid success", "forbidden": "Forbidden", @@ -44,8 +48,10 @@ "invite_member": "Invite members", "invited": "Invited", "join_team": "Join the team", + "join_update_time": "Join/Update Time", "kick_out_team": "Remove members", "label_sync": "Tag sync", + "leave": "Resigned", "leave_team_failed": "Leaving the team exception", "log_assign_permission": "[{{name}}] Updated the permissions of [{{objectName}}]: [Application creation: [{{appCreate}}], Knowledge Base: [{{datasetCreate}}], API Key: [{{apiKeyCreate}}], Management: [{{manage}}]]", "log_change_department": "【{{name}}】Updated department【{{departmentName}}】", @@ -70,6 +76,7 @@ "member_group": "Belonging to member group", "move_member": "Move member", "move_org": "Move organization", + "notification_recieve": "Team notification reception", "operation_log": "log", "org": "organization", "org_description": "Organization description", @@ -84,13 +91,22 @@ "permission_datasetCreate_Tip": "Can create knowledge bases in the root directory (creation permissions in folders are controlled by the folder)", "permission_manage": "Admin", "permission_manage_tip": "Can manage members, create groups, manage all groups, and assign permissions to groups and members", + "please_bind_contact": "Please bind the contact information", "recover_team_member": "Member Recovery", "relocate_department": "Department Mobile", "remark": "remark", "remove_tip": "Confirm to remove {{username}} from the team?", + "restore_tip": "Confirm to join the team {{username}}? \nOnly the availability and related permissions of this member account are restored, and the resources under the account cannot be restored.", + "restore_tip_title": "Recovery confirmation", "retain_admin_permissions": "Keep administrator rights", "search_log": "Search log", + "search_member": "Search for members", "search_member_group_name": "Search member/group name", + "search_org": "Search Department", + "set_name_avatar": "Team avatar", + "sync_immediately": "Synchronize now", + "sync_member_failed": "Synchronization of members failed", + "sync_member_success": "Synchronize members successfully", "total_team_members": "{{amount}} members in total", "transfer_ownership": "transfer owner", "unlimited": "Unlimited", diff --git a/packages/web/i18n/en/account_thirdParty.json b/packages/web/i18n/en/account_thirdParty.json index 7b3424cc5..a0b7dc46d 100644 --- a/packages/web/i18n/en/account_thirdParty.json +++ b/packages/web/i18n/en/account_thirdParty.json @@ -1,13 +1,17 @@ { "configured": "Configured", "error.no_permission": "Please contact the administrator to configure", + "get_usage_failed": "Failed to get usage", "laf_account": "laf account", "no_intro": "No explanation yet", "not_configured": "Not configured", "open_api_notice": "You can fill in the relevant key of OpenAI/OneAPI. \nIf you fill in this content, the online platform using [AI Dialogue], [Problem Classification] and [Content Extraction] will use the Key you filled in, and there will be no charge. \nPlease pay attention to whether your Key has permission to access the corresponding model. \nGPT models can choose FastAI.", "openai_account_configuration": "OpenAI/OneAPI account", + "openai_account_setting_exception": "Setting up an exception to OpenAI account", "request_address_notice": "Request address, default is openai official. \nThe forwarding address can be filled in, but \\\"v1\\\" is not automatically completed.", "third_party_account": "Third-party account", + "third_party_account.configured": "Configured", + "third_party_account.not_configured": "Not configured", "third_party_account_desc": "The administrator can configure third-party accounts or variables here, and the account will be used by all team members.", "unavailable": "Get usage exception", "usage": "Usage", diff --git a/packages/web/i18n/en/account_usage.json b/packages/web/i18n/en/account_usage.json index 01f5e0a73..6a07f8f05 100644 --- a/packages/web/i18n/en/account_usage.json +++ b/packages/web/i18n/en/account_usage.json @@ -13,8 +13,10 @@ "embedding_index": "Embedding", "every_day": "Day", "every_month": "Moon", + "every_week": "weekly", "export_confirm": "Export confirmation", "export_confirm_tip": "There are currently {{total}} usage records in total. Are you sure to export?", + "export_success": "Export successfully", "export_title": "Time,Members,Type,Project name,AI points", "feishu": "Feishu", "generation_time": "Generation time", diff --git a/packages/web/i18n/en/app.json b/packages/web/i18n/en/app.json index 413871f56..fc9e8913e 100644 --- a/packages/web/i18n/en/app.json +++ b/packages/web/i18n/en/app.json @@ -1,7 +1,11 @@ { "Click_to_delete_this_field": "Click to delete this field", "Filed_is_deprecated": "This field is deprecated", + "MCP_tools_debug": "debug", + "MCP_tools_detail": "check the details", + "MCP_tools_list": "Tool list", "MCP_tools_list_is_empty": "MCP tool not resolved", + "MCP_tools_list_with_number": "Tool list: {{total}}", "MCP_tools_parse_failed": "Failed to parse MCP address", "MCP_tools_url": "MCP Address", "MCP_tools_url_is_empty": "The MCP address cannot be empty", @@ -130,6 +134,7 @@ "response_format": "Response format", "saved_success": "Saved successfully! \nTo use this version externally, click Save and Publish", "search_app": "Search apps", + "search_tool": "Search Tools", "setting_app": "Workflow", "setting_plugin": "Workflow", "show_top_p_tip": "An alternative method of temperature sampling, called Nucleus sampling, the model considers the results of tokens with TOP_P probability mass quality. \nTherefore, 0.1 means that only tokens containing the highest probability quality are considered. \nThe default is 1.", @@ -165,6 +170,7 @@ "template_market_description": "Explore more features in the template market, with configuration tutorials and usage guides to help you understand and get started with various applications.", "template_market_empty_data": "No suitable templates found", "time_zone": "Time Zone", + "tool_detail": "Tool details", "tool_input_param_tip": "This plugin requires configuration of related information to run properly.", "tools_no_description": "This tool has not been introduced ~", "transition_to_workflow": "Convert to Workflow", @@ -175,6 +181,7 @@ "tts_close": "Close", "type.All": "All", "type.Create http plugin tip": "Batch create plugins through OpenAPI Schema, compatible with GPTs format.", + "type.Create mcp tools tip": "Automatically parse and batch create callable MCP tools by entering the MCP address", "type.Create one plugin tip": "Customizable input and output workflows, usually used to encapsulate reusable workflows.", "type.Create plugin bot": "Create Plugin", "type.Create simple bot": "Create Simple App", @@ -186,6 +193,8 @@ "type.Import from json tip": "Create applications directly through JSON configuration files", "type.Import from json_error": "Failed to get workflow data, please check the URL or manually paste the JSON data", "type.Import from json_loading": "Workflow data is being retrieved, please wait...", + "type.MCP tools": "MCP Toolset", + "type.MCP_tools_url": "MCP Address", "type.Plugin": "Plugin", "type.Simple bot": "Simple App", "type.Workflow bot": "Workflow", diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index b262c652a..d40141666 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -186,7 +186,6 @@ "commercial_function_tip": "Please Upgrade to the Commercial Version to Use This Feature: https://doc.fastgpt.cn/docs/commercial/intro/", "comon.Continue_Adding": "Continue Adding", "compliance.chat": "The content is generated by third-party AI and cannot be guaranteed to be true and accurate. It is for reference only.", - "compliance.compliance.dataset": "Please ensure that your content strictly complies with relevant laws and regulations and avoid containing any illegal or infringing content. \nPlease be careful when uploading materials that may contain sensitive information.", "compliance.dataset": "Please ensure that your content strictly complies with relevant laws and regulations and avoid containing any illegal or infringing content. \nPlease be careful when uploading materials that may contain sensitive information.", "confirm_choice": "Confirm Choice", "confirm_move": "Move Here", @@ -431,7 +430,6 @@ "core.dataset.Read Dataset": "View Dataset Details", "core.dataset.Set Website Config": "Start Configuring", "core.dataset.Start export": "Export Started", - "core.dataset.Table collection": "Table Dataset", "core.dataset.Text collection": "Text Dataset", "core.dataset.apiFile": "API File", "core.dataset.collection.Click top config website": "Click to Configure Website", @@ -476,6 +474,7 @@ "core.dataset.error.unAuthDatasetData": "Unauthorized to Operate This Data", "core.dataset.error.unAuthDatasetFile": "Unauthorized to Operate This File", "core.dataset.error.unCreateCollection": "Unauthorized to Operate This Data", + "core.dataset.error.unExistDataset": "The knowledge base does not exist", "core.dataset.error.unLinkCollection": "Not a Web Link Collection", "core.dataset.externalFile": "External File Library", "core.dataset.file": "File", @@ -529,7 +528,6 @@ "core.dataset.search.mode.fullTextRecall desc": "Use traditional full-text search, suitable for finding some keywords and subject-predicate special data", "core.dataset.search.mode.mixedRecall": "Mixed Search", "core.dataset.search.mode.mixedRecall desc": "Use a combination of vector search and full-text search results, sorted using the RRF algorithm.", - "core.dataset.search.score.embedding": "Semantic Search", "core.dataset.search.score.embedding desc": "Get scores by calculating the distance between vectors, ranging from 0 to 1.", "core.dataset.search.score.fullText": "Full Text Search", "core.dataset.search.score.fullText desc": "Calculate the score of the same keywords, ranging from 0 to infinity.", @@ -1271,8 +1269,6 @@ "user.reset_password_tip": "The initial password is not set/the password has not been modified for a long time, please reset the password", "user.team.Balance": "Team Balance", "user.team.Check Team": "Switch", - "user.team.Confirm Invite": "Confirm Invite", - "user.team.Create Team": "Create New Team", "user.team.Leave Team": "Leave Team", "user.team.Leave Team Failed": "Failed to Leave Team", "user.team.Member": "Member", @@ -1283,13 +1279,10 @@ "user.team.Processing invitations Tips": "You have {{amount}} team invitations to process", "user.team.Remove Member Confirm Tip": "Confirm to remove {{username}} from the team?", "user.team.Select Team": "Select Team", - "user.team.Set Name": "Name the Team", "user.team.Switch Team Failed": "Failed to Switch Team", "user.team.Tags Async": "Save", - "user.team.Team Name": "Team Name", "user.team.Team Tags Async": "Tag Sync", "user.team.Team Tags Async Success": "Link Error Successful, Tag Information Updated", - "user.team.Update Team": "Update Team Information", "user.team.invite.Accepted": "Joined Team", "user.team.invite.Deal Width Footer Tip": "It will automatically close after processing", "user.team.invite.Reject": "Invitation Rejected", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 82879ff41..1421aa505 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -7,6 +7,13 @@ "auto_indexes": "Automatically generate supplementary indexes", "auto_indexes_tips": "Additional index generation is performed through large models to improve semantic richness and improve retrieval accuracy.", "auto_training_queue": "Enhanced index queueing", + "backup_collection": "Backup data", + "backup_data_parse": "Backup data is being parsed", + "backup_data_uploading": "Backup data is being uploaded: {{num}}%", + "backup_dataset": "Backup import", + "backup_dataset_success": "The backup was created successfully", + "backup_dataset_tip": "You can reimport the downloaded csv file when exporting the knowledge base.", + "backup_mode": "Backup import", "chunk_max_tokens": "max_tokens", "chunk_size": "Block size", "close_auto_sync": "Are you sure you want to turn off automatic sync?", @@ -115,6 +122,7 @@ "process.Get QA": "Q&A extraction", "process.Image_Index": "Image index generation", "process.Is_Ready": "Ready", + "process.Is_Ready_Count": "{{count}} Group is ready", "process.Parsing": "Parsing", "process.Vectorizing": "Index vectorization", "process.Waiting": "Queue", @@ -143,6 +151,7 @@ "sync_collection_failed": "Synchronization collection error, please check whether the source file can be accessed normally", "sync_schedule": "Timing synchronization", "sync_schedule_tip": "Only existing collections will be synchronized. \nIncludes linked collections and all collections in the API knowledge base. \nThe system will poll for updates every day, and the specific update time cannot be determined.", + "table_model_tip": "Store each row of data as a chunk", "tag.Add_new_tag": "add_new Tag", "tag.Edit_tag": "Edit Tag", "tag.add": "Create", diff --git a/packages/web/i18n/en/login.json b/packages/web/i18n/en/login.json index 01694516e..7fee938ec 100644 --- a/packages/web/i18n/en/login.json +++ b/packages/web/i18n/en/login.json @@ -16,5 +16,6 @@ "register": "Register", "root_password_placeholder": "The root user password is the value of the environment variable DEFAULT_ROOT_PSW", "terms": "Terms", - "use_root_login": "Log in as root user" + "use_root_login": "Log in as root user", + "wecom": "Enterprise WeChat" } diff --git a/packages/web/i18n/en/workflow.json b/packages/web/i18n/en/workflow.json index 8add00b03..8764a2660 100644 --- a/packages/web/i18n/en/workflow.json +++ b/packages/web/i18n/en/workflow.json @@ -175,6 +175,7 @@ "text_content_extraction": "Text Extract", "text_to_extract": "Text to Extract", "these_variables_will_be_input_parameters_for_code_execution": "These variables will be input parameters for code execution", + "tool.tool_result": "Tool operation results", "tool_call_termination": "Stop ToolCall", "tool_custom_field": "Custom Tool", "tool_field": " Tool Field Parameter Configuration", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index d76d7f262..ce6ade586 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -430,7 +430,6 @@ "core.dataset.Read Dataset": "查看知识库详情", "core.dataset.Set Website Config": "开始配置", "core.dataset.Start export": "已开始导出", - "core.dataset.Table collection": "表格数据集", "core.dataset.Text collection": "文本数据集", "core.dataset.apiFile": "API 文件", "core.dataset.collection.Click top config website": "点击配置网站", @@ -475,7 +474,7 @@ "core.dataset.error.unAuthDatasetData": "无权操作该数据", "core.dataset.error.unAuthDatasetFile": "无权操作该文件", "core.dataset.error.unCreateCollection": "无权操作该数据", - "core.dataset.error.unExistDataset": "数据集不存在", + "core.dataset.error.unExistDataset": "知识库不存在", "core.dataset.error.unLinkCollection": "不是网络链接集合", "core.dataset.externalFile": "外部文件库", "core.dataset.file": "文件", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index e0e99199a..153e0cd91 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -7,6 +7,13 @@ "auto_indexes": "自动生成补充索引", "auto_indexes_tips": "通过大模型进行额外索引生成，提高语义丰富度，提高检索的精度。", "auto_training_queue": "增强索引排队", + "backup_collection": "备份数据", + "backup_data_parse": "备份数据解析中", + "backup_data_uploading": "备份数据上传中: {{num}}%", + "backup_dataset": "备份导入", + "backup_dataset_success": "备份创建成功", + "backup_dataset_tip": "可以将导出知识库时，下载的 csv 文件重新导入。", + "backup_mode": "备份导入", "chunk_max_tokens": "分块上限", "chunk_size": "分块大小", "close_auto_sync": "确认关闭自动同步功能？", diff --git a/packages/web/i18n/zh-CN/user.json b/packages/web/i18n/zh-CN/user.json index 9e1c94f38..4015933ca 100644 --- a/packages/web/i18n/zh-CN/user.json +++ b/packages/web/i18n/zh-CN/user.json @@ -88,6 +88,7 @@ "team.add_writer": "添加可写成员", "team.avatar_and_name": "头像 & 名称", "team.belong_to_group": "所属群组", + "team.collaborator.added": "已添加", "team.group.avatar": "群头像", "team.group.create": "创建群组", "team.group.create_failed": "创建群组失败", @@ -99,9 +100,10 @@ "team.group.keep_admin": "保留管理员权限", "team.group.manage_member": "管理成员", "team.group.manage_tip": "可以管理成员、创建群组、管理所有群组、为群组和成员分配权限", - "team.group.permission_tip": "单独配置权限的成员，将遵循个人权限配置，不再受群组权限影响。\n若成员在多个权限组，则该成员的权限取并集。", "team.group.members": "成员", "team.group.name": "群组名称", + "team.group.permission.write": "", + "team.group.permission_tip": "单独配置权限的成员，将遵循个人权限配置，不再受群组权限影响。\n若成员在多个权限组，则该成员的权限取并集。", "team.group.role.admin": "管理员", "team.group.role.member": "成员", "team.group.role.owner": "所有者", @@ -111,6 +113,5 @@ "team.manage_collaborators": "管理协作者", "team.no_collaborators": "暂无协作者", "team.org.org": "部门", - "team.write_role_member": "可写权限", - "team.collaborator.added": "已添加" + "team.write_role_member": "可写权限" } diff --git a/packages/web/i18n/zh-Hant/account.json b/packages/web/i18n/zh-Hant/account.json index 068644db5..e65f2fed1 100644 --- a/packages/web/i18n/zh-Hant/account.json +++ b/packages/web/i18n/zh-Hant/account.json @@ -15,6 +15,7 @@ "model.active": "啟用", "model.alias": "別名", "model.alias_tip": "模型在系統中展示的名字，方便使用者理解", + "model.censor": "啟用敏感校驗", "model.censor_tip": "如果需要進行敏感校驗，則開啟該開關", "model.charsPointsPrice": "模型綜合價格", "model.charsPointsPrice_tip": "將模型輸入和輸出合併起來進行 Token 計費，語言模型如果單獨設定了輸入和輸出計費，則按輸入和輸出分別計算", @@ -65,6 +66,7 @@ "model.tool_choice_tip": "如果該模型支援工具呼叫，則開啟該開關", "model.used_in_classify": "用於問題分類", "model.used_in_extract_fields": "用於文字擷取", + "model.used_in_query_extension": "用於問題優化", "model.used_in_tool_call": "用於工具呼叫節點", "model.vision": "支援圖片識別", "model.vision_tag": "視覺", diff --git a/packages/web/i18n/zh-Hant/account_info.json b/packages/web/i18n/zh-Hant/account_info.json index 20cf1f792..05da4a404 100644 --- a/packages/web/i18n/zh-Hant/account_info.json +++ b/packages/web/i18n/zh-Hant/account_info.json @@ -39,8 +39,6 @@ "new_password": "新密碼", "notification_receiving": "通知接收", "old_password": "舊密碼", - "openai_account_configuration": "OpenAI 帳號設定", - "openai_account_setting_exception": "設定 OpenAI 帳號異常", "package_and_usage": "套餐與用量", "package_details": "套餐詳細資訊", "package_expiry_time": "套餐到期時間", @@ -52,8 +50,10 @@ "password_update_success": "修改密碼成功", "pending_usage": "待使用", "phone_label": "手機號", + "please_bind_contact": "請綁定聯繫方式", "please_bind_notification_receiving_path": "請先綁定通知接收途徑", "purchase_extra_package": "購買額外套餐", + "redeem_coupon": "兌換代碼", "reminder_create_bound_notification_account": "提醒建立者綁定通知帳號", "reset_password": "重置密碼", "resource_usage": "資源用量", @@ -75,6 +75,5 @@ "user_team_team_name": "團隊", "verification_code": "驗證碼", "you_can_convert": "您可以兌換", - "yuan": "元", - "redeem_coupon": "兌換代碼" + "yuan": "元" } diff --git a/packages/web/i18n/zh-Hant/account_model.json b/packages/web/i18n/zh-Hant/account_model.json index 103f4b660..704abd16b 100644 --- a/packages/web/i18n/zh-Hant/account_model.json +++ b/packages/web/i18n/zh-Hant/account_model.json @@ -18,6 +18,8 @@ "create_channel": "新增管道", "default_url": "預設地址", "detail": "詳細資訊", + "duration": "耗時", + "edit": "編輯", "edit_channel": "管道設定", "enable_channel": "啟用", "forbid_channel": "停用", diff --git a/packages/web/i18n/zh-Hant/account_team.json b/packages/web/i18n/zh-Hant/account_team.json index 9c31f6320..9bbe81774 100644 --- a/packages/web/i18n/zh-Hant/account_team.json +++ b/packages/web/i18n/zh-Hant/account_team.json @@ -8,8 +8,9 @@ "assign_permission": "權限變更", "change_department_name": "部門編輯", "change_member_name": "成員改名", + "confirm_delete_from_org": "確認將 {{username}} 移出部門？", + "confirm_delete_from_team": "確認將 {{username}} 移出團隊？", "confirm_delete_group": "確認刪除群組？", - "confirm_delete_member": "確認刪除成員？", "confirm_delete_org": "確認刪除該部門？", "confirm_forbidden": "確認停用", "confirm_leave_team": "確認離開該團隊？ \n結束後，您在該團隊所有的資源轉讓給團隊所有者。", @@ -21,6 +22,8 @@ "create_sub_org": "建立子部門", "delete": "刪除", "delete_department": "刪除子部門", + "delete_from_org": "移出部門", + "delete_from_team": "移出團隊", "delete_group": "刪除群組", "delete_org": "刪除部門", "edit_info": "編輯訊息", @@ -28,6 +31,7 @@ "edit_member_tip": "成員名", "edit_org_info": "編輯部門資訊", "expires": "過期時間", + "export_members": "導出成員", "forbid_hint": "停用後，該邀請連結將失效。該操作不可撤銷，是否確認停用？", "forbid_success": "停用成功", "forbidden": "停用", @@ -44,8 +48,10 @@ "invite_member": "邀請成員", "invited": "已邀請", "join_team": "加入團隊", + "join_update_time": "加入/更新時間", "kick_out_team": "移除成員", "label_sync": "標籤同步", + "leave": "已離職", "leave_team_failed": "離開團隊異常", "log_assign_permission": "【{{name}}】更新了【{{objectName}}】的權限：[應用創建:【{{appCreate}}】, 知識庫:【{{datasetCreate}}】, API密鑰:【{{apiKeyCreate}}】, 管理:【{{manage}}】]", "log_change_department": "【{{name}}】更新了部門【{{departmentName}}】", @@ -70,6 +76,7 @@ "member_group": "所屬成員組", "move_member": "移動成員", "move_org": "行動部門", + "notification_recieve": "團隊通知接收", "operation_log": "紀錄", "org": "組織", "org_description": "介紹", @@ -84,13 +91,22 @@ "permission_datasetCreate_Tip": "可以在根目錄建立知識庫，（資料夾下的建立權限由資料夾控制）", "permission_manage": "管理員", "permission_manage_tip": "可以管理成員、建立群組、管理所有群組、為群組和成員分配權限", + "please_bind_contact": "請綁定聯繫方式", "recover_team_member": "成員恢復", "relocate_department": "部門移動", "remark": "備註", "remove_tip": "確認將 {{username}} 移出團隊？", + "restore_tip": "確認將 {{username}} 加入團隊嗎？\n僅恢復該成員賬號可用性及相關權限，無法恢復賬號下資源。", + "restore_tip_title": "恢復確認", "retain_admin_permissions": "保留管理員權限", "search_log": "搜索日誌", + "search_member": "搜索成員", "search_member_group_name": "搜尋成員/群組名稱", + "search_org": "搜索部門", + "set_name_avatar": "團隊頭像", + "sync_immediately": "立即同步", + "sync_member_failed": "同步成員失敗", + "sync_member_success": "同步成員成功", "total_team_members": "共 {{amount}} 名成員", "transfer_ownership": "轉讓所有者", "unlimited": "無限制", diff --git a/packages/web/i18n/zh-Hant/account_thirdParty.json b/packages/web/i18n/zh-Hant/account_thirdParty.json index c128350aa..248343df2 100644 --- a/packages/web/i18n/zh-Hant/account_thirdParty.json +++ b/packages/web/i18n/zh-Hant/account_thirdParty.json @@ -1,13 +1,17 @@ { "configured": "已設定", "error.no_permission": "請聯絡管理員設定", + "get_usage_failed": "獲取使用量失敗", "laf_account": "af 帳號", "no_intro": "暫無說明", "not_configured": "未設定", "open_api_notice": "可以填寫 OpenAI/OneAPI 的相關金鑰。\n如果你填寫了該內容，在線上平臺使用【AI 對話】、【問題分類】和【內容提取】將會走你填寫的 Key，不會計費用。\n請注意你的 Key 是否有存取對應模型的權限。 \nGPT 模型可以選擇 FastAI。", "openai_account_configuration": "OpenAI/OneAPI 帳號", + "openai_account_setting_exception": "設置 OpenAI 賬號異常", "request_address_notice": "請求地址，預設為 openai 官方。可填中轉位址，未自動補全 \"v1\"", - "third_party_account": "第三方帳號", + "third_party_account": "第三方賬號", + "third_party_account.configured": "已配置", + "third_party_account.not_configured": "未配置", "third_party_account_desc": "管理員可以在這裡設定第三方帳號或變數，該帳號將被團隊所有人使用", "unavailable": "取得使用量異常", "usage": "使用量：", diff --git a/packages/web/i18n/zh-Hant/account_usage.json b/packages/web/i18n/zh-Hant/account_usage.json index 4f0fa5f28..fe1145801 100644 --- a/packages/web/i18n/zh-Hant/account_usage.json +++ b/packages/web/i18n/zh-Hant/account_usage.json @@ -13,8 +13,10 @@ "embedding_index": "索引生成", "every_day": "天", "every_month": "月", + "every_week": "每週", "export_confirm": "匯出確認", "export_confirm_tip": "目前共 {{total}} 筆使用記錄，確認匯出？", + "export_success": "導出成功", "export_title": "時間,成員,類型,項目名,AI 積分消耗", "feishu": "飛書", "generation_time": "生成時間", diff --git a/packages/web/i18n/zh-Hant/app.json b/packages/web/i18n/zh-Hant/app.json index aed7c1053..b9550c14e 100644 --- a/packages/web/i18n/zh-Hant/app.json +++ b/packages/web/i18n/zh-Hant/app.json @@ -1,7 +1,11 @@ { "Click_to_delete_this_field": "點擊刪除該字段", "Filed_is_deprecated": "該字段已棄用", + "MCP_tools_debug": "偵錯", + "MCP_tools_detail": "查看詳情", + "MCP_tools_list": "工具列表", "MCP_tools_list_is_empty": "未解析到 MCP 工具", + "MCP_tools_list_with_number": "工具列表: {{total}}", "MCP_tools_parse_failed": "解析 MCP 地址失敗", "MCP_tools_url": "MCP 地址", "MCP_tools_url_is_empty": "MCP 地址不能為空", @@ -130,6 +134,7 @@ "response_format": "回覆格式", "saved_success": "儲存成功！\n如需在外部使用該版本，請點選“儲存並發布”", "search_app": "搜尋應用程式", + "search_tool": "搜索工具", "setting_app": "應用程式設定", "setting_plugin": "外掛設定", "show_top_p_tip": "用溫度取樣的替代方法，稱為 Nucleus 取樣，該模型考慮了具有 TOP_P 機率質量質量的令牌的結果。\n因此，0.1 表示僅考慮包含最高機率質量的令牌。\n預設為 1。", @@ -165,6 +170,7 @@ "template_market_description": "在範本市集探索更多玩法，設定教學與使用指引，帶您理解並上手各種應用程式", "template_market_empty_data": "找不到合適的範本", "time_zone": "時區", + "tool_detail": "工具詳情", "tool_input_param_tip": "這個外掛正常執行需要設定相關資訊", "tools_no_description": "這個工具沒有介紹~", "transition_to_workflow": "轉換成工作流程", @@ -175,6 +181,7 @@ "tts_close": "關閉", "type.All": "全部", "type.Create http plugin tip": "透過 OpenAPI Schema 批次建立外掛，相容 GPTs 格式", + "type.Create mcp tools tip": "通過輸入 MCP 地址，自動解析並批量創建可調用的 MCP 工具", "type.Create one plugin tip": "可以自訂輸入和輸出的工作流程，通常用於封裝重複使用的工作流程", "type.Create plugin bot": "建立外掛", "type.Create simple bot": "建立簡易應用程式", @@ -186,6 +193,8 @@ "type.Import from json tip": "透過 JSON 設定文件，直接建立應用", "type.Import from json_error": "獲取工作流數據失敗，請檢查URL或手動粘貼JSON數據", "type.Import from json_loading": "正在獲取工作流數據，請稍候...", + "type.MCP tools": "MCP 工具集", + "type.MCP_tools_url": "MCP 地址", "type.Plugin": "外掛", "type.Simple bot": "簡易應用程式", "type.Workflow bot": "工作流程", diff --git a/packages/web/i18n/zh-Hant/chat.json b/packages/web/i18n/zh-Hant/chat.json index 33306098b..53164268f 100644 --- a/packages/web/i18n/zh-Hant/chat.json +++ b/packages/web/i18n/zh-Hant/chat.json @@ -26,6 +26,8 @@ "content_empty": "無內容", "contextual": "{{num}} 筆上下文", "contextual_preview": "上下文預覽 {{num}} 筆", + "core.chat.moveCancel": "上滑取消", + "core.chat.shortSpeak": "說話時間太短", "csv_input_lexicon_tip": "僅支援 CSV 批次匯入，點選下載範本", "custom_input_guide_url": "自訂詞彙庫網址", "data_source": "來源知識庫：{{name}}", @@ -48,7 +50,6 @@ "items": "筆", "llm_tokens": "LLM tokens", "module_runtime_and": "模組執行總時間", - "moveCancel": "上滑取消", "multiple_AI_conversations": "多組 AI 對話", "new_input_guide_lexicon": "新增詞彙庫", "no_workflow_response": "無工作流程資料", @@ -57,6 +58,7 @@ "plugins_output": "外掛程式輸出", "press_to_speak": "按住說話", "query_extension_IO_tokens": "問題最佳化輸入/輸出 Tokens", + "query_extension_result": "問題優化結果", "question_tip": "由上至下，各個模組的回應順序", "read_raw_source": "開啟原文", "reasoning_text": "思考過程", @@ -73,7 +75,6 @@ "select_file": "上傳檔案", "select_file_img": "上傳檔案 / 圖片", "select_img": "上傳圖片", - "shortSpeak ": "說話時間太短", "source_cronJob": "定時執行", "stream_output": "串流輸出", "to_dataset": "前往知識庫", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index f2d1eeefb..dbde30e8b 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -114,6 +114,7 @@ "click_to_resume": "點選繼續", "code_editor": "程式碼編輯器", "code_error.account_error": "帳號名稱或密碼錯誤", + "code_error.account_exist": "賬號已註冊", "code_error.account_not_found": "使用者未註冊", "code_error.app_error.invalid_app_type": "無效的應用程式類型", "code_error.app_error.invalid_owner": "非法的應用程式擁有者", @@ -185,7 +186,6 @@ "commercial_function_tip": "請升級為商業版後使用此功能：https://doc.fastgpt.cn/docs/commercial/intro/", "comon.Continue_Adding": "繼續新增", "compliance.chat": "內容由第三方 AI 產生，無法保證其真實性與準確性，僅供參考。", - "compliance.compliance.dataset": "請確保您的內容嚴格遵守相關法律法規，避免包含任何違法或侵權的內容。\n在上傳可能涉及敏感資訊的資料時請務必謹慎。", "compliance.dataset": "請確保您的內容嚴格遵守相關法律法規，避免包含任何違法或侵權的內容。\n在上傳可能涉及敏感資訊的資料時請務必謹慎。", "confirm_choice": "確認選擇", "confirm_move": "移動至此", @@ -430,7 +430,6 @@ "core.dataset.Read Dataset": "檢視知識庫詳細資料", "core.dataset.Set Website Config": "開始設定", "core.dataset.Start export": "已開始匯出", - "core.dataset.Table collection": "表格資料集", "core.dataset.Text collection": "文字資料集", "core.dataset.apiFile": "API 檔案", "core.dataset.collection.Click top config website": "點選設定網站", @@ -475,6 +474,7 @@ "core.dataset.error.unAuthDatasetData": "無權操作此資料", "core.dataset.error.unAuthDatasetFile": "無權操作此檔案", "core.dataset.error.unCreateCollection": "無權操作此資料", + "core.dataset.error.unExistDataset": "知識庫不存在", "core.dataset.error.unLinkCollection": "不是網路連結集合", "core.dataset.externalFile": "外部檔案庫", "core.dataset.file": "檔案", @@ -528,7 +528,6 @@ "core.dataset.search.mode.fullTextRecall desc": "使用傳統的全文檢索，適合尋找特定關鍵字和主謂語的特殊資料", "core.dataset.search.mode.mixedRecall": "混合檢索", "core.dataset.search.mode.mixedRecall desc": "使用向量檢索與全文檢索的綜合結果，並使用 RRF 演算法進行排序。", - "core.dataset.search.score.embedding": "語意檢索", "core.dataset.search.score.embedding desc": "透過計算向量之間的距離取得分數，範圍為 0 到 1。", "core.dataset.search.score.fullText": "全文檢索", "core.dataset.search.score.fullText desc": "計算相同關鍵字的分數，範圍為 0 到無限大。", @@ -760,7 +759,6 @@ "dataset.Create Folder": "建立資料夾", "dataset.Create manual collection": "建立手動資料集", "dataset.Delete Dataset Error": "刪除知識庫錯誤", - "dataset.Edit API Service": "編輯 API 檔案介面", "dataset.Edit Folder": "編輯資料夾", "dataset.Edit Info": "編輯資訊", "dataset.Export": "匯出", @@ -878,6 +876,7 @@ "model.provider": "模型提供者", "model.search_name_placeholder": "根據模型名搜尋", "model.type.chat": "語言模型", + "model.type.embedding": "索引模型", "model.type.reRank": "重排模型", "model.type.stt": "語音辨識", "model.type.tts": "語音合成", @@ -1270,8 +1269,6 @@ "user.reset_password_tip": "未設置初始密碼/長時間未修改密碼，請重置密碼", "user.team.Balance": "團隊餘額", "user.team.Check Team": "切換", - "user.team.Confirm Invite": "確認邀請", - "user.team.Create Team": "建立新團隊", "user.team.Leave Team": "離開團隊", "user.team.Leave Team Failed": "離開團隊失敗", "user.team.Member": "成員", @@ -1282,13 +1279,10 @@ "user.team.Processing invitations Tips": "您有 {{amount}} 個需要處理的團隊邀請", "user.team.Remove Member Confirm Tip": "確認將 {{username}} 移出團隊？", "user.team.Select Team": "選擇團隊", - "user.team.Set Name": "為團隊命名", "user.team.Switch Team Failed": "切換團隊失敗", "user.team.Tags Async": "儲存", - "user.team.Team Name": "團隊名稱", "user.team.Team Tags Async": "標籤同步", "user.team.Team Tags Async Success": "連結錯誤修正成功，標籤資訊已更新", - "user.team.Update Team": "更新團隊資訊", "user.team.invite.Accepted": "已加入團隊", "user.team.invite.Deal Width Footer Tip": "處理完會自動關閉", "user.team.invite.Reject": "已拒絕邀請", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index fbdaceae4..ab13a5ec5 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -7,6 +7,12 @@ "auto_indexes": "自動生成補充索引", "auto_indexes_tips": "透過大模型進行額外索引生成，提高語義豐富度，提高檢索的精度。", "auto_training_queue": "增強索引排隊", + "backup_collection": "備份數據", + "backup_data_uploading": "備份數據上傳中: {{num}}%", + "backup_dataset": "備份導入", + "backup_dataset_success": "備份創建成功", + "backup_dataset_tip": "可以將導出知識庫時，下載的 csv 文件重新導入。", + "backup_mode": "備份導入", "chunk_max_tokens": "分塊上限", "chunk_size": "分塊大小", "close_auto_sync": "確認關閉自動同步功能？", @@ -115,6 +121,7 @@ "process.Get QA": "問答對提取", "process.Image_Index": "圖片索引生成", "process.Is_Ready": "已就緒", + "process.Is_Ready_Count": "{{count}} 組已就緒", "process.Parsing": "內容解析中", "process.Vectorizing": "索引向量化", "process.Waiting": "排隊中", diff --git a/packages/web/i18n/zh-Hant/login.json b/packages/web/i18n/zh-Hant/login.json index 8321b62f9..a4371809d 100644 --- a/packages/web/i18n/zh-Hant/login.json +++ b/packages/web/i18n/zh-Hant/login.json @@ -16,5 +16,6 @@ "register": "註冊帳號", "root_password_placeholder": "root 使用者密碼為環境變數 DEFAULT_ROOT_PSW 的值", "terms": "服務條款", - "use_root_login": "使用 root 使用者登入" + "use_root_login": "使用 root 使用者登入", + "wecom": "企業微信" } diff --git a/packages/web/i18n/zh-Hant/workflow.json b/packages/web/i18n/zh-Hant/workflow.json index 866979c03..35fd4409f 100644 --- a/packages/web/i18n/zh-Hant/workflow.json +++ b/packages/web/i18n/zh-Hant/workflow.json @@ -175,6 +175,7 @@ "text_content_extraction": "文字內容擷取", "text_to_extract": "要擷取的文字", "these_variables_will_be_input_parameters_for_code_execution": "這些變數會作為程式碼執行的輸入參數", + "tool.tool_result": "工具運行結果", "tool_call_termination": "工具呼叫終止", "tool_custom_field": "自訂工具變數", "tool_field": "工具參數設定", diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx new file mode 100644 index 000000000..267ff3b2c --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx @@ -0,0 +1,94 @@ +import React, { useState } from 'react'; +import MyModal from '@fastgpt/web/components/common/MyModal'; +import { useTranslation } from 'next-i18next'; +import { Box, Button, HStack, ModalBody, ModalFooter, VStack } from '@chakra-ui/react'; +import FileSelector, { type SelectFileItemType } from '../components/FileSelector'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyIconButton from '@fastgpt/web/components/common/Icon/button'; +import { postBackupDatasetCollection } from '@/web/core/dataset/api'; +import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { useContextSelector } from 'use-context-selector'; +import LightTip from '@fastgpt/web/components/common/LightTip'; + +const BackupImportModal = ({ + onFinish, + onClose +}: { + onFinish: () => void; + onClose: () => void; +}) => { + const { t } = useTranslation(); + const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId); + + const [selectFiles, setSelectFiles] = useState([]); + const [percent, setPercent] = useState(0); + + const { runAsync: onBackupImport, loading: isBackupLoading } = useRequest2( + async () => { + await postBackupDatasetCollection({ + datasetId, + file: selectFiles[0].file, + percentListen: setPercent + }); + }, + { + onSuccess() { + onFinish(); + onClose(); + }, + successToast: t('dataset:backup_dataset_success') + } + ); + + return ( + + + + + + {/* File render */} + {selectFiles.length > 0 && ( + + {selectFiles.map((item, index) => ( + + + {item.name} + + {item.size} + + { + setSelectFiles(selectFiles.filter((_, i) => i !== index)); + }} + /> + + ))} + + )} + + + + + + + ); +}; + +export default BackupImportModal; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx index 94e97c313..6f5c857e0 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx @@ -36,6 +36,7 @@ import MyTag from '@fastgpt/web/components/common/Tag/index'; import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector')); +const BackupImportModal = dynamic(() => import('./BackupImportModal')); const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { const { t } = useTranslation(); @@ -76,6 +77,12 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { onOpen: onOpenFileSourceSelector, onClose: onCloseFileSourceSelector } = useDisclosure(); + // Backup import modal + const { + isOpen: isOpenBackupImportModal, + onOpen: onOpenBackupImportModal, + onClose: onCloseBackupImportModal + } = useDisclosure(); const { runAsync: onCreateCollection } = useRequest2( async ({ name, type }: { name: string; type: DatasetCollectionTypeEnum }) => { @@ -220,11 +227,11 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { { label: ( - - {t('common:Folder')} + + {t('common:core.dataset.Text collection')} ), - onClick: () => setEditFolderData({}) + onClick: onOpenFileSourceSelector }, { label: ( @@ -244,27 +251,24 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { { label: ( - - {t('common:core.dataset.Text collection')} + + {t('dataset:backup_dataset')} ), - onClick: onOpenFileSourceSelector - }, + onClick: onOpenBackupImportModal + } + ] + }, + { + children: [ { label: ( - - {t('common:core.dataset.Table collection')} + + {t('common:Folder')} ), - onClick: () => - router.replace({ - query: { - ...router.query, - currentTab: TabEnum.import, - source: ImportDataSourceEnum.csvTable - } - }) + onClick: () => setEditFolderData({}) } ] } @@ -471,6 +475,14 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { )} {isOpenFileSourceSelector && } + {isOpenBackupImportModal && ( + { + getData(1); + }} + onClose={onCloseBackupImportModal} + /> + )} ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx index 85dc3800c..33a7de218 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx @@ -257,18 +257,12 @@ const CollectionCard = () => { )} - {!checkCollectionIsFolder(collection.type) ? ( - <> - {collection.trainingType - ? t( - (DatasetCollectionDataProcessModeMap[collection.trainingType] - ?.label || '-') as any - ) - : '-'} - - ) : ( - '-' - )} + {collection.trainingType + ? t( + (DatasetCollectionDataProcessModeMap[collection.trainingType]?.label || + '-') as any + ) + : '-'} {collection.dataAmount || '-'} diff --git a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx index 5d1efbec7..9e54d1493 100644 --- a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx +++ b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx @@ -27,7 +27,10 @@ import Markdown from '@/components/Markdown'; import { useMemoizedFn } from 'ahooks'; import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; import { TabEnum } from './NavBar'; -import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; +import { + DatasetCollectionDataProcessModeEnum, + ImportDataSourceEnum +} from '@fastgpt/global/core/dataset/constants'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import TrainingStates from './CollectionCard/TrainingStates'; import { getTextValidLength } from '@fastgpt/global/common/string/utils'; diff --git a/projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx b/projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx index 20a29607d..f372199cc 100644 --- a/projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx @@ -118,14 +118,18 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn { - const list = Object.entries(DatasetCollectionDataProcessModeMap); - return list - .filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto) - .map(([key, value]) => ({ - title: t(value.label as any), - value: key as DatasetCollectionDataProcessModeEnum, - tooltip: t(value.tooltip as any) - })); + const list = { + [DatasetCollectionDataProcessModeEnum.chunk]: + DatasetCollectionDataProcessModeMap[DatasetCollectionDataProcessModeEnum.chunk], + [DatasetCollectionDataProcessModeEnum.qa]: + DatasetCollectionDataProcessModeMap[DatasetCollectionDataProcessModeEnum.qa] + }; + + return Object.entries(list).map(([key, value]) => ({ + title: t(value.label as any), + value: key as DatasetCollectionDataProcessModeEnum, + tooltip: t(value.tooltip as any) + })); }, [t]); const { chunkSizeField, diff --git a/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx b/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx index 09b401bdc..46cb6f837 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx @@ -144,20 +144,6 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode title: t('dataset:import_confirm') } ], - [ImportDataSourceEnum.csvTable]: [ - { - title: t('dataset:import_select_file') - }, - { - title: t('dataset:import_param_setting') - }, - { - title: t('dataset:import_data_preview') - }, - { - title: t('dataset:import_confirm') - } - ], [ImportDataSourceEnum.externalFile]: [ { title: t('dataset:import_select_file') @@ -206,7 +192,7 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode chunkSettingMode: ChunkSettingModeEnum.auto, chunkSplitMode: DataChunkSplitModeEnum.size, - embeddingChunkSize: 2000, + embeddingChunkSize: chunkAutoChunkSize, indexSize: vectorModel?.defaultToken || 512, qaChunkSize: getLLMDefaultChunkSize(agentModel), chunkSplitter: '', diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx index 644c9c6f4..76e7f7d97 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx @@ -75,7 +75,6 @@ const PreviewData = () => { overlapRatio: chunkOverlapRatio, selector: processParamsForm.getValues('webSelector'), - isQAImport: importSource === ImportDataSourceEnum.csvTable, externalFileId: previewFile.externalFileId }); }, diff --git a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx index 0ca3290e5..ffe4947a1 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx @@ -26,7 +26,6 @@ import { useRouter } from 'next/router'; import { TabEnum } from '../../../../../pages/dataset/detail/index'; import { postCreateDatasetApiDatasetCollection, - postCreateDatasetCsvTableCollection, postCreateDatasetExternalFileCollection, postCreateDatasetFileCollection, postCreateDatasetLinkCollection, @@ -146,11 +145,6 @@ const Upload = () => { ...commonParams, text: item.rawText }); - } else if (importSource === ImportDataSourceEnum.csvTable && item.dbFileId) { - await postCreateDatasetCsvTableCollection({ - ...commonParams, - fileId: item.dbFileId - }); } else if (importSource === ImportDataSourceEnum.externalFile && item.externalFileUrl) { await postCreateDatasetExternalFileCollection({ ...commonParams, diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/TableLocal.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/TableLocal.tsx deleted file mode 100644 index 0c5bce520..000000000 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/TableLocal.tsx +++ /dev/null @@ -1,101 +0,0 @@ -import React, { useEffect, useMemo, useState } from 'react'; -import { type ImportSourceItemType } from '@/web/core/dataset/type.d'; -import { Box, Button } from '@chakra-ui/react'; -import FileSelector from '../components/FileSelector'; -import { useTranslation } from 'next-i18next'; - -import dynamic from 'next/dynamic'; -import { fileDownload } from '@/web/common/file/utils'; -import { RenderUploadFiles } from '../components/RenderFiles'; -import { useContextSelector } from 'use-context-selector'; -import { DatasetImportContext } from '../Context'; - -const PreviewData = dynamic(() => import('../commonProgress/PreviewData')); -const Upload = dynamic(() => import('../commonProgress/Upload')); - -const fileType = '.csv'; - -const FileLocal = () => { - const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); - - return ( - <> - {activeStep === 0 && } - {activeStep === 1 && } - {activeStep === 2 && } - - ); -}; - -export default React.memo(FileLocal); - -const csvTemplate = `index,content -"第一列内容","第二列内容" -"必填列","可选列。CSV 中请注意内容不能包含双引号，双引号是列分割符号" -"只会将第一和第二列内容导入，其余列会被忽略","" -"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。","" -"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`; - -const SelectFile = React.memo(function SelectFile() { - const { t } = useTranslation(); - const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v); - const [selectFiles, setSelectFiles] = useState( - sources.map((source) => ({ - isUploading: false, - ...source - })) - ); - const [uploading, setUploading] = useState(false); - - const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]); - - useEffect(() => { - setSources(successFiles); - }, [successFiles]); - - return ( - - setUploading(true)} - onFinishSelect={() => setUploading(false)} - /> - - - fileDownload({ - text: csvTemplate, - type: 'text/csv;charset=utf-8', - filename: 'template.csv' - }) - } - > - {t('common:core.dataset.import.Down load csv template')} - - - {/* render files */} - - - - - - - ); -}); diff --git a/projects/app/src/pageComponents/dataset/detail/Import/index.tsx b/projects/app/src/pageComponents/dataset/detail/Import/index.tsx index ea476c1a5..f9d253ff3 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/index.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/index.tsx @@ -8,7 +8,6 @@ import DatasetImportContextProvider, { DatasetImportContext } from './Context'; const FileLocal = dynamic(() => import('./diffSource/FileLocal')); const FileLink = dynamic(() => import('./diffSource/FileLink')); const FileCustomText = dynamic(() => import('./diffSource/FileCustomText')); -const TableLocal = dynamic(() => import('./diffSource/TableLocal')); const ExternalFileCollection = dynamic(() => import('./diffSource/ExternalFile')); const APIDatasetCollection = dynamic(() => import('./diffSource/APIDataset')); const ReTraining = dynamic(() => import('./diffSource/ReTraining')); @@ -21,7 +20,6 @@ const ImportDataset = () => { if (importSource === ImportDataSourceEnum.fileLocal) return FileLocal; if (importSource === ImportDataSourceEnum.fileLink) return FileLink; if (importSource === ImportDataSourceEnum.fileCustom) return FileCustomText; - if (importSource === ImportDataSourceEnum.csvTable) return TableLocal; if (importSource === ImportDataSourceEnum.externalFile) return ExternalFileCollection; if (importSource === ImportDataSourceEnum.apiDataset) return APIDatasetCollection; }, [importSource]); diff --git a/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx b/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx index 39ba676c3..7c748d29a 100644 --- a/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx +++ b/projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx @@ -84,14 +84,22 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => { label: t('dataset:collection.training_type'), value: t(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label as any) }, - { - label: t('dataset:chunk_size'), - value: collection.chunkSize || '-' - }, - { - label: t('dataset:index_size'), - value: collection.indexSize || '-' - }, + ...(collection.chunkSize + ? [ + { + label: t('dataset:chunk_size'), + value: collection.chunkSize + } + ] + : []), + ...(collection.indexSize + ? [ + { + label: t('dataset:index_size'), + value: collection.indexSize + } + ] + : []), ...(webSelector ? [ { diff --git a/projects/app/src/pageComponents/dataset/detail/components/FileSelector.tsx b/projects/app/src/pageComponents/dataset/detail/components/FileSelector.tsx new file mode 100644 index 000000000..c6db33437 --- /dev/null +++ b/projects/app/src/pageComponents/dataset/detail/components/FileSelector.tsx @@ -0,0 +1,218 @@ +import MyBox from '@fastgpt/web/components/common/MyBox'; +import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { Box, type FlexProps } from '@chakra-ui/react'; +import { formatFileSize } from '@fastgpt/global/common/file/tools'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { useTranslation } from 'next-i18next'; +import React, { type DragEvent, useCallback, useMemo, useState } from 'react'; +import { getFileIcon } from '@fastgpt/global/common/file/icon'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; + +export type SelectFileItemType = { + file: File; + icon: string; + name: string; + size: string; +}; + +const FileSelector = ({ + fileType, + selectFiles, + setSelectFiles, + maxCount = 1000, + ...props +}: { + fileType: string; + selectFiles: SelectFileItemType[]; + setSelectFiles: React.Dispatch>; + maxCount?: number; +} & FlexProps) => { + const { t } = useTranslation(); + + const { toast } = useToast(); + const { feConfigs } = useSystemStore(); + + const maxSize = (feConfigs?.uploadFileMaxSize || 1024) * 1024 * 1024; + + const { File, onOpen } = useSelectFile({ + fileType, + multiple: maxCount > 1, + maxCount + }); + const [isDragging, setIsDragging] = useState(false); + const isMaxSelected = useMemo( + () => selectFiles.length >= maxCount, + [maxCount, selectFiles.length] + ); + + const filterTypeReg = new RegExp( + `(${fileType + .split(',') + .map((item) => item.trim()) + .join('|')})$`, + 'i' + ); + + const onSelectFile = useCallback( + async (files: File[]) => { + const fileList = files.map((file) => ({ + file, + icon: getFileIcon(file.name), + name: file.name, + size: formatFileSize(file.size) + })); + setSelectFiles((state) => { + return [...fileList, ...state].slice(0, maxCount); + }); + }, + [maxCount, setSelectFiles] + ); + + const handleDragEnter = (e: DragEvent) => { + e.preventDefault(); + setIsDragging(true); + }; + + const handleDragLeave = (e: DragEvent) => { + e.preventDefault(); + setIsDragging(false); + }; + + const handleDrop = async (e: DragEvent) => { + e.preventDefault(); + setIsDragging(false); + + const items = e.dataTransfer.items; + + const firstEntry = items[0].webkitGetAsEntry(); + + if (firstEntry?.isDirectory && items.length === 1) { + { + const readFile = (entry: any) => { + return new Promise((resolve) => { + entry.file((file: File) => { + if (filterTypeReg.test(file.name)) { + onSelectFile([file]); + } + resolve(file); + }); + }); + }; + const traverseFileTree = (dirReader: any) => { + return new Promise((resolve) => { + let fileNum = 0; + dirReader.readEntries(async (entries: any[]) => { + for await (const entry of entries) { + if (entry.isFile) { + await readFile(entry); + fileNum++; + } else if (entry.isDirectory) { + await traverseFileTree(entry.createReader()); + } + } + + // chrome: readEntries will return 100 entries at most + if (fileNum === 100) { + await traverseFileTree(dirReader); + } + resolve(''); + }); + }); + }; + + for await (const item of items) { + const entry = item.webkitGetAsEntry(); + if (entry) { + if (entry.isFile) { + await readFile(entry); + } else if (entry.isDirectory) { + //@ts-ignore + await traverseFileTree(entry.createReader()); + } + } + } + } + } else if (firstEntry?.isFile) { + const files = Array.from(e.dataTransfer.files); + let isErr = files.some((item) => item.type === ''); + if (isErr) { + return toast({ + title: t('file:upload_error_description'), + status: 'error' + }); + } + + onSelectFile(files.filter((item) => filterTypeReg.test(item.name))); + } else { + return toast({ + title: t('file:upload_error_description'), + status: 'error' + }); + } + }; + + return ( + e.preventDefault(), + onDragLeave: handleDragLeave, + onDrop: handleDrop, + onClick: onOpen + })} + {...props} + > + + {isMaxSelected ? ( + <> + + {t('file:reached_max_file_count')} + + + ) : ( + <> + + {isDragging + ? t('file:release_the_mouse_to_upload_the_file') + : t('file:select_and_drag_file_tip')} + + {/* file type */} + + {t('file:support_file_type', { fileType })} + + + {/* max count */} + {maxCount && t('file:support_max_count', { maxCount })} + {/* max size */} + {maxSize && t('file:support_max_size', { maxSize: formatFileSize(maxSize) })} + + + onSelectFile(files)} /> + + )} + + ); +}; + +export default React.memo(FileSelector); diff --git a/projects/app/src/pages/api/core/dataset/collection/create/backup.ts b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts new file mode 100644 index 000000000..03c580935 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/create/backup.ts @@ -0,0 +1,86 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { NextAPI } from '@/service/middleware/entry'; +import { getUploadModel } from '@fastgpt/service/common/file/multer'; +import { removeFilesByPaths } from '@fastgpt/service/common/file/utils'; +import { addLog } from '@fastgpt/service/common/system/log'; +import { readRawTextByLocalFile } from '@fastgpt/service/common/file/read/utils'; +import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; +import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; +import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; +import { + DatasetCollectionDataProcessModeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; + +export type backupQuery = {}; + +export type backupBody = {}; + +export type backupResponse = {}; + +async function handler(req: ApiRequestProps, res: ApiResponseType) { + const filePaths: string[] = []; + + try { + const upload = getUploadModel({ + maxSize: global.feConfigs?.uploadFileMaxSize + }); + const { file, data } = await upload.doUpload<{ datasetId: string }>(req, res); + filePaths.push(file.path); + + if (file.mimetype !== 'text/csv') { + throw new Error('File must be a CSV file'); + } + + const { teamId, tmbId, dataset } = await authDataset({ + req, + authToken: true, + authApiKey: true, + per: WritePermissionVal, + datasetId: data.datasetId + }); + + // 1. Read + const { rawText } = await readRawTextByLocalFile({ + teamId, + tmbId, + path: file.path, + encoding: file.encoding, + getFormatText: false + }); + if (!rawText.startsWith('q,a,indexes')) { + return Promise.reject('Backup file start with "q,a,indexes"'); + } + // 2. delete tmp file + removeFilesByPaths(filePaths); + + // 3. Create collection + await createCollectionAndInsertData({ + dataset, + rawText, + backupParse: true, + createCollectionParams: { + teamId, + tmbId, + datasetId: dataset._id, + name: file.originalname, + type: DatasetCollectionTypeEnum.virtual, + trainingType: DatasetCollectionDataProcessModeEnum.backup + } + }); + + return {}; + } catch (error) { + addLog.error(`Backup dataset collection create error: ${error}`); + removeFilesByPaths(filePaths); + return Promise.reject(error); + } +} + +export default NextAPI(handler); + +export const config = { + api: { + bodyParser: false + } +}; diff --git a/projects/app/src/pages/api/core/dataset/collection/create/csvTable.ts b/projects/app/src/pages/api/core/dataset/collection/create/csvTable.ts deleted file mode 100644 index 9b00e2ece..000000000 --- a/projects/app/src/pages/api/core/dataset/collection/create/csvTable.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { NextApiRequest } from 'next'; -import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller'; -import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; -import { type FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; -import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; -import { - DatasetCollectionDataProcessModeEnum, - DatasetCollectionTypeEnum, - TrainingModeEnum -} from '@fastgpt/global/core/dataset/constants'; -import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; -import { NextAPI } from '@/service/middleware/entry'; -import { type CreateCollectionResponse } from '@/global/core/dataset/api'; -import { MongoRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/schema'; - -async function handler(req: NextApiRequest): CreateCollectionResponse { - const { datasetId, parentId, fileId, ...body } = req.body as FileIdCreateDatasetCollectionParams; - const { teamId, tmbId, dataset } = await authDataset({ - req, - authToken: true, - authApiKey: true, - per: WritePermissionVal, - datasetId: datasetId - }); - - // 1. read file - const { rawText, filename } = await readFileContentFromMongo({ - teamId, - tmbId, - bucketName: BucketNameEnum.dataset, - fileId, - isQAImport: true - }); - - const { collectionId, insertResults } = await createCollectionAndInsertData({ - dataset, - rawText, - isQAImport: true, - createCollectionParams: { - ...body, - teamId, - tmbId, - name: filename, - parentId, - datasetId, - type: DatasetCollectionTypeEnum.file, - fileId, - - // special metadata - trainingType: DatasetCollectionDataProcessModeEnum.chunk, - chunkSize: 0 - } - }); - - // remove buffer - await MongoRawTextBuffer.deleteOne({ sourceId: fileId }); - - return { collectionId, results: insertResults }; -} -export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/collection/create/link.ts b/projects/app/src/pages/api/core/dataset/collection/create/link.ts index d2aa5cb6b..8f8bfa90f 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/link.ts @@ -2,15 +2,11 @@ import type { NextApiRequest } from 'next'; import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller'; -import { - TrainingModeEnum, - DatasetCollectionTypeEnum -} from '@fastgpt/global/core/dataset/constants'; +import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { type CreateCollectionResponse } from '@/global/core/dataset/api'; import { urlsFetch } from '@fastgpt/service/common/string/cheerio'; -import { hashStr } from '@fastgpt/global/common/string/tools'; async function handler(req: NextApiRequest): CreateCollectionResponse { const { link, ...body } = req.body as LinkCreateDatasetCollectionParams; diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index de0a8f000..23a484c58 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -1,5 +1,5 @@ /* push data to training queue */ -import type { NextApiRequest, NextApiResponse } from 'next'; +import type { NextApiResponse } from 'next'; import type { PushDatasetDataProps } from '@fastgpt/global/core/dataset/api.d'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit'; @@ -8,9 +8,10 @@ import { pushDataListToTrainingQueue } from '@fastgpt/service/core/dataset/train import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { getTrainingModeByCollection } from '@fastgpt/service/core/dataset/collection/utils'; +import type { ApiRequestProps } from '@fastgpt/service/type/next'; -async function handler(req: NextApiRequest, res: NextApiResponse) { - const body = req.body as PushDatasetDataProps; +async function handler(req: ApiRequestProps, res: NextApiResponse) { + const body = req.body; // Adapter 4.9.0 body.trainingType = body.trainingType || body.trainingMode; diff --git a/projects/app/src/pages/api/core/dataset/exportAll.ts b/projects/app/src/pages/api/core/dataset/exportAll.ts index 26e5de5ed..6c6800a7f 100644 --- a/projects/app/src/pages/api/core/dataset/exportAll.ts +++ b/projects/app/src/pages/api/core/dataset/exportAll.ts @@ -12,6 +12,14 @@ import { NextAPI } from '@/service/middleware/entry'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; +import type { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type'; + +type DataItemType = { + _id: string; + q: string; + a: string; + indexes: DatasetDataSchemaType['indexes']; +}; async function handler(req: NextApiRequest, res: NextApiResponse) { let { datasetId } = req.query as { @@ -23,7 +31,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { } // 凭证校验 - const { teamId } = await authDataset({ + const { teamId, dataset } = await authDataset({ req, authToken: true, datasetId, @@ -42,19 +50,14 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { }); res.setHeader('Content-Type', 'text/csv; charset=utf-8;'); - res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; '); + res.setHeader('Content-Disposition', `attachment; filename=${dataset.name}-backup.csv;`); - const cursor = MongoDatasetData.find<{ - _id: string; - collectionId: { name: string }; - q: string; - a: string; - }>( + const cursor = MongoDatasetData.find( { teamId, datasetId: { $in: datasets.map((d) => d._id) } }, - 'q a', + 'q a indexes', { ...readFromSecondary } @@ -67,13 +70,14 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { readStream: cursor }); - write(`\uFEFFindex,content`); + write(`\uFEFFq,a,indexes`); - cursor.on('data', (doc) => { + cursor.on('data', (doc: DataItemType) => { const q = doc.q.replace(/"/g, '""') || ''; const a = doc.a.replace(/"/g, '""') || ''; + const indexes = doc.indexes.map((i) => `"${i.text.replace(/"/g, '""')}"`).join(','); - write(`\n"${q}","${a}"`); + write(`\n"${q}","${a}",${indexes}`); }); cursor.on('end', () => { diff --git a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts index b11a9aff3..6aab2139b 100644 --- a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts +++ b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts @@ -1,7 +1,7 @@ -import type { - ChunkSettingModeEnum, - DataChunkSplitModeEnum, - DatasetCollectionDataProcessModeEnum +import { + type ChunkSettingModeEnum, + type DataChunkSplitModeEnum, + type DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants'; import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read'; @@ -39,7 +39,6 @@ export type PostPreviewFilesChunksProps = { // Read params selector?: string; - isQAImport?: boolean; externalFileId?: string; }; export type PreviewChunksResponse = { @@ -66,7 +65,6 @@ async function handler( overlapRatio, selector, - isQAImport, datasetId, externalFileId } = req.body; @@ -118,7 +116,6 @@ async function handler( type, sourceId, selector, - isQAImport, apiServer: dataset.apiServer, feishuServer: dataset.feishuServer, yuqueServer: dataset.yuqueServer, @@ -131,9 +128,9 @@ async function handler( chunkSize, maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)), overlapRatio, - customReg: chunkSplitter ? [chunkSplitter] : [], - isQAImport: isQAImport + customReg: chunkSplitter ? [chunkSplitter] : [] }); + return { chunks: chunks.slice(0, 10), total: chunks.length diff --git a/projects/app/src/pages/login/index.tsx b/projects/app/src/pages/login/index.tsx index 97dc769a4..e5553e8fd 100644 --- a/projects/app/src/pages/login/index.tsx +++ b/projects/app/src/pages/login/index.tsx @@ -29,7 +29,6 @@ import { GET } from '@/web/common/api/request'; import { getDocPath } from '@/web/common/system/doc'; import { getWebReqUrl } from '@fastgpt/web/common/system/utils'; import LoginForm from '@/pageComponents/login/LoginForm/LoginForm'; -import { useToast } from '@fastgpt/web/hooks/useToast'; import { getBdVId } from '@/web/support/marketing/utils'; const RegisterForm = dynamic(() => import('@/pageComponents/login/RegisterForm')); @@ -49,7 +48,6 @@ const Login = ({ ChineseRedirectUrl }: { ChineseRedirectUrl: string }) => { const { setLastChatAppId } = useChatStore(); const { isOpen, onOpen, onClose } = useDisclosure(); const { isPc } = useSystem(); - const { toast } = useToast(); const { isOpen: isOpenCookiesDrawer, diff --git a/projects/app/src/service/common/system/volumnMongoWatch.ts b/projects/app/src/service/common/system/volumnMongoWatch.ts index 85e3a825e..342986079 100644 --- a/projects/app/src/service/common/system/volumnMongoWatch.ts +++ b/projects/app/src/service/common/system/volumnMongoWatch.ts @@ -23,9 +23,11 @@ const reloadConfigWatch = () => { changeStream.on('change', async (change) => { try { if ( + change.operationType === 'update' || (change.operationType === 'insert' && - change.fullDocument.type === SystemConfigsTypeEnum.fastgptPro) || - change.operationType === 'update' + [SystemConfigsTypeEnum.fastgptPro, SystemConfigsTypeEnum.license].includes( + change.fullDocument.type + )) ) { await initSystemConfig(); console.log('refresh system config'); diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index 488fc3583..dea1bbeca 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -11,7 +11,7 @@ import { type DatasetDataIndexItemType, type DatasetDataItemType } from '@fastgpt/global/core/dataset/type'; -import { getEmbeddingModel, getLLMModel } from '@fastgpt/service/core/ai/model'; +import { getEmbeddingModel } from '@fastgpt/service/core/ai/model'; import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun'; import { type ClientSession } from '@fastgpt/service/common/mongo'; import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema'; @@ -93,13 +93,15 @@ const formatIndexes = async ({ return item; } }); - indexes = indexes.filter((item) => item.type !== DatasetDataIndexTypeEnum.default); - indexes.push(...concatDefaultIndexes); - // Remove same text + // 其他索引不能与默认索引相同，且不能自己有重复 indexes = indexes.filter( - (item, index, self) => index === self.findIndex((t) => t.text === item.text) + (item, index, self) => + item.type !== DatasetDataIndexTypeEnum.default && + !concatDefaultIndexes.find((t) => t.text === item.text) && + index === self.findIndex((t) => t.text === item.text) ); + indexes.push(...concatDefaultIndexes); const chekcIndexes = ( await Promise.all( diff --git a/projects/app/src/service/events/generateVector.ts b/projects/app/src/service/events/generateVector.ts index 8c0783115..9df13e073 100644 --- a/projects/app/src/service/events/generateVector.ts +++ b/projects/app/src/service/events/generateVector.ts @@ -262,6 +262,7 @@ const insertData = async ({ q: trainingData.q, a: trainingData.a, chunkIndex: trainingData.chunkIndex, + indexSize: trainingData.indexSize, indexes: trainingData.indexes, embeddingModel: trainingData.model, session diff --git a/projects/app/src/web/core/dataset/api.ts b/projects/app/src/web/core/dataset/api.ts index 2959d1d91..65733c2e7 100644 --- a/projects/app/src/web/core/dataset/api.ts +++ b/projects/app/src/web/core/dataset/api.ts @@ -1,7 +1,6 @@ import { GET, POST, PUT, DELETE } from '@/web/common/api/request'; import type { GetPathProps, - ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d'; import type { @@ -120,6 +119,33 @@ export const resumeInheritPer = (datasetId: string) => export const postChangeOwner = (data: { ownerId: string; datasetId: string }) => POST(`/proApi/core/dataset/changeOwner`, data); +export const postBackupDatasetCollection = ({ + file, + percentListen, + datasetId +}: { + file: File; + percentListen: (percent: number) => void; + datasetId: string; +}) => { + const formData = new FormData(); + formData.append('file', file, encodeURIComponent(file.name)); + formData.append('data', JSON.stringify({ datasetId })); + + return POST(`/core/dataset/collection/create/backup`, formData, { + timeout: 600000, + onUploadProgress: (e) => { + if (!e.total) return; + + const percent = Math.round((e.loaded / e.total) * 100); + percentListen?.(percent); + }, + headers: { + 'Content-Type': 'multipart/form-data; charset=utf-8' + } + }); +}; + /* =========== search test ============ */ export const postSearchText = (data: SearchTestProps) => POST(`/core/dataset/searchTest`, data); @@ -149,10 +175,7 @@ export const postCreateDatasetLinkCollection = (data: LinkCreateDatasetCollectio POST<{ collectionId: string }>(`/core/dataset/collection/create/link`, data); export const postCreateDatasetTextCollection = (data: TextCreateDatasetCollectionParams) => POST<{ collectionId: string }>(`/core/dataset/collection/create/text`, data); -export const postCreateDatasetCsvTableCollection = (data: CsvTableCreateDatasetCollectionParams) => - POST<{ collectionId: string }>(`/core/dataset/collection/create/csvTable`, data, { - timeout: 360000 - }); + export const postCreateDatasetExternalFileCollection = ( data: ExternalFileCreateDatasetCollectionParams ) =>