From cd876251b70deb2ccd6171f6de82e33a036ab948 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Wed, 15 May 2024 10:19:51 +0800 Subject: [PATCH] External dataset (#1485) * fix: revert version * feat: external collection * import context * external ui * doc * fix: ts * clear invalid data * feat: rename sub name * fix: node if else edge remove * fix: init * api size * fix: if else node refresh --- .vscode/nextapi.code-snippets | 13 +- Dockerfile | 2 + docSite/content/docs/development/faq.md | 3 +- .../content/docs/development/upgrading/481.md | 3 +- packages/global/core/dataset/api.d.ts | 8 +- .../core/dataset/collection/constants.ts | 6 + packages/global/core/dataset/constants.ts | 17 +- packages/global/core/dataset/type.d.ts | 23 +- packages/global/core/dataset/utils.ts | 2 +- .../template/system/assignedAnswer.ts | 1 + .../service/core/dataset/collection/schema.ts | 25 +- packages/service/core/dataset/schema.ts | 3 +- packages/service/support/wallet/sub/schema.ts | 2 +- .../components/common/DndDrag/DragIcon.tsx | 6 +- projects/app/.eslintrc.json | 8 +- projects/app/i18n/en/common.json | 7 +- projects/app/i18n/en/dataset.json | 13 +- projects/app/i18n/zh/common.json | 6 +- projects/app/i18n/zh/dataset.json | 13 +- .../src/components/common/NextHead/index.tsx | 14 + .../core/dataset/DatasetTypeTag.tsx | 9 +- .../Flow/nodes/NodeIfElse/ListItem.tsx | 146 +++--- .../workflow/Flow/nodes/NodeIfElse/index.tsx | 9 +- .../components/PublishHistoriesSlider.tsx | 35 +- .../src/components/core/workflow/context.tsx | 4 +- projects/app/src/global/core/dataset/api.d.ts | 4 +- projects/app/src/pages/_app.tsx | 26 +- .../src/pages/api/admin/clearInvalidData.ts | 13 +- projects/app/src/pages/api/admin/initv481.ts | 32 +- .../src/pages/api/common/file/uploadImage.ts | 1 - .../src/pages/api/core/dataset/allDataset.ts | 54 +-- .../app/src/pages/api/core/dataset/list.ts | 2 +- .../app/src/pages/api/core/dataset/update.ts | 17 +- .../app/src/pages/api/core/plugin/list.ts | 2 +- .../app/src/pages/api/core/workflow/debug.ts | 2 +- .../pages/api/plugins/textEditor/v2/index.ts | 3 + .../app/src/pages/api/v1/chat/completions.ts | 3 + projects/app/src/pages/chat/share.tsx | 250 +++++----- .../components/CollectionCard/Context.tsx | 158 +++++++ .../CollectionCard/EmptyCollectionTip.tsx | 55 +++ .../components/CollectionCard/Header.tsx | 399 ++++++++++++++++ .../WebsiteConfig.tsx | 0 .../index.tsx} | 437 +----------------- .../detail/components/Import/Context.tsx | 302 ++++++++++++ .../detail/components/Import/Provider.tsx | 165 ------- .../Import/commonProgress/DataProcess.tsx | 15 +- .../Import/commonProgress/PreviewData.tsx | 11 +- .../Import/commonProgress/Upload.tsx | 10 +- .../Import/components/FileSourceSelector.tsx | 2 +- .../components/Import/components/Preview.tsx | 5 +- .../Import/components/PreviewChunks.tsx | 8 +- .../Import/components/PreviewRawText.tsx | 5 +- .../Import/diffSource/ExternalFile.tsx | 188 ++++++++ .../Import/diffSource/FileCustomText.tsx | 15 +- .../components/Import/diffSource/FileLink.tsx | 20 +- .../Import/diffSource/FileLocal.tsx | 17 +- .../Import/diffSource/TableLocal.tsx | 17 +- .../detail/components/Import/index.tsx | 155 +------ .../pages/dataset/detail/components/Info.tsx | 62 ++- .../dataset/detail/components/Slider.tsx | 52 +-- .../pages/dataset/detail/components/Test.tsx | 4 +- .../app/src/pages/dataset/detail/index.tsx | 79 ++-- .../dataset/list/component/CreateModal.tsx | 18 +- .../dataset/list/component/MoveModal.tsx | 5 +- projects/app/src/pages/dataset/list/index.tsx | 33 +- .../src/service/support/wallet/usage/push.ts | 2 +- projects/app/src/web/core/dataset/api.ts | 10 +- .../dataset/components/SelectCollections.tsx | 5 +- .../app/src/web/core/dataset/constants.ts | 7 +- .../dataset/context/datasetPageContext.tsx | 62 ++- .../core/dataset/context/datasetsContext.tsx | 11 + .../dataset/context/useDatasetContext.tsx | 18 - .../app/src/web/core/dataset/store/dataset.ts | 83 +--- projects/app/src/web/core/dataset/type.d.ts | 13 +- 74 files changed, 1882 insertions(+), 1353 deletions(-) create mode 100644 packages/global/core/dataset/collection/constants.ts create mode 100644 projects/app/src/components/common/NextHead/index.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx rename projects/app/src/pages/dataset/detail/components/{Import => CollectionCard}/WebsiteConfig.tsx (100%) rename projects/app/src/pages/dataset/detail/components/{CollectionCard.tsx => CollectionCard/index.tsx} (52%) create mode 100644 projects/app/src/pages/dataset/detail/components/Import/Context.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/Provider.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx create mode 100644 projects/app/src/web/core/dataset/context/datasetsContext.tsx delete mode 100644 projects/app/src/web/core/dataset/context/useDatasetContext.tsx diff --git a/.vscode/nextapi.code-snippets b/.vscode/nextapi.code-snippets index 19864409b..e597e0bac 100644 --- a/.vscode/nextapi.code-snippets +++ b/.vscode/nextapi.code-snippets @@ -40,18 +40,11 @@ "", "type ContextType = {$1};", "", - "type ContextValueType = {};", - "", "export const Context = createContext({});", "", - "export const ContextProvider = ({", - " children,", - " value", - "}: {", - " children: ReactNode;", - " value: ContextValueType;", - "}) => {", - " return {children};", + "export const ContextProvider = ({ children }: { children: ReactNode }) => {", + " const contextValue: ContextType = {};", + " return {children};", "};", ], "description": "FastGPT usecontext template" diff --git a/Dockerfile b/Dockerfile index 2e97b1489..6a4be70a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,8 @@ COPY --from=mainDeps /app/projects/$name/node_modules ./projects/$name/node_modu RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0 + +ENV NODE_OPTIONS="--max-old-space-size=4096" RUN pnpm --filter=$name build # --------- runner ----------- diff --git a/docSite/content/docs/development/faq.md b/docSite/content/docs/development/faq.md index 8733145db..e1bada463 100644 --- a/docSite/content/docs/development/faq.md +++ b/docSite/content/docs/development/faq.md @@ -118,4 +118,5 @@ OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并 ### bad_response_status_code bad response status code 503 1. 模型服务不可用 -2. .... \ No newline at end of file +2. 模型接口参数异常（温度、max token等可能不适配） +3. .... \ No newline at end of file diff --git a/docSite/content/docs/development/upgrading/481.md b/docSite/content/docs/development/upgrading/481.md index 1fa86b622..21a391593 100644 --- a/docSite/content/docs/development/upgrading/481.md +++ b/docSite/content/docs/development/upgrading/481.md @@ -35,4 +35,5 @@ curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \ ## V4.8.1 更新说明 1. 新增 - 知识库重新选择向量模型重建 -2. 修复 - 定时器清理脏数据任务 \ No newline at end of file +2. 修复 - 工作流删除节点的动态输入和输出时候，没有正确的删除连接线，导致可能出现逻辑异常。 +3. 修复 - 定时器清理脏数据任务 \ No newline at end of file diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 7a34b6c8c..af0eb5e76 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -11,14 +11,16 @@ export type DatasetUpdateBody = { intro?: string; permission?: DatasetSchemaType['permission']; agentModel?: LLMModelItemType; - websiteConfig?: DatasetSchemaType['websiteConfig']; status?: DatasetSchemaType['status']; + + websiteConfig?: DatasetSchemaType['websiteConfig']; + externalReadUrl?: DatasetSchemaType['externalReadUrl']; }; /* ================= collection ===================== */ export type DatasetCollectionChunkMetadataType = { parentId?: string; - trainingType?: `${TrainingModeEnum}`; + trainingType?: TrainingModeEnum; chunkSize?: number; chunkSplitter?: string; qaPrompt?: string; @@ -78,7 +80,7 @@ export type PostWebsiteSyncParams = { export type PushDatasetDataProps = { collectionId: string; data: PushDatasetDataChunkProps[]; - trainingMode: `${TrainingModeEnum}`; + trainingMode: TrainingModeEnum; prompt?: string; billId?: string; }; diff --git a/packages/global/core/dataset/collection/constants.ts b/packages/global/core/dataset/collection/constants.ts new file mode 100644 index 000000000..0b0cda0b1 --- /dev/null +++ b/packages/global/core/dataset/collection/constants.ts @@ -0,0 +1,6 @@ +/* sourceId = prefix-id; id=fileId;link url;externalId */ +export enum CollectionSourcePrefixEnum { + local = 'local', + link = 'link', + external = 'external' +} diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 0e8be0f5d..570fc1417 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -2,23 +2,29 @@ export enum DatasetTypeEnum { folder = 'folder', dataset = 'dataset', - websiteDataset = 'websiteDataset' // depp link + websiteDataset = 'websiteDataset', // depp link + externalFile = 'externalFile' } export const DatasetTypeMap = { [DatasetTypeEnum.folder]: { icon: 'common/folderFill', - label: 'core.dataset.Folder Dataset', + label: 'Folder Dataset', collectionLabel: 'common.Folder' }, [DatasetTypeEnum.dataset]: { icon: 'core/dataset/commonDataset', - label: 'core.dataset.Common Dataset', + label: 'Common Dataset', collectionLabel: 'common.File' }, [DatasetTypeEnum.websiteDataset]: { icon: 'core/dataset/websiteDataset', - label: 'core.dataset.Website Dataset', + label: 'Website Dataset', collectionLabel: 'common.Website' + }, + [DatasetTypeEnum.externalFile]: { + icon: 'core/dataset/commonDataset', + label: 'External File', + collectionLabel: 'common.File' } }; @@ -77,7 +83,8 @@ export enum ImportDataSourceEnum { fileLocal = 'fileLocal', fileLink = 'fileLink', fileCustom = 'fileCustom', - csvTable = 'csvTable' + csvTable = 'csvTable', + externalFile = 'externalFile' } export enum TrainingModeEnum { diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 7d89b322c..65cd20a92 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -22,13 +22,16 @@ export type DatasetSchemaType = { vectorModel: string; agentModel: string; intro: string; - type: `${DatasetTypeEnum}`; + type: DatasetTypeEnum; status: `${DatasetStatusEnum}`; permission: `${PermissionTypeEnum}`; + + // metadata websiteConfig?: { url: string; selector: string; }; + externalReadUrl?: string; }; export type DatasetCollectionSchemaType = { @@ -42,16 +45,18 @@ export type DatasetCollectionSchemaType = { createTime: Date; updateTime: Date; - trainingType: `${TrainingModeEnum}`; + trainingType: TrainingModeEnum; chunkSize: number; chunkSplitter?: string; qaPrompt?: string; - fileId?: string; - rawLink?: string; + sourceId?: string; // relate CollectionSourcePrefixEnum + fileId?: string; // local file id + rawLink?: string; // link url rawTextLength?: number; hashRawText?: string; + externalSourceUrl?: string; // external import url metadata?: { webPageSelector?: string; relatedImgId?: string; // The id of the associated image collections @@ -93,7 +98,7 @@ export type DatasetTrainingSchemaType = { billId: string; expireAt: Date; lockTime: Date; - mode: `${TrainingModeEnum}`; + mode: TrainingModeEnum; model: string; prompt: string; dataId?: string; @@ -112,13 +117,19 @@ export type DatasetDataWithCollectionType = Omit { +export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => { if (mode === TrainingModeEnum.qa) return data.length * 20; if (mode === TrainingModeEnum.auto) return data.length * 5; return data.length; diff --git a/packages/global/core/workflow/template/system/assignedAnswer.ts b/packages/global/core/workflow/template/system/assignedAnswer.ts index 6f133b557..26c6f6bae 100644 --- a/packages/global/core/workflow/template/system/assignedAnswer.ts +++ b/packages/global/core/workflow/template/system/assignedAnswer.ts @@ -18,6 +18,7 @@ export const AssignedAnswerModule: FlowNodeTemplateType = { intro: '该模块可以直接回复一段指定的内容。常用于引导、提示。非字符串内容传入时，会转成字符串进行输出。', version: '481', + isTool: true, inputs: [ { key: NodeInputKeyEnum.answerText, diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index e8cb6e53a..dea33a72b 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -16,11 +16,6 @@ const DatasetCollectionSchema = new Schema({ ref: DatasetColCollectionName, default: null }, - userId: { - // abandoned - type: Schema.Types.ObjectId, - ref: 'user' - }, teamId: { type: Schema.Types.ObjectId, ref: TeamCollectionName, @@ -54,6 +49,7 @@ const DatasetCollectionSchema = new Schema({ default: () => new Date() }, + // chunk filed trainingType: { type: String, enum: Object.keys(TrainingTypeMap), @@ -70,20 +66,21 @@ const DatasetCollectionSchema = new Schema({ type: String }, + sourceId: String, + // local file collection fileId: { type: Schema.Types.ObjectId, ref: 'dataset.files' }, - rawLink: { - type: String - }, + // web link collection + rawLink: String, - rawTextLength: { - type: Number - }, - hashRawText: { - type: String - }, + // external collection + + // metadata + rawTextLength: Number, + hashRawText: String, + externalSourceUrl: String, // external import url metadata: { type: Object, default: {} diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 4139e042b..95f81daf6 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -89,7 +89,8 @@ const DatasetSchema = new Schema({ default: 'body' } } - } + }, + externalReadUrl: String }); try { diff --git a/packages/service/support/wallet/sub/schema.ts b/packages/service/support/wallet/sub/schema.ts index b90cd9b4b..42709e802 100644 --- a/packages/service/support/wallet/sub/schema.ts +++ b/packages/service/support/wallet/sub/schema.ts @@ -14,7 +14,7 @@ import { } from '@fastgpt/global/support/wallet/sub/constants'; import type { TeamSubSchema } from '@fastgpt/global/support/wallet/sub/type'; -export const subCollectionName = 'team.subscriptions'; +export const subCollectionName = 'team_subscriptions'; const SubSchema = new Schema({ teamId: { diff --git a/packages/web/components/common/DndDrag/DragIcon.tsx b/packages/web/components/common/DndDrag/DragIcon.tsx index 493a409d2..36eedd13c 100644 --- a/packages/web/components/common/DndDrag/DragIcon.tsx +++ b/packages/web/components/common/DndDrag/DragIcon.tsx @@ -1,11 +1,11 @@ import { DragHandleIcon } from '@chakra-ui/icons'; -import { Box } from '@chakra-ui/react'; +import { Box, BoxProps } from '@chakra-ui/react'; import React from 'react'; import { DraggableProvided } from 'react-beautiful-dnd'; -const DragIcon = ({ provided }: { provided: DraggableProvided }) => { +const DragIcon = ({ provided, ...props }: { provided: DraggableProvided } & BoxProps) => { return ( - + ); diff --git a/projects/app/.eslintrc.json b/projects/app/.eslintrc.json index 3a5d07f92..be661eb5c 100644 --- a/projects/app/.eslintrc.json +++ b/projects/app/.eslintrc.json @@ -1,12 +1,6 @@ - { - "parser": "@typescript-eslint/parser", // 确保使用了 TypeScript 解析器 - "plugins": ["@typescript-eslint"], // 引入 TypeScript 插件 - "extends": "next/core-web-vitals", "rules": { - "react-hooks/rules-of-hooks": 0, - "@typescript-eslint/consistent-type-imports": "warn" // 或者 "error" 来强制执行 - + "react-hooks/rules-of-hooks": 0 } } diff --git a/projects/app/i18n/en/common.json b/projects/app/i18n/en/common.json index e8de25278..14fe00397 100644 --- a/projects/app/i18n/en/common.json +++ b/projects/app/i18n/en/common.json @@ -1,4 +1,5 @@ { + "Add new": "Add new", "App": "App", "Export": "Export", "Folder": "Folder", @@ -509,18 +510,14 @@ "Choose Dataset": "Associate dataset", "Chunk amount": "Number of chunks", "Collection": "Dataset", - "Common Dataset": "Common dataset", - "Common Dataset Desc": "Can be built by importing files, web links, or manual entry", "Create dataset": "Create a dataset", "Dataset": "Dataset", "Dataset ID": "Dataset ID", "Dataset Type": "Dataset type", "Delete Confirm": "Confirm to delete this dataset? Data cannot be recovered after deletion, please confirm!", "Delete Website Tips": "Confirm to delete this site?", - "Empty Dataset": "", "Empty Dataset Tips": "No datasets yet, go create one!", "File collection": "File dataset", - "Folder Dataset": "Folder", "Folder placeholder": "This is a directory", "Go Dataset": "Go to dataset", "Intro Placeholder": "This dataset has no introduction~", @@ -540,8 +537,6 @@ "Table collection": "Table dataset", "Text collection": "Text dataset", "Total chunks": "Total chunks: {{total}}", - "Website Dataset": "Web site synchronization", - "Website Dataset Desc": "Web site synchronization allows you to use a web page link to build a dataset", "collection": { "Click top config website": "Click to configure website", "Collection name": "Dataset name", diff --git a/projects/app/i18n/en/dataset.json b/projects/app/i18n/en/dataset.json index 0c2bc01d8..510955b1c 100644 --- a/projects/app/i18n/en/dataset.json +++ b/projects/app/i18n/en/dataset.json @@ -1,6 +1,17 @@ { + "Common Dataset": "Common dataset", + "Common Dataset Desc": "Can be built by importing files, web links, or manual entry", "Confirm to rebuild embedding tip": "Are you sure to switch the knowledge base index? Switching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.", + "External file": "External file", + "External file Dataset Desc": "You can import files from an external file library to build a knowledge base. Files are not stored twice", + "External id": "File id", + "External read url": "External read url", + "External url": "File read url", + "Folder Dataset": "Folder", "Rebuild embedding start tip": "The task of switching index models has begun", "Rebuilding index count": "Rebuilding count: {{count}}", - "The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt" + "The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt", + "Website Dataset": "Web site", + "Website Dataset Desc": "Web site synchronization allows you to use a web page link to build a dataset", + "filename": "filename" } diff --git a/projects/app/i18n/zh/common.json b/projects/app/i18n/zh/common.json index 55ed8ff92..ca14b35f8 100644 --- a/projects/app/i18n/zh/common.json +++ b/projects/app/i18n/zh/common.json @@ -1,4 +1,5 @@ { + "Add new": "新增", "App": "应用", "Export": "导出", "Folder": "文件夹", @@ -509,8 +510,6 @@ "Choose Dataset": "关联知识库", "Chunk amount": "分段数", "Collection": "数据集", - "Common Dataset": "通用知识库", - "Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库", "Create dataset": "创建一个知识库", "Dataset": "知识库", "Dataset ID": "知识库 ID", @@ -520,7 +519,6 @@ "Empty Dataset": "", "Empty Dataset Tips": "还没有知识库，快去创建一个吧！", "File collection": "文件数据集", - "Folder Dataset": "文件夹", "Folder placeholder": "这是一个目录", "Go Dataset": "前往知识库", "Intro Placeholder": "这个知识库还没有介绍~", @@ -540,8 +538,6 @@ "Table collection": "表格数据集", "Text collection": "文本数据集", "Total chunks": "总分段: {{total}}", - "Website Dataset": "Web 站点同步", - "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", "collection": { "Click top config website": "点击配置网站", "Collection name": "数据集名称", diff --git a/projects/app/i18n/zh/dataset.json b/projects/app/i18n/zh/dataset.json index 4289048f7..d07a16bbe 100644 --- a/projects/app/i18n/zh/dataset.json +++ b/projects/app/i18n/zh/dataset.json @@ -1,6 +1,17 @@ { + "Common Dataset": "通用知识库", + "Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库", "Confirm to rebuild embedding tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。", + "External File": "外部文件库", + "External file Dataset Desc": "可以从外部文件库导入文件构建知识库，文件不会进行二次存储", + "External id": "文件阅读ID", + "External read url": "外部预览地址", + "External url": "文件访问URL", + "Folder Dataset": "文件夹", "Rebuild embedding start tip": "切换索引模型任务已开始", "Rebuilding index count": "重建中索引数量: {{count}}", - "The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引" + "The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引", + "Website Dataset": "Web 站点同步", + "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", + "filename": "文件名" } diff --git a/projects/app/src/components/common/NextHead/index.tsx b/projects/app/src/components/common/NextHead/index.tsx new file mode 100644 index 000000000..a3f529bc0 --- /dev/null +++ b/projects/app/src/components/common/NextHead/index.tsx @@ -0,0 +1,14 @@ +import Head from 'next/head'; +import React from 'react'; + +const NextHead = ({ title, icon, desc }: { title?: string; icon?: string; desc?: string }) => { + return ( + + {title} + {desc && } + {icon && } + + ); +}; + +export default NextHead; diff --git a/projects/app/src/components/core/dataset/DatasetTypeTag.tsx b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx index 55122d094..f31adebfb 100644 --- a/projects/app/src/components/core/dataset/DatasetTypeTag.tsx +++ b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx @@ -1,12 +1,12 @@ import { Box, Flex, FlexProps } from '@chakra-ui/react'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import MyIcon from '@fastgpt/web/components/common/Icon'; -import { useTranslation } from 'next-i18next'; import React from 'react'; import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constants'; +import { useI18n } from '@/web/context/I18n'; -const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => { - const { t } = useTranslation(); +const DatasetTypeTag = ({ type, ...props }: { type: DatasetTypeEnum } & FlexProps) => { + const { datasetT } = useI18n(); const item = DatasetTypeMap[type] || DatasetTypeMap['dataset']; @@ -22,7 +22,8 @@ const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & Fle {...props} > - {t(item.label)} + {/* @ts-ignore */} + {datasetT(item.label)} ); }; diff --git a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx index 5f23a34f7..33c8e4574 100644 --- a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx +++ b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx @@ -50,16 +50,11 @@ const ListItem = ({ }) => { const { t } = useTranslation(); const { getZoom } = useReactFlow(); + const onDelEdge = useContextSelector(WorkflowContext, (v) => v.onDelEdge); + const handleId = getHandleId(nodeId, 'source', getElseIFLabel(conditionIndex)); - return ( - + const Render = useMemo(() => { + return ( - {ifElseList.length > 1 && } + 1 ? 'visible' : 'hidden'} + provided={provided} + /> {getElseIFLabel(conditionIndex)} @@ -109,6 +107,10 @@ const ListItem = ({ color={'myGray.400'} onClick={() => { onUpdateIfElseList(ifElseList.filter((_, index) => index !== conditionIndex)); + onDelEdge({ + nodeId, + sourceHandle: handleId + }); }} /> )} @@ -185,21 +187,21 @@ const ListItem = ({ onChange={(e) => { onUpdateIfElseList( ifElseList.map((ifElse, index) => { - if (index === conditionIndex) { - return { - ...ifElse, - list: ifElse.list.map((item, index) => { - if (index === i) { - return { - ...item, - value: e - }; - } - return item; - }) - }; - } - return ifElse; + return { + ...ifElse, + list: + index === conditionIndex + ? ifElse.list.map((item, index) => { + if (index === i) { + return { + ...item, + value: e + }; + } + return item; + }) + : ifElse.list + }; }) ); }} @@ -263,12 +265,38 @@ const ListItem = ({ {!snapshot.isDragging && ( )} + ); + }, [ + conditionIndex, + conditionItem.condition, + conditionItem.list, + getZoom, + handleId, + ifElseList, + nodeId, + onDelEdge, + onUpdateIfElseList, + provided, + snapshot.isDragging, + t + ]); + + return ( + + {Render} ); }; @@ -387,35 +415,39 @@ const ConditionValueInput = ({ return output.valueType; }, [nodeList, variable]); - if (valueType === WorkflowIOValueTypeEnum.boolean) { - return ( - - ); - } else { - return ( - onChange(e.target.value)} - /> - ); - } + const Render = useMemo(() => { + if (valueType === WorkflowIOValueTypeEnum.boolean) { + return ( + + ); + } else { + return ( + onChange(e.target.value)} + /> + ); + } + }, [condition, onChange, value, valueType]); + + return Render; }; diff --git a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx index 026245d98..f393a0c36 100644 --- a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx +++ b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo } from 'react'; import NodeCard from '../render/NodeCard'; import { useTranslation } from 'next-i18next'; import { Box, Button, Flex } from '@chakra-ui/react'; @@ -9,7 +9,7 @@ import { IfElseListItemType } from '@fastgpt/global/core/workflow/template/syste import { useContextSelector } from 'use-context-selector'; import { WorkflowContext } from '../../../context'; import Container from '../../components/Container'; -import DndDrag, { Draggable, DropResult } from '@fastgpt/web/components/common/DndDrag/index'; +import DndDrag, { Draggable } from '@fastgpt/web/components/common/DndDrag/index'; import { SourceHandle } from '../render/Handle'; import { getHandleId } from '@fastgpt/global/core/workflow/utils'; import ListItem from './ListItem'; @@ -19,6 +19,7 @@ const NodeIfElse = ({ data, selected }: NodeProps) => { const { t } = useTranslation(); const { nodeId, inputs = [] } = data; const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode); + const elseHandleId = getHandleId(nodeId, 'source', IfElseResultEnum.ELSE); const ifElseList = useMemo( () => @@ -49,7 +50,7 @@ const NodeIfElse = ({ data, selected }: NodeProps) => { - onDragEndCb={(list) => onUpdateIfElseList(list)} + onDragEndCb={(list: IfElseListItemType[]) => onUpdateIfElseList(list)} dataList={ifElseList} renderClone={(provided, snapshot, rubric) => ( ) => { diff --git a/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx b/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx index 80617fde5..c38df1523 100644 --- a/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx +++ b/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useCallback, useState } from 'react'; import { getPublishList, postRevertVersion } from '@/web/core/app/versionApi'; import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; import CustomRightDrawer from '@fastgpt/web/components/common/MyDrawer/CustomRightDrawer'; @@ -14,6 +14,8 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type'; +import { StoreEdgeItemType } from '@fastgpt/global/core/workflow/type/edge'; const PublishHistoriesSlider = () => { const { t } = useTranslation(); @@ -45,29 +47,29 @@ const PublishHistoriesSlider = () => { setIsShowVersionHistories(false); }); - const onPreview = useMemoizedFn((data: AppVersionSchemaType) => { + const onPreview = useCallback((data: AppVersionSchemaType) => { setSelectedHistoryId(data._id); initData({ nodes: data.nodes, edges: data.edges }); - }); - const onCloseSlider = useMemoizedFn(() => { - setSelectedHistoryId(undefined); - initData({ - nodes: appDetail.modules, - edges: appDetail.edges - }); - onClose(); - }); + }, []); + const onCloseSlider = useCallback( + (data: { nodes: StoreNodeItemType[]; edges: StoreEdgeItemType[] }) => { + setSelectedHistoryId(undefined); + initData(data); + onClose(); + }, + [appDetail] + ); const { mutate: onRevert, isLoading: isReverting } = useRequest({ mutationFn: async (data: AppVersionSchemaType) => { if (!appId) return; await postRevertVersion(appId, { versionId: data._id, - editNodes: appDetail.modules, + editNodes: appDetail.modules, // old workflow editEdges: appDetail.edges }); @@ -77,7 +79,7 @@ const PublishHistoriesSlider = () => { edges: data.edges }); - onCloseSlider(); + onCloseSlider(data); } }); @@ -86,7 +88,12 @@ const PublishHistoriesSlider = () => { return ( <> + onCloseSlider({ + nodes: appDetail.modules, + edges: appDetail.edges + }) + } iconSrc="core/workflow/versionHistories" title={t('core.workflow.publish.histories')} maxW={'300px'} diff --git a/projects/app/src/components/core/workflow/context.tsx b/projects/app/src/components/core/workflow/context.tsx index a2ffd6d81..faf5c1cd1 100644 --- a/projects/app/src/components/core/workflow/context.tsx +++ b/projects/app/src/components/core/workflow/context.tsx @@ -430,8 +430,8 @@ const WorkflowContextProvider = ({ const initData = useMemoizedFn( async (e: { nodes: StoreNodeItemType[]; edges: StoreEdgeItemType[] }) => { - setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item }))); - setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item }))); + setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item })) || []); + setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || []); } ); diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index 69ef8c9ec..e5fb7cf9f 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -14,7 +14,7 @@ import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; /* ================= dataset ===================== */ export type CreateDatasetParams = { parentId?: string; - type: `${DatasetTypeEnum}`; + type: DatasetTypeEnum; name: string; intro: string; avatar: string; @@ -76,7 +76,7 @@ export type SearchTestResponse = { /* =========== training =========== */ export type PostPreviewFilesChunksProps = { - type: `${ImportDataSourceEnum}`; + type: ImportDataSourceEnum; sourceId: string; chunkSize: number; overlapRatio: number; diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx index 69f3d589d..c8b133c8f 100644 --- a/projects/app/src/pages/_app.tsx +++ b/projects/app/src/pages/_app.tsx @@ -10,28 +10,22 @@ import I18nContextProvider from '@/web/context/I18n'; import { useInitApp } from '@/web/context/useInitApp'; import '@/web/styles/reset.scss'; +import NextHead from '@/components/common/NextHead'; function App({ Component, pageProps }: AppProps) { const { feConfigs, scripts, title } = useInitApp(); return ( <> - - {title} - - - - + {scripts?.map((item, i) => )} diff --git a/projects/app/src/pages/api/admin/clearInvalidData.ts b/projects/app/src/pages/api/admin/clearInvalidData.ts index c6e3a71b6..d3f85a6a7 100644 --- a/projects/app/src/pages/api/admin/clearInvalidData.ts +++ b/projects/app/src/pages/api/admin/clearInvalidData.ts @@ -58,17 +58,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) try { await connectToDatabase(); await authCert({ req, authRoot: true }); + const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number }; (async () => { try { console.log('执行脏数据清理任务'); // 360天 ~ 2小时前 - const end = addHours(new Date(), -2); - const start = addHours(new Date(), -360 * 24); - await checkInvalidDatasetFiles(start, end); - await checkInvalidImg(start, end); - await checkInvalidDatasetData(start, end); - await checkInvalidVector(start, end); + const endTime = addHours(new Date(), start); + const startTime = addHours(new Date(), end); + await checkInvalidDatasetFiles(startTime, endTime); + await checkInvalidImg(startTime, endTime); + await checkInvalidDatasetData(startTime, endTime); + await checkInvalidVector(startTime, endTime); console.log('执行脏数据清理任务完毕'); } catch (error) { console.log('执行脏数据清理任务出错了'); diff --git a/projects/app/src/pages/api/admin/initv481.ts b/projects/app/src/pages/api/admin/initv481.ts index 3ef361868..bfa3894c6 100644 --- a/projects/app/src/pages/api/admin/initv481.ts +++ b/projects/app/src/pages/api/admin/initv481.ts @@ -141,11 +141,18 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { const collections = await connectionMongo.connection.db .listCollections({ name: 'team.members' }) .toArray(); + if (collections.length > 0) { const sourceCol = connectionMongo.connection.db.collection('team.members'); + const targetCol = connectionMongo.connection.db.collection('team_members'); - await sourceCol.rename('team_members', { dropTarget: true }); - console.log('success rename team.members -> team_members'); + if ((await targetCol.countDocuments()) > 1) { + // 除了root + console.log('team_members 中有数据，无法自动将 buffer.tts 迁移到 team_members，请手动操作'); + } else { + await sourceCol.rename('team_members', { dropTarget: true }); + console.log('success rename team.members -> team_members'); + } } } catch (error) { console.log('error： rename team.members -> team_members', error); @@ -170,6 +177,27 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { console.log('error： rename team.tags -> team_tags', error); } + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'team.subscriptions' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('team.subscriptions'); + const targetCol = connectionMongo.connection.db.collection('team_subscriptions'); + + if ((await targetCol.countDocuments()) > 0) { + console.log( + 'team_subscriptions 中有数据，无法自动将 team.subscriptions 迁移到 team_subscriptions，请手动操作' + ); + } else { + await sourceCol.rename('team_subscriptions', { dropTarget: true }); + console.log('success rename team.subscriptions -> team_subscriptions'); + } + } + } catch (error) { + console.log('error： rename team.subscriptions -> team_subscriptions', error); + } + jsonRes(res, { message: 'success' }); diff --git a/projects/app/src/pages/api/common/file/uploadImage.ts b/projects/app/src/pages/api/common/file/uploadImage.ts index e88ac33cc..637b70ada 100644 --- a/projects/app/src/pages/api/common/file/uploadImage.ts +++ b/projects/app/src/pages/api/common/file/uploadImage.ts @@ -28,7 +28,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) export const config = { api: { - sizeLimit: '10mb', bodyParser: { sizeLimit: '16mb' } diff --git a/projects/app/src/pages/api/core/dataset/allDataset.ts b/projects/app/src/pages/api/core/dataset/allDataset.ts index 0783b178e..2440dda83 100644 --- a/projects/app/src/pages/api/core/dataset/allDataset.ts +++ b/projects/app/src/pages/api/core/dataset/allDataset.ts @@ -1,45 +1,31 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { getVectorModel } from '@fastgpt/service/core/ai/model'; -import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d'; +import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { NextAPI } from '@/service/middle/entry'; /* get all dataset by teamId or tmbId */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await connectToDatabase(); - // 凭证校验 - const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true }); +async function handler( + req: NextApiRequest, + res: NextApiResponse +): Promise { + // 凭证校验 + const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true }); - const datasets = await MongoDataset.find({ - ...mongoRPermission({ teamId, tmbId, role }), - type: { $ne: DatasetTypeEnum.folder } - }).lean(); + const datasets = await MongoDataset.find({ + ...mongoRPermission({ teamId, tmbId, role }), + type: { $ne: DatasetTypeEnum.folder } + }).lean(); - const data = datasets.map((item) => ({ - _id: item._id, - parentId: item.parentId, - avatar: item.avatar, - name: item.name, - intro: item.intro, - type: item.type, - permission: item.permission, - vectorModel: getVectorModel(item.vectorModel), - canWrite: String(item.tmbId) === tmbId, - isOwner: teamOwner || String(item.tmbId) === tmbId - })); - - jsonRes(res, { - data - }); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } + return datasets.map((item) => ({ + _id: item._id, + avatar: item.avatar, + name: item.name, + vectorModel: getVectorModel(item.vectorModel) + })); } + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/list.ts b/projects/app/src/pages/api/core/dataset/list.ts index 1bfddb1e3..f31f7899c 100644 --- a/projects/app/src/pages/api/core/dataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/list.ts @@ -9,7 +9,7 @@ import { getVectorModel } from '@fastgpt/service/core/ai/model'; import { NextAPI } from '@/service/middle/entry'; async function handler(req: NextApiRequest, res: NextApiResponse) { - const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` }; + const { parentId, type } = req.query as { parentId?: string; type?: DatasetTypeEnum }; // 凭证校验 const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({ req, diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index 9d4b5ae4f..cdb8a09dc 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -8,8 +8,18 @@ import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { id, parentId, name, avatar, intro, permission, agentModel, websiteConfig, status } = - req.body as DatasetUpdateBody; + const { + id, + parentId, + name, + avatar, + intro, + permission, + agentModel, + websiteConfig, + externalReadUrl, + status + } = req.body as DatasetUpdateBody; if (!id) { throw new Error('缺少参数'); @@ -33,7 +43,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...(agentModel && { agentModel: agentModel.model }), ...(websiteConfig && { websiteConfig }), ...(status && { status }), - ...(intro && { intro }) + ...(intro && { intro }), + ...(externalReadUrl && { externalReadUrl }) } ); diff --git a/projects/app/src/pages/api/core/plugin/list.ts b/projects/app/src/pages/api/core/plugin/list.ts index 94f676b4a..1d935bbf7 100644 --- a/projects/app/src/pages/api/core/plugin/list.ts +++ b/projects/app/src/pages/api/core/plugin/list.ts @@ -9,7 +9,7 @@ import { PluginListItemType } from '@fastgpt/global/core/plugin/controller'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` }; + const { parentId, type } = req.query as { parentId?: string; type?: DatasetTypeEnum }; const { teamId } = await authCert({ req, authToken: true }); diff --git a/projects/app/src/pages/api/core/workflow/debug.ts b/projects/app/src/pages/api/core/workflow/debug.ts index e4749d8b5..63c5096e7 100644 --- a/projects/app/src/pages/api/core/workflow/debug.ts +++ b/projects/app/src/pages/api/core/workflow/debug.ts @@ -82,7 +82,7 @@ export default NextAPI(handler); export const config = { api: { bodyParser: { - sizeLimit: '10mb' + sizeLimit: '20mb' }, responseLimit: '20mb' } diff --git a/projects/app/src/pages/api/plugins/textEditor/v2/index.ts b/projects/app/src/pages/api/plugins/textEditor/v2/index.ts index c01824ce3..01477a9ae 100644 --- a/projects/app/src/pages/api/plugins/textEditor/v2/index.ts +++ b/projects/app/src/pages/api/plugins/textEditor/v2/index.ts @@ -43,6 +43,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< export const config = { api: { + bodyParser: { + sizeLimit: '16mb' + }, responseLimit: '16mb' } }; diff --git a/projects/app/src/pages/api/v1/chat/completions.ts b/projects/app/src/pages/api/v1/chat/completions.ts index 72c57a970..179b9143b 100644 --- a/projects/app/src/pages/api/v1/chat/completions.ts +++ b/projects/app/src/pages/api/v1/chat/completions.ts @@ -523,6 +523,9 @@ const authHeaderRequest = async ({ export const config = { api: { + bodyParser: { + sizeLimit: '20mb' + }, responseLimit: '20mb' } }; diff --git a/projects/app/src/pages/chat/share.tsx b/projects/app/src/pages/chat/share.tsx index f82de4226..e73586159 100644 --- a/projects/app/src/pages/chat/share.tsx +++ b/projects/app/src/pages/chat/share.tsx @@ -269,139 +269,141 @@ const OutLink = ({ }, []); return ( - + <> {appName || chatData.app?.name} - - {showHistory === '1' - ? ((children: React.ReactNode) => { - return isPc ? ( - {children} - ) : ( - - - - {children} - - - ); - })( - ({ - id: item.chatId, - title: item.title, - customTitle: item.customTitle, - top: item.top - }))} - onClose={onCloseSlider} - onChangeChat={(chatId) => { - router.replace({ - query: { - ...router.query, - chatId: chatId || '' - } - }); - if (!isPc) { - onCloseSlider(); - } - }} - onDelHistory={({ chatId }) => - delOneHistory({ appId: chatData.appId, chatId, shareId, outLinkUid }) - } - onClearHistory={() => { - clearHistories({ shareId, outLinkUid }); - router.replace({ - query: { - ...router.query, - chatId: '' - } - }); - }} - onSetHistoryTop={(e) => { - updateHistory({ - ...e, - appId: chatData.appId, - shareId, - outLinkUid - }); - }} - onSetCustomTitle={async (e) => { - updateHistory({ - appId: chatData.appId, - chatId: e.chatId, - title: e.title, - customTitle: e.title, - shareId, - outLinkUid - }); - }} - /> - ) - : null} - - {/* chat container */} - - {/* header */} - - {/* chat box */} - - { + return isPc ? ( + {children} + ) : ( + + + + {children} + + + ); + })( + ({ + id: item.chatId, + title: item.title, + customTitle: item.customTitle, + top: item.top + }))} + onClose={onCloseSlider} + onChangeChat={(chatId) => { + router.replace({ + query: { + ...router.query, + chatId: chatId || '' + } + }); + if (!isPc) { + onCloseSlider(); + } + }} + onDelHistory={({ chatId }) => + delOneHistory({ appId: chatData.appId, chatId, shareId, outLinkUid }) + } + onClearHistory={() => { + clearHistories({ shareId, outLinkUid }); + router.replace({ + query: { + ...router.query, + chatId: '' + } + }); + }} + onSetHistoryTop={(e) => { + updateHistory({ + ...e, + appId: chatData.appId, + shareId, + outLinkUid + }); + }} + onSetCustomTitle={async (e) => { + updateHistory({ + appId: chatData.appId, + chatId: e.chatId, + title: e.title, + customTitle: e.title, + shareId, + outLinkUid + }); + }} + /> + ) + : null} + + {/* chat container */} + + {/* header */} + {}} - onStartChat={startChat} - onDelMessage={(e) => - delOneHistoryItem({ ...e, appId: chatData.appId, chatId, shareId, outLinkUid }) - } - appId={chatData.appId} - chatId={chatId} - shareId={shareId} - outLinkUid={outLinkUid} + appName={chatData.app.name} + history={chatData.history} + showHistory={showHistory === '1'} + onOpenSlider={onOpenSlider} /> - - - - + {/* chat box */} + + {}} + onStartChat={startChat} + onDelMessage={(e) => + delOneHistoryItem({ ...e, appId: chatData.appId, chatId, shareId, outLinkUid }) + } + appId={chatData.appId} + chatId={chatId} + shareId={shareId} + outLinkUid={outLinkUid} + /> + + + + + ); }; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx new file mode 100644 index 000000000..6da143e75 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx @@ -0,0 +1,158 @@ +import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; +import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react'; +import { useTranslation } from 'next-i18next'; +import { createContext, useContextSelector } from 'use-context-selector'; +import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; +import { useDisclosure } from '@chakra-ui/react'; +import { checkTeamWebSyncLimit } from '@/web/support/user/team/api'; +import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api'; +import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api'; +import dynamic from 'next/dynamic'; +import { usePagination } from '@fastgpt/web/hooks/usePagination'; +import { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; +import { useRouter } from 'next/router'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig')); + +type CollectionPageContextType = { + openWebSyncConfirm: () => void; + onOpenWebsiteModal: () => void; + collections: DatasetCollectionsListItemType[]; + Pagination: () => JSX.Element; + total: number; + getData: (e: number) => void; + isGetting: boolean; + pageNum: number; + pageSize: number; + searchText: string; + setSearchText: Dispatch>; +}; + +export const CollectionPageContext = createContext({ + openWebSyncConfirm: function (): () => void { + throw new Error('Function not implemented.'); + }, + onOpenWebsiteModal: function (): void { + throw new Error('Function not implemented.'); + }, + collections: [], + Pagination: function (): JSX.Element { + throw new Error('Function not implemented.'); + }, + total: 0, + getData: function (e: number): void { + throw new Error('Function not implemented.'); + }, + isGetting: false, + pageNum: 0, + pageSize: 0, + searchText: '', + setSearchText: function (value: SetStateAction): void { + throw new Error('Function not implemented.'); + } +}); + +const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => { + const { t } = useTranslation(); + const router = useRouter(); + const { parentId = '' } = router.query as { parentId: string }; + + const { datasetDetail, datasetId, updateDataset } = useContextSelector( + DatasetPageContext, + (v) => v + ); + + // website config + const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({ + content: t('core.dataset.collection.Start Sync Tip') + }); + const { + isOpen: isOpenWebsiteModal, + onOpen: onOpenWebsiteModal, + onClose: onCloseWebsiteModal + } = useDisclosure(); + const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({ + mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => { + onCloseWebsiteModal(); + await checkTeamWebSyncLimit(); + const billId = await postCreateTrainingUsage({ + name: t('core.dataset.training.Website Sync'), + datasetId: datasetId + }); + await postWebsiteSync({ datasetId: datasetId, billId }); + + await updateDataset({ + id: datasetId, + websiteConfig, + status: DatasetStatusEnum.syncing + }); + + return; + }, + errorToast: t('common.Update Failed') + }); + + // collection list + const [searchText, setSearchText] = useState(''); + const { + data: collections, + Pagination, + total, + getData, + isLoading: isGetting, + pageNum, + pageSize + } = usePagination({ + api: getDatasetCollections, + pageSize: 20, + params: { + datasetId, + parentId, + searchText + }, + defaultRequest: false + }); + useEffect(() => { + getData(1); + }, [parentId]); + + const contextValue: CollectionPageContextType = { + openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig), + onOpenWebsiteModal, + + searchText, + setSearchText, + collections, + Pagination, + total, + getData, + isGetting, + pageNum, + pageSize + }; + + return ( + + {children} + {datasetDetail.type === DatasetTypeEnum.websiteDataset && ( + <> + {isOpenWebsiteModal && ( + + )} + + + )} + + ); +}; +export default CollectionPageContextProvider; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx new file mode 100644 index 000000000..047c6bfa4 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx @@ -0,0 +1,55 @@ +import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; +import React from 'react'; +import { useTranslation } from 'next-i18next'; +import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { Box, Flex } from '@chakra-ui/react'; +import { useContextSelector } from 'use-context-selector'; +import { CollectionPageContext } from './Context'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +const EmptyCollectionTip = () => { + const { t } = useTranslation(); + const onOpenWebsiteModal = useContextSelector(CollectionPageContext, (v) => v.onOpenWebsiteModal); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + + return ( + <> + {(datasetDetail.type === DatasetTypeEnum.dataset || + datasetDetail.type === DatasetTypeEnum.externalFile) && ( + + )} + {datasetDetail.type === DatasetTypeEnum.websiteDataset && ( + + {datasetDetail.status === DatasetStatusEnum.syncing && ( + <>{t('core.dataset.status.syncing')} + )} + {datasetDetail.status === DatasetStatusEnum.active && ( + <> + {!datasetDetail?.websiteConfig?.url ? ( + <> + {t('core.dataset.collection.Website Empty Tip')} + {', '} + + {t('core.dataset.collection.Click top config website')} + + + ) : ( + <>{t('core.dataset.website.UnValid Website Tip')} + )} + + )} + + } + /> + )} + + ); +}; + +export default EmptyCollectionTip; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx new file mode 100644 index 000000000..425c85409 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx @@ -0,0 +1,399 @@ +import React, { useCallback, useRef } from 'react'; +import { Box, Flex, MenuButton, Button, Link, useTheme, useDisclosure } from '@chakra-ui/react'; +import { + getDatasetCollectionPathById, + postDatasetCollection, + putDatasetCollectionById +} from '@/web/core/dataset/api'; +import { useQuery } from '@tanstack/react-query'; +import { debounce } from 'lodash'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyInput from '@/components/MyInput'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { useRouter } from 'next/router'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; +import MyMenu from '@fastgpt/web/components/common/MyMenu'; +import { useEditTitle } from '@/web/common/hooks/useEditTitle'; +import { + DatasetCollectionTypeEnum, + TrainingModeEnum, + DatasetTypeEnum, + DatasetTypeMap, + DatasetStatusEnum +} from '@fastgpt/global/core/dataset/constants'; +import EditFolderModal, { useEditFolder } from '../../../component/EditFolderModal'; +import { TabEnum } from '../../index'; +import ParentPath from '@/components/common/ParentPaths'; +import dynamic from 'next/dynamic'; +import { useUserStore } from '@/web/support/user/useUserStore'; +import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; + +import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; +import { useContextSelector } from 'use-context-selector'; +import { CollectionPageContext } from './Context'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector')); + +const Header = ({}: {}) => { + const { t } = useTranslation(); + const theme = useTheme(); + const { setLoading } = useSystemStore(); + const { userInfo } = useUserStore(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + + const router = useRouter(); + const { parentId = '' } = router.query as { parentId: string; datasetId: string }; + const { isPc } = useSystemStore(); + + const lastSearch = useRef(''); + const { searchText, setSearchText, total, getData, pageNum, onOpenWebsiteModal } = + useContextSelector(CollectionPageContext, (v) => v); + + // change search + const debounceRefetch = useCallback( + debounce(() => { + getData(1); + lastSearch.current = searchText; + }, 300), + [] + ); + + const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () => + getDatasetCollectionPathById(parentId) + ); + + const { editFolderData, setEditFolderData } = useEditFolder(); + const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } = + useEditTitle({ + title: t('dataset.Create manual collection'), + tip: t('dataset.Manual collection Tip'), + canEmpty: false + }); + const { + isOpen: isOpenFileSourceSelector, + onOpen: onOpenFileSourceSelector, + onClose: onCloseFileSourceSelector + } = useDisclosure(); + const { mutate: onCreateCollection } = useRequest({ + mutationFn: async ({ + name, + type, + callback, + ...props + }: { + name: string; + type: `${DatasetCollectionTypeEnum}`; + callback?: (id: string) => void; + trainingType?: TrainingModeEnum; + rawLink?: string; + chunkSize?: number; + }) => { + setLoading(true); + const id = await postDatasetCollection({ + parentId, + datasetId: datasetDetail._id, + name, + type, + ...props + }); + callback?.(id); + return id; + }, + onSuccess() { + getData(pageNum); + }, + onSettled() { + setLoading(false); + }, + + successToast: t('common.Create Success'), + errorToast: t('common.Create Failed') + }); + + return ( + + + ({ + parentId: path.parentId, + parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName + }))} + FirstPathDom={ + <> + + {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) + + {datasetDetail?.websiteConfig?.url && ( + + {t('core.dataset.website.Base Url')}: + + {datasetDetail.websiteConfig.url} + + + )} + + } + onClick={(e) => { + router.replace({ + query: { + ...router.query, + parentId: e + } + }); + }} + /> + + + {/* search input */} + {isPc && ( + + + } + onChange={(e) => { + setSearchText(e.target.value); + debounceRefetch(); + }} + onBlur={() => { + if (searchText === lastSearch.current) return; + getData(1); + }} + onKeyDown={(e) => { + if (searchText === lastSearch.current) return; + if (e.key === 'Enter') { + getData(1); + } + }} + /> + + )} + + {/* diff collection button */} + {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( + <> + {datasetDetail?.type === DatasetTypeEnum.dataset && ( + + + + {t('dataset.collections.Create And Import')} + + + } + menuList={[ + { + label: ( + + + {t('Folder')} + + ), + onClick: () => setEditFolderData({}) + }, + { + label: ( + + + {t('core.dataset.Manual collection')} + + ), + onClick: () => { + onOpenCreateVirtualFileModal({ + defaultVal: '', + onSuccess: (name) => { + onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); + } + }); + } + }, + { + label: ( + + + {t('core.dataset.Text collection')} + + ), + onClick: onOpenFileSourceSelector + }, + { + label: ( + + + {t('core.dataset.Table collection')} + + ), + onClick: () => + router.replace({ + query: { + ...router.query, + currentTab: TabEnum.import, + source: ImportDataSourceEnum.csvTable + } + }) + } + ]} + /> + )} + {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( + <> + {datasetDetail?.websiteConfig?.url ? ( + + {datasetDetail.status === DatasetStatusEnum.active && ( + + )} + {datasetDetail.status === DatasetStatusEnum.syncing && ( + + + + {t('core.dataset.status.syncing')} + + + )} + + ) : ( + + )} + + )} + {datasetDetail?.type === DatasetTypeEnum.externalFile && ( + + + + {t('dataset.collections.Create And Import')} + + + } + menuList={[ + { + label: ( + + + {t('Folder')} + + ), + onClick: () => setEditFolderData({}) + }, + { + label: ( + + + {t('core.dataset.Text collection')} + + ), + onClick: () => + router.replace({ + query: { + ...router.query, + currentTab: TabEnum.import, + source: ImportDataSourceEnum.externalFile + } + }) + } + ]} + /> + )} + + )} + + {/* modal */} + {!!editFolderData && ( + setEditFolderData(undefined)} + editCallback={async (name) => { + try { + if (editFolderData.id) { + await putDatasetCollectionById({ + id: editFolderData.id, + name + }); + getData(pageNum); + } else { + onCreateCollection({ + name, + type: DatasetCollectionTypeEnum.folder + }); + } + } catch (error) { + return Promise.reject(error); + } + }} + isEdit={!!editFolderData.id} + name={editFolderData.name} + /> + )} + + {isOpenFileSourceSelector && } + + ); +}; + +export default Header; diff --git a/projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/WebsiteConfig.tsx similarity index 100% rename from projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx rename to projects/app/src/pages/dataset/detail/components/CollectionCard/WebsiteConfig.tsx diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx similarity index 52% rename from projects/app/src/pages/dataset/detail/components/CollectionCard.tsx rename to projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx index 87896ef17..1bb485953 100644 --- a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useState, useRef, useMemo, useEffect } from 'react'; +import React, { useState, useRef, useMemo } from 'react'; import { Box, Flex, @@ -9,47 +9,29 @@ import { Th, Td, Tbody, - Image, - MenuButton, - useDisclosure, - Button, - Link, - useTheme + MenuButton } from '@chakra-ui/react'; import { - getDatasetCollections, delDatasetCollectionById, putDatasetCollectionById, - postDatasetCollection, - getDatasetCollectionPathById, postLinkCollectionSync } from '@/web/core/dataset/api'; import { useQuery } from '@tanstack/react-query'; -import { debounce } from 'lodash'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; -import MyInput from '@/components/MyInput'; import dayjs from 'dayjs'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useRouter } from 'next/router'; -import { useSystemStore } from '@/web/common/system/useSystemStore'; import MyMenu from '@fastgpt/web/components/common/MyMenu'; import { useEditTitle } from '@/web/common/hooks/useEditTitle'; -import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d'; -import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import { DatasetCollectionTypeEnum, - TrainingModeEnum, - DatasetTypeEnum, - DatasetTypeMap, DatasetStatusEnum, DatasetCollectionSyncResultMap } from '@fastgpt/global/core/dataset/constants'; import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; -import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal'; -import { TabEnum } from '..'; -import ParentPath from '@/components/common/ParentPaths'; +import { TabEnum } from '../../index'; import dynamic from 'next/dynamic'; import { useDrag } from '@/web/common/hooks/useDrag'; import SelectCollections from '@/web/core/dataset/components/SelectCollections'; @@ -58,27 +40,22 @@ import MyTooltip from '@/components/MyTooltip'; import { useUserStore } from '@/web/support/user/useUserStore'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; -import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants'; import MyBox from '@fastgpt/web/components/common/MyBox'; -import { usePagination } from '@fastgpt/web/hooks/usePagination'; -import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; +import { useContextSelector } from 'use-context-selector'; +import { CollectionPageContext } from './Context'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {}); -const FileSourceSelector = dynamic(() => import('./Import/components/FileSourceSelector'), {}); +const Header = dynamic(() => import('./Header')); +const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip')); const CollectionCard = () => { const BoxRef = useRef(null); - const lastSearch = useRef(''); const router = useRouter(); - const theme = useTheme(); const { toast } = useToast(); - const { parentId = '', datasetId } = router.query as { parentId: string; datasetId: string }; const { t } = useTranslation(); - const { isPc } = useSystemStore(); const { userInfo } = useUserStore(); - const [searchText, setSearchText] = useState(''); - const { datasetDetail, updateDataset, startWebsiteSync, loadDatasetDetail } = useDatasetStore(); + const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v); const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({ content: t('dataset.Confirm to delete the file'), @@ -88,66 +65,18 @@ const CollectionCard = () => { content: t('core.dataset.collection.Start Sync Tip') }); - const { - isOpen: isOpenFileSourceSelector, - onOpen: onOpenFileSourceSelector, - onClose: onCloseFileSourceSelector - } = useDisclosure(); - const { - isOpen: isOpenWebsiteModal, - onOpen: onOpenWebsiteModal, - onClose: onCloseWebsiteModal - } = useDisclosure(); - const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } = - useEditTitle({ - title: t('dataset.Create manual collection'), - tip: t('dataset.Manual collection Tip'), - canEmpty: false - }); - const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({ title: t('Rename') }); - const { editFolderData, setEditFolderData } = useEditFolder(); const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>(); - const { - data: collections, - Pagination, - total, - getData, - isLoading: isGetting, - pageNum, - pageSize - } = usePagination({ - api: getDatasetCollections, - pageSize: 20, - params: { - datasetId, - parentId, - searchText - }, - defaultRequest: false, - onChange() { - if (BoxRef.current) { - BoxRef.current.scrollTop = 0; - } - } - }); + const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } = + useContextSelector(CollectionPageContext, (v) => v); const { dragStartId, setDragStartId, dragTargetId, setDragTargetId } = useDrag(); - // change search - const debounceRefetch = useCallback( - debounce(() => { - getData(1); - lastSearch.current = searchText; - }, 300), - [] - ); - - // add file icon + // Ad file status icon const formatCollections = useMemo( () => collections.map((collection) => { @@ -180,37 +109,6 @@ const CollectionCard = () => { [collections, t] ); - const { mutate: onCreateCollection, isLoading: isCreating } = useRequest({ - mutationFn: async ({ - name, - type, - callback, - ...props - }: { - name: string; - type: `${DatasetCollectionTypeEnum}`; - callback?: (id: string) => void; - trainingType?: `${TrainingModeEnum}`; - rawLink?: string; - chunkSize?: number; - }) => { - const id = await postDatasetCollection({ - parentId, - datasetId, - name, - type, - ...props - }); - callback?.(id); - return id; - }, - onSuccess() { - getData(pageNum); - }, - - successToast: t('common.Create Success'), - errorToast: t('common.Create Failed') - }); const { mutate: onUpdateCollectionName } = useRequest({ mutationFn: ({ collectionId, name }: { collectionId: string; name: string }) => { return putDatasetCollectionById({ @@ -237,17 +135,7 @@ const CollectionCard = () => { successToast: t('common.Delete Success'), errorToast: t('common.Delete Failed') }); - const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({ - mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => { - onCloseWebsiteModal(); - await updateDataset({ - id: datasetDetail._id, - websiteConfig - }); - return startWebsiteSync(); - }, - errorToast: t('common.Update Failed') - }); + const { mutate: onclickStartSync, isLoading: isSyncing } = useRequest({ mutationFn: (collectionId: string) => { return postLinkCollectionSync(collectionId); @@ -262,22 +150,13 @@ const CollectionCard = () => { errorToast: t('core.dataset.error.Start Sync Failed') }); - const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () => - getDatasetCollectionPathById(parentId) - ); - const hasTrainingData = useMemo( () => !!formatCollections.find((item) => item.trainingAmount > 0), [formatCollections] ); const isLoading = useMemo( - () => - isCreating || - isDeleting || - isUpdating || - isSyncing || - (isGetting && collections.length === 0), - [collections.length, isCreating, isDeleting, isGetting, isSyncing, isUpdating] + () => isDeleting || isSyncing || (isGetting && collections.length === 0), + [collections.length, isDeleting, isGetting, isSyncing] ); useQuery( @@ -285,7 +164,7 @@ const CollectionCard = () => { () => { getData(1); if (datasetDetail.status === DatasetStatusEnum.syncing) { - loadDatasetDetail(datasetId, true); + loadDatasetDetail(datasetDetail._id); } return null; }, @@ -295,207 +174,11 @@ const CollectionCard = () => { } ); - useEffect(() => { - getData(1); - }, [parentId]); - return ( {/* header */} - - - ({ - parentId: path.parentId, - parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName - }))} - FirstPathDom={ - <> - - {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) - - {datasetDetail?.websiteConfig?.url && ( - - {t('core.dataset.website.Base Url')}: - - {datasetDetail.websiteConfig.url} - - - )} - - } - onClick={(e) => { - router.replace({ - query: { - ...router.query, - parentId: e - } - }); - }} - /> - - - {isPc && ( - - - } - onChange={(e) => { - setSearchText(e.target.value); - debounceRefetch(); - }} - onBlur={() => { - if (searchText === lastSearch.current) return; - getData(1); - }} - onKeyDown={(e) => { - if (searchText === lastSearch.current) return; - if (e.key === 'Enter') { - getData(1); - } - }} - /> - - )} - {datasetDetail?.type === DatasetTypeEnum.dataset && ( - <> - {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( - - - - {t('dataset.collections.Create And Import')} - - - } - menuList={[ - { - label: ( - - - {t('Folder')} - - ), - onClick: () => setEditFolderData({}) - }, - { - label: ( - - - {t('core.dataset.Manual collection')} - - ), - onClick: () => { - onOpenCreateVirtualFileModal({ - defaultVal: '', - onSuccess: (name) => { - onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); - } - }); - } - }, - { - label: ( - - - {t('core.dataset.Text collection')} - - ), - onClick: onOpenFileSourceSelector - }, - { - label: ( - - - {t('core.dataset.Table collection')} - - ), - onClick: () => - router.replace({ - query: { - ...router.query, - currentTab: TabEnum.import, - source: ImportDataSourceEnum.csvTable - } - }) - } - ]} - /> - )} - - )} - {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( - <> - {datasetDetail?.websiteConfig?.url ? ( - - {datasetDetail.status === DatasetStatusEnum.active && ( - - )} - {datasetDetail.status === DatasetStatusEnum.syncing && ( - - - - {t('core.dataset.status.syncing')} - - - )} - - ) : ( - - )} - - )} - +
{/* collection table */} { )} - {total === 0 && ( - - {datasetDetail.status === DatasetStatusEnum.syncing && ( - <>{t('core.dataset.status.syncing')} - )} - {datasetDetail.status === DatasetStatusEnum.active && ( - <> - {!datasetDetail?.websiteConfig?.url ? ( - <> - {t('core.dataset.collection.Website Empty Tip')} - {', '} - - {t('core.dataset.collection.Click top config website')} - - - ) : ( - <>{t('core.dataset.website.UnValid Website Tip')} - )} - - )} - - ) - } - /> - )} + {total === 0 && } - - {/* {isOpenFileImportModal && ( - { - getData(1); - onCloseFileImportModal(); - }} - onClose={onCloseFileImportModal} - /> - )} */} - {isOpenFileSourceSelector && } - {!!editFolderData && ( - setEditFolderData(undefined)} - editCallback={async (name) => { - try { - if (editFolderData.id) { - await putDatasetCollectionById({ - id: editFolderData.id, - name - }); - getData(pageNum); - } else { - onCreateCollection({ - name, - type: DatasetCollectionTypeEnum.folder - }); - } - } catch (error) { - return Promise.reject(error); - } - }} - isEdit={!!editFolderData.id} - name={editFolderData.name} - /> - )} + {!!moveCollectionData && ( setMoveCollectionData(undefined)} @@ -828,16 +441,6 @@ const CollectionCard = () => { }} /> )} - {isOpenWebsiteModal && ( - - )} ); diff --git a/projects/app/src/pages/dataset/detail/components/Import/Context.tsx b/projects/app/src/pages/dataset/detail/components/Import/Context.tsx new file mode 100644 index 000000000..6074df1f2 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/Import/Context.tsx @@ -0,0 +1,302 @@ +import { useRouter } from 'next/router'; +import { SetStateAction, useState } from 'react'; +import { useTranslation } from 'next-i18next'; +import { createContext, useContextSelector } from 'use-context-selector'; +import { ImportDataSourceEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { useMyStep } from '@fastgpt/web/hooks/useStep'; +import { Box, Button, Flex, IconButton } from '@chakra-ui/react'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { TabEnum } from '../Slider'; +import { ImportProcessWayEnum } from '@/web/core/dataset/constants'; +import { UseFormReturn, useForm } from 'react-hook-form'; +import { ImportSourceItemType } from '@/web/core/dataset/type'; +import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +type TrainingFiledType = { + chunkOverlapRatio: number; + maxChunkSize: number; + minChunkSize: number; + autoChunkSize: number; + chunkSize: number; + showChunkInput: boolean; + showPromptInput: boolean; + charsPointsPrice: number; + priceTip: string; + uploadRate: number; + chunkSizeField?: ChunkSizeFieldType; +}; +type DatasetImportContextType = { + importSource: ImportDataSourceEnum; + parentId: string | undefined; + activeStep: number; + goToNext: () => void; + + processParamsForm: UseFormReturn; + sources: ImportSourceItemType[]; + setSources: React.Dispatch>; +} & TrainingFiledType; + +type ChunkSizeFieldType = 'embeddingChunkSize'; +export type ImportFormType = { + mode: TrainingModeEnum; + way: ImportProcessWayEnum; + embeddingChunkSize: number; + customSplitChar: string; + qaPrompt: string; + webSelector: string; +}; + +export const DatasetImportContext = createContext({ + importSource: ImportDataSourceEnum.fileLocal, + goToNext: function (): void { + throw new Error('Function not implemented.'); + }, + activeStep: 0, + parentId: undefined, + + maxChunkSize: 0, + minChunkSize: 0, + showChunkInput: false, + showPromptInput: false, + sources: [], + setSources: function (value: SetStateAction): void { + throw new Error('Function not implemented.'); + }, + chunkSize: 0, + chunkOverlapRatio: 0, + uploadRate: 0, + //@ts-ignore + processParamsForm: undefined, + autoChunkSize: 0, + charsPointsPrice: 0, + priceTip: '' +}); + +const DatasetImportContextProvider = ({ children }: { children: React.ReactNode }) => { + const { t } = useTranslation(); + const router = useRouter(); + const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as { + source: ImportDataSourceEnum; + parentId?: string; + }; + + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + + // step + const modeSteps: Record = { + [ImportDataSourceEnum.fileLocal]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.fileLink]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.fileCustom]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.csvTable]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.externalFile]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ] + }; + const steps = modeSteps[source]; + const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({ + defaultStep: 0, + steps + }); + + // ----- + const vectorModel = datasetDetail.vectorModel; + const agentModel = datasetDetail.agentModel; + + const processParamsForm = useForm({ + defaultValues: { + mode: TrainingModeEnum.chunk, + way: ImportProcessWayEnum.auto, + embeddingChunkSize: vectorModel?.defaultToken || 512, + customSplitChar: '', + qaPrompt: Prompt_AgentQA.description, + webSelector: '' + } + }); + + const [sources, setSources] = useState([]); + + // watch form + const mode = processParamsForm.watch('mode'); + const way = processParamsForm.watch('way'); + const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize'); + const customSplitChar = processParamsForm.watch('customSplitChar'); + + const modeStaticParams: Record = { + [TrainingModeEnum.auto]: { + chunkOverlapRatio: 0.2, + maxChunkSize: 2048, + minChunkSize: 100, + autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, + chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, + showChunkInput: false, + showPromptInput: false, + charsPointsPrice: agentModel.charsPointsPrice, + priceTip: t('core.dataset.import.Auto mode Estimated Price Tips', { + price: agentModel.charsPointsPrice + }), + uploadRate: 100 + }, + [TrainingModeEnum.chunk]: { + chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType, + chunkOverlapRatio: 0.2, + maxChunkSize: vectorModel?.maxToken || 512, + minChunkSize: 100, + autoChunkSize: vectorModel?.defaultToken || 512, + chunkSize: embeddingChunkSize, + showChunkInput: true, + showPromptInput: false, + charsPointsPrice: vectorModel.charsPointsPrice, + priceTip: t('core.dataset.import.Embedding Estimated Price Tips', { + price: vectorModel.charsPointsPrice + }), + uploadRate: 150 + }, + [TrainingModeEnum.qa]: { + chunkOverlapRatio: 0, + maxChunkSize: 8000, + minChunkSize: 3000, + autoChunkSize: agentModel.maxContext * 0.55 || 6000, + chunkSize: agentModel.maxContext * 0.55 || 6000, + showChunkInput: false, + showPromptInput: true, + charsPointsPrice: agentModel.charsPointsPrice, + priceTip: t('core.dataset.import.QA Estimated Price Tips', { + price: agentModel?.charsPointsPrice + }), + uploadRate: 30 + } + }; + const selectModelStaticParam = modeStaticParams[mode]; + + const wayStaticPrams = { + [ImportProcessWayEnum.auto]: { + chunkSize: selectModelStaticParam.autoChunkSize, + customSplitChar: '' + }, + [ImportProcessWayEnum.custom]: { + chunkSize: modeStaticParams[mode].chunkSize, + customSplitChar + } + }; + + const chunkSize = wayStaticPrams[way].chunkSize; + + const contextValue = { + importSource: source, + parentId, + activeStep, + goToNext, + + processParamsForm, + ...selectModelStaticParam, + sources, + setSources, + chunkSize + }; + + return ( + + + {activeStep === 0 ? ( + + } + aria-label={''} + size={'smSquare'} + w={'26px'} + h={'26px'} + borderRadius={'50%'} + variant={'whiteBase'} + mr={2} + onClick={() => + router.replace({ + query: { + ...router.query, + currentTab: TabEnum.collectionCard + } + }) + } + /> + {t('common.Exit')} + + ) : ( + + )} + + + {/* step */} + + + + + + {children} + + ); +}; + +export default DatasetImportContextProvider; diff --git a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx deleted file mode 100644 index 228e6684a..000000000 --- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx +++ /dev/null @@ -1,165 +0,0 @@ -import React, { useContext, createContext, useState, useMemo, useEffect } from 'react'; - -import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { useTranslation } from 'next-i18next'; -import { DatasetItemType } from '@fastgpt/global/core/dataset/type'; -import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; -import { UseFormReturn, useForm } from 'react-hook-form'; -import { ImportProcessWayEnum } from '@/web/core/dataset/constants'; -import { ImportSourceItemType } from '@/web/core/dataset/type'; -import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; - -type ChunkSizeFieldType = 'embeddingChunkSize'; -export type FormType = { - mode: `${TrainingModeEnum}`; - way: `${ImportProcessWayEnum}`; - embeddingChunkSize: number; - customSplitChar: string; - qaPrompt: string; - webSelector: string; -}; - -type useImportStoreType = { - parentId?: string; - processParamsForm: UseFormReturn; - chunkSizeField?: ChunkSizeFieldType; - maxChunkSize: number; - minChunkSize: number; - showChunkInput: boolean; - showPromptInput: boolean; - sources: ImportSourceItemType[]; - setSources: React.Dispatch>; - chunkSize: number; - chunkOverlapRatio: number; - priceTip: string; - uploadRate: number; - importSource: `${ImportDataSourceEnum}`; -}; -const StateContext = createContext({ - processParamsForm: {} as any, - sources: [], - setSources: function (value: React.SetStateAction): void { - throw new Error('Function not implemented.'); - }, - maxChunkSize: 0, - minChunkSize: 0, - showChunkInput: false, - showPromptInput: false, - chunkSizeField: 'embeddingChunkSize', - chunkSize: 0, - chunkOverlapRatio: 0, - priceTip: '', - uploadRate: 50, - importSource: ImportDataSourceEnum.fileLocal -}); - -export const useImportStore = () => useContext(StateContext); - -const Provider = ({ - importSource, - dataset, - parentId, - children -}: { - importSource: `${ImportDataSourceEnum}`; - dataset: DatasetItemType; - parentId?: string; - children: React.ReactNode; -}) => { - const vectorModel = dataset.vectorModel; - const agentModel = dataset.agentModel; - - const processParamsForm = useForm({ - defaultValues: { - mode: TrainingModeEnum.chunk, - way: ImportProcessWayEnum.auto, - embeddingChunkSize: vectorModel?.defaultToken || 512, - customSplitChar: '', - qaPrompt: Prompt_AgentQA.description, - webSelector: '' - } - }); - - const { t } = useTranslation(); - const [sources, setSources] = useState([]); - - // watch form - const mode = processParamsForm.watch('mode'); - const way = processParamsForm.watch('way'); - const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize'); - const customSplitChar = processParamsForm.watch('customSplitChar'); - - const modeStaticParams = { - [TrainingModeEnum.auto]: { - chunkOverlapRatio: 0.2, - maxChunkSize: 2048, - minChunkSize: 100, - autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, - chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, - showChunkInput: false, - showPromptInput: false, - charsPointsPrice: agentModel.charsPointsPrice, - priceTip: t('core.dataset.import.Auto mode Estimated Price Tips', { - price: agentModel.charsPointsPrice - }), - uploadRate: 100 - }, - [TrainingModeEnum.chunk]: { - chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType, - chunkOverlapRatio: 0.2, - maxChunkSize: vectorModel?.maxToken || 512, - minChunkSize: 100, - autoChunkSize: vectorModel?.defaultToken || 512, - chunkSize: embeddingChunkSize, - showChunkInput: true, - showPromptInput: false, - charsPointsPrice: vectorModel.charsPointsPrice, - priceTip: t('core.dataset.import.Embedding Estimated Price Tips', { - price: vectorModel.charsPointsPrice - }), - uploadRate: 150 - }, - [TrainingModeEnum.qa]: { - chunkOverlapRatio: 0, - maxChunkSize: 8000, - minChunkSize: 3000, - autoChunkSize: agentModel.maxContext * 0.55 || 6000, - chunkSize: agentModel.maxContext * 0.55 || 6000, - showChunkInput: false, - showPromptInput: true, - charsPointsPrice: agentModel.charsPointsPrice, - priceTip: t('core.dataset.import.QA Estimated Price Tips', { - price: agentModel?.charsPointsPrice - }), - uploadRate: 30 - } - }; - const selectModelStaticParam = useMemo(() => modeStaticParams[mode], [mode]); - - const wayStaticPrams = { - [ImportProcessWayEnum.auto]: { - chunkSize: selectModelStaticParam.autoChunkSize, - customSplitChar: '' - }, - [ImportProcessWayEnum.custom]: { - chunkSize: modeStaticParams[mode].chunkSize, - customSplitChar - } - }; - - const chunkSize = wayStaticPrams[way].chunkSize; - - const value: useImportStoreType = { - parentId, - processParamsForm, - ...selectModelStaticParam, - sources, - setSources, - chunkSize, - - importSource - }; - return {children}; -}; - -export default React.memo(Provider); diff --git a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx index 99644e8bd..e0987b657 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx @@ -20,23 +20,20 @@ import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio'; import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants'; import { ImportProcessWayEnum } from '@/web/core/dataset/constants'; import MyTooltip from '@/components/MyTooltip'; -import { useImportStore } from '../Provider'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; import Preview from '../components/Preview'; import Tag from '@fastgpt/web/components/common/Tag/index'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; -function DataProcess({ - showPreviewChunks = true, - goToNext -}: { - showPreviewChunks: boolean; - goToNext: () => void; -}) { +function DataProcess({ showPreviewChunks = true }: { showPreviewChunks: boolean }) { const { t } = useTranslation(); const { feConfigs } = useSystemStore(); + const { + goToNext, processParamsForm, chunkSizeField, minChunkSize, @@ -44,7 +41,7 @@ function DataProcess({ showPromptInput, maxChunkSize, priceTip - } = useImportStore(); + } = useContextSelector(DatasetImportContext, (v) => v); const { getValues, setValue, register } = processParamsForm; const [refresh, setRefresh] = useState(false); diff --git a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx index 0c2070bd1..f018beb7e 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx @@ -2,15 +2,12 @@ import React from 'react'; import Preview from '../components/Preview'; import { Box, Button, Flex } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; -const PreviewData = ({ - showPreviewChunks, - goToNext -}: { - showPreviewChunks: boolean; - goToNext: () => void; -}) => { +const PreviewData = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => { const { t } = useTranslation(); + const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext); return ( diff --git a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx index c7b99fa5f..f5b441ff9 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx @@ -11,7 +11,6 @@ import { Flex, Button } from '@chakra-ui/react'; -import { useImportStore, type FormType } from '../Provider'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -28,20 +27,23 @@ import { } from '@/web/core/dataset/api'; import Tag from '@fastgpt/web/components/common/Tag/index'; import { useI18n } from '@/web/context/I18n'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { DatasetImportContext, type ImportFormType } from '../Context'; const Upload = () => { const { t } = useTranslation(); const { fileT } = useI18n(); const { toast } = useToast(); const router = useRouter(); - const { datasetDetail } = useDatasetStore(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); const { importSource, parentId, sources, setSources, processParamsForm, chunkSize } = - useImportStore(); + useContextSelector(DatasetImportContext, (v) => v); const { handleSubmit } = processParamsForm; const { mutate: startUpload, isLoading } = useRequest({ - mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: FormType) => { + mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: ImportFormType) => { if (sources.length === 0) return; const filterWaitingSources = sources.filter((item) => item.createStatus === 'waiting'); diff --git a/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx b/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx index 38af361d8..e6726ddb3 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx @@ -10,7 +10,7 @@ import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; const FileModeSelector = ({ onClose }: { onClose: () => void }) => { const { t } = useTranslation(); const router = useRouter(); - const [value, setValue] = useState<`${ImportDataSourceEnum}`>(ImportDataSourceEnum.fileLocal); + const [value, setValue] = useState(ImportDataSourceEnum.fileLocal); return ( import('./PreviewRawText')); const PreviewChunks = dynamic(() => import('./PreviewChunks')); const Preview = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => { const { t } = useTranslation(); - const { sources } = useImportStore(); + const { sources } = useContextSelector(DatasetImportContext, (v) => v); const [previewRawTextSource, setPreviewRawTextSource] = useState(); const [previewChunkSource, setPreviewChunkSource] = useState(); diff --git a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx index e0c1a6b7e..65b30ce74 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx @@ -4,11 +4,12 @@ import { ImportSourceItemType } from '@/web/core/dataset/type'; import { useQuery } from '@tanstack/react-query'; import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer'; import { getPreviewChunks } from '@/web/core/dataset/api'; -import { useImportStore } from '../Provider'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const PreviewChunks = ({ previewSource, @@ -18,7 +19,10 @@ const PreviewChunks = ({ onClose: () => void; }) => { const { toast } = useToast(); - const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useImportStore(); + const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useContextSelector( + DatasetImportContext, + (v) => v + ); const { data = [], isLoading } = useQuery( ['previewSource'], diff --git a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx index b995d4ed8..8c86d0b69 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx @@ -4,10 +4,11 @@ import { ImportSourceItemType } from '@/web/core/dataset/type'; import { useQuery } from '@tanstack/react-query'; import { getPreviewFileContent } from '@/web/common/file/api'; import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer'; -import { useImportStore } from '../Provider'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const PreviewRawText = ({ previewSource, @@ -17,7 +18,7 @@ const PreviewRawText = ({ onClose: () => void; }) => { const { toast } = useToast(); - const { importSource } = useImportStore(); + const { importSource } = useContextSelector(DatasetImportContext, (v) => v); const { data, isLoading } = useQuery( ['previewSource', previewSource?.dbFileId], diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx new file mode 100644 index 000000000..f00f49bda --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx @@ -0,0 +1,188 @@ +import React, { useEffect } from 'react'; +import dynamic from 'next/dynamic'; +import { useTranslation } from 'next-i18next'; +import { useFieldArray, useForm } from 'react-hook-form'; +import { + Box, + Button, + Flex, + Table, + Thead, + Tbody, + Tr, + Th, + Td, + TableContainer, + Input +} from '@chakra-ui/react'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import Loading from '@fastgpt/web/components/common/MyLoading'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; +import { getFileIcon } from '@fastgpt/global/common/file/icon'; +import { useI18n } from '@/web/context/I18n'; +import { SmallAddIcon } from '@chakra-ui/icons'; + +const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { + loading: () => +}); +const Upload = dynamic(() => import('../commonProgress/Upload')); + +const ExternalFileCollection = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + + return ( + <> + {activeStep === 0 && } + {activeStep === 1 && } + {activeStep === 2 && } + + ); +}; + +export default React.memo(ExternalFileCollection); + +const CustomLinkInput = () => { + const { t } = useTranslation(); + const { datasetT, commonT } = useI18n(); + const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v); + const { register, reset, handleSubmit, control } = useForm<{ + list: { + sourceName: string; + sourceUrl: string; + externalId: string; + }[]; + }>({ + defaultValues: { + list: [ + { + sourceName: '', + sourceUrl: '', + externalId: '' + } + ] + } + }); + + const { + fields: list, + append, + remove, + update + } = useFieldArray({ + control, + name: 'list' + }); + + useEffect(() => { + if (sources.length > 0) { + reset({ + list: sources.map((item) => ({ + sourceName: item.sourceName, + sourceUrl: item.sourceUrl || '', + externalId: item.externalId || '' + })) + }); + } + }, []); + + return ( + + + + + + + + + + + + + {list.map((item, index) => ( + + + + + + + ))} + +
{datasetT('External url')}{datasetT('External id')}{datasetT('filename')}
+ + + + + + + remove(index)} + /> +
+ + + + + + + ); +}; diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx index 5161376dc..befac68ef 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx @@ -1,24 +1,25 @@ import React, { useCallback, useEffect } from 'react'; -import { ImportDataComponentProps } from '@/web/core/dataset/type.d'; import dynamic from 'next/dynamic'; -import { useImportStore } from '../Provider'; import { useTranslation } from 'next-i18next'; import { useForm } from 'react-hook-form'; import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react'; import { getNanoid } from '@fastgpt/global/common/string/tools'; import Loading from '@fastgpt/web/components/common/MyLoading'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); -const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const CustomTet = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -26,9 +27,9 @@ const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => { export default React.memo(CustomTet); -const CustomTextInput = ({ goToNext }: { goToNext: () => void }) => { +const CustomTextInput = () => { const { t } = useTranslation(); - const { sources, setSources } = useImportStore(); + const { sources, goToNext, setSources } = useContextSelector(DatasetImportContext, (v) => v); const { register, reset, handleSubmit } = useForm({ defaultValues: { name: '', diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx index c4b4d7a12..6a7d3349c 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx @@ -1,8 +1,5 @@ import React, { useEffect } from 'react'; -import { ImportDataComponentProps } from '@/web/core/dataset/type.d'; - import dynamic from 'next/dynamic'; -import { useImportStore } from '../Provider'; import { useTranslation } from 'next-i18next'; import { useForm } from 'react-hook-form'; import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react'; @@ -12,17 +9,21 @@ import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { getDocPath } from '@/web/common/system/doc'; import Loading from '@fastgpt/web/components/common/MyLoading'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); -const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const LinkCollection = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -30,10 +31,13 @@ const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => { export default React.memo(LinkCollection); -const CustomLinkImport = ({ goToNext }: { goToNext: () => void }) => { +const CustomLinkImport = () => { const { t } = useTranslation(); const { feConfigs } = useSystemStore(); - const { sources, setSources, processParamsForm } = useImportStore(); + const { goToNext, sources, setSources, processParamsForm } = useContextSelector( + DatasetImportContext, + (v) => v + ); const { register, reset, handleSubmit, watch } = useForm({ defaultValues: { link: '' diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx index efa1692ca..c2f8325e4 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx @@ -1,13 +1,14 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; -import { ImportDataComponentProps, ImportSourceItemType } from '@/web/core/dataset/type.d'; +import { ImportSourceItemType } from '@/web/core/dataset/type.d'; import { Box, Button } from '@chakra-ui/react'; import FileSelector from '../components/FileSelector'; import { useTranslation } from 'next-i18next'; -import { useImportStore } from '../Provider'; import dynamic from 'next/dynamic'; import Loading from '@fastgpt/web/components/common/MyLoading'; import { RenderUploadFiles } from '../components/RenderFiles'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => @@ -16,11 +17,13 @@ const Upload = dynamic(() => import('../commonProgress/Upload')); const fileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx'; -const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const FileLocal = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -28,9 +31,9 @@ const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => { export default React.memo(FileLocal); -const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) { +const SelectFile = React.memo(function SelectFile() { const { t } = useTranslation(); - const { sources, setSources } = useImportStore(); + const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v); const [selectFiles, setSelectFiles] = useState( sources.map((source) => ({ isUploading: false, diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx index b574cb396..9c51d296a 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx @@ -1,24 +1,27 @@ import React, { useEffect, useMemo, useState } from 'react'; -import { ImportDataComponentProps, ImportSourceItemType } from '@/web/core/dataset/type.d'; +import { ImportSourceItemType } from '@/web/core/dataset/type.d'; import { Box, Button } from '@chakra-ui/react'; import FileSelector from '../components/FileSelector'; import { useTranslation } from 'next-i18next'; -import { useImportStore } from '../Provider'; import dynamic from 'next/dynamic'; import { fileDownload } from '@/web/common/file/utils'; import { RenderUploadFiles } from '../components/RenderFiles'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const PreviewData = dynamic(() => import('../commonProgress/PreviewData')); const Upload = dynamic(() => import('../commonProgress/Upload')); const fileType = '.csv'; -const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const FileLocal = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -32,9 +35,9 @@ const csvTemplate = `"第一列内容","第二列内容" "结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。","" "AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`; -const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) { +const SelectFile = React.memo(function SelectFile() { const { t } = useTranslation(); - const { sources, setSources } = useImportStore(); + const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v); const [selectFiles, setSelectFiles] = useState( sources.map((source) => ({ isUploading: false, diff --git a/projects/app/src/pages/dataset/detail/components/Import/index.tsx b/projects/app/src/pages/dataset/detail/components/Import/index.tsx index f72507690..6e2c44a3e 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/index.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/index.tsx @@ -1,147 +1,42 @@ import React, { useMemo } from 'react'; -import { Box, Button, Flex, IconButton } from '@chakra-ui/react'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import { useTranslation } from 'next-i18next'; -import { useRouter } from 'next/router'; -import { TabEnum } from '../../index'; -import { useMyStep } from '@fastgpt/web/hooks/useStep'; +import { Box, Flex } from '@chakra-ui/react'; import dynamic from 'next/dynamic'; -import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; -import Provider from './Provider'; +import { useContextSelector } from 'use-context-selector'; +import DatasetImportContextProvider, { DatasetImportContext } from './Context'; const FileLocal = dynamic(() => import('./diffSource/FileLocal')); const FileLink = dynamic(() => import('./diffSource/FileLink')); const FileCustomText = dynamic(() => import('./diffSource/FileCustomText')); const TableLocal = dynamic(() => import('./diffSource/TableLocal')); +const ExternalFileCollection = dynamic(() => import('./diffSource/ExternalFile')); const ImportDataset = () => { - const { t } = useTranslation(); - const router = useRouter(); - const { datasetDetail } = useDatasetStore(); - const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as { - source: `${ImportDataSourceEnum}`; - parentId?: string; - }; - - const modeSteps: Record<`${ImportDataSourceEnum}`, { title: string }[]> = { - [ImportDataSourceEnum.fileLocal]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ], - [ImportDataSourceEnum.fileLink]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ], - [ImportDataSourceEnum.fileCustom]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ], - [ImportDataSourceEnum.csvTable]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ] - }; - const steps = modeSteps[source]; - - const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({ - defaultStep: 0, - steps - }); + const importSource = useContextSelector(DatasetImportContext, (v) => v.importSource); const ImportComponent = useMemo(() => { - if (source === ImportDataSourceEnum.fileLocal) return FileLocal; - if (source === ImportDataSourceEnum.fileLink) return FileLink; - if (source === ImportDataSourceEnum.fileCustom) return FileCustomText; - if (source === ImportDataSourceEnum.csvTable) return TableLocal; - }, [source]); + if (importSource === ImportDataSourceEnum.fileLocal) return FileLocal; + if (importSource === ImportDataSourceEnum.fileLink) return FileLink; + if (importSource === ImportDataSourceEnum.fileCustom) return FileCustomText; + if (importSource === ImportDataSourceEnum.csvTable) return TableLocal; + if (importSource === ImportDataSourceEnum.externalFile) return ExternalFileCollection; + }, [importSource]); return ImportComponent ? ( - - - {activeStep === 0 ? ( - - } - aria-label={''} - size={'smSquare'} - w={'26px'} - h={'26px'} - borderRadius={'50%'} - variant={'whiteBase'} - mr={2} - onClick={() => - router.replace({ - query: { - ...router.query, - currentTab: TabEnum.collectionCard - } - }) - } - /> - {t('common.Exit')} - - ) : ( - - )} - - - {/* step */} - - - - - - - - - - - + + + ) : null; }; -export default React.memo(ImportDataset); +const Render = () => { + return ( + + + + + + ); +}; + +export default React.memo(Render); diff --git a/projects/app/src/pages/dataset/detail/components/Info.tsx b/projects/app/src/pages/dataset/detail/components/Info.tsx index b26bbd76c..56654ff28 100644 --- a/projects/app/src/pages/dataset/detail/components/Info.tsx +++ b/projects/app/src/pages/dataset/detail/components/Info.tsx @@ -23,13 +23,14 @@ import type { VectorModelItemType } from '@fastgpt/global/core/ai/model.d'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import MyDivider from '@fastgpt/web/components/common/MyDivider/index'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; const Info = ({ datasetId }: { datasetId: string }) => { const { t } = useTranslation(); const { datasetT } = useI18n(); - const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore(); - const rebuildingCount = useContextSelector(DatasetPageContext, (v) => v.rebuildingCount); - const trainingCount = useContextSelector(DatasetPageContext, (v) => v.trainingCount); + const { datasetDetail, loadDatasetDetail, updateDataset, rebuildingCount, trainingCount } = + useContextSelector(DatasetPageContext, (v) => v); + const refetchDatasetTraining = useContextSelector( DatasetPageContext, (v) => v.refetchDatasetTraining @@ -82,9 +83,6 @@ const Info = ({ datasetId }: { datasetId: string }) => { ...data }); }, - onSuccess() { - loadDatasets(); - }, successToast: t('common.Update Success'), errorToast: t('common.Update Failed') }); @@ -117,7 +115,7 @@ const Info = ({ datasetId }: { datasetId: string }) => { }, onSuccess() { refetchDatasetTraining(); - loadDatasetDetail(datasetId, true); + loadDatasetDetail(datasetId); }, successToast: datasetT('Rebuild embedding start tip'), errorToast: t('common.Update Failed') @@ -128,16 +126,16 @@ const Info = ({ datasetId }: { datasetId: string }) => { return ( - + {t('core.dataset.Dataset ID')} {datasetDetail._id} - + {t('core.ai.model.Vector Model')} - + { - + {t('core.Max Token')} - {vectorModel.maxToken} + {vectorModel.maxToken} - + {t('core.ai.model.Dataset Agent Model')} - + { - + + + {datasetDetail.type === DatasetTypeEnum.externalFile && ( + <> + + + {datasetT('External read url')} + + + + + + )} - + {t('core.dataset.Avatar')} - + { - + {t('core.dataset.Name')} - + - {t('common.Intro')} -