From c031e6dcc9a2e16abe6e0cd37784fdc0dc38cba4 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 19 Jan 2024 11:17:28 +0800 Subject: [PATCH] 4.6.7-alpha commit (#743) Co-authored-by: Archer <545436317@qq.com> Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com> --- docSite/content/docs/commercial/intro.md | 10 +- .../content/docs/development/configuration.md | 2 +- .../docs/development/openapi/dataset.md | 1155 ++++++++++++++++- docSite/content/docs/development/qa.md | 11 +- .../content/docs/development/upgrading/467.md | 33 + .../content/docs/use-cases/datasetEngine.md | 4 +- packages/global/common/file/icon.ts | 10 +- .../global/common/file/image/constants.ts | 6 +- packages/global/common/file/image/type.d.ts | 5 +- packages/global/common/string/textSplitter.ts | 11 +- .../global/common/system/types/index.d.ts | 2 + packages/global/core/app/type.d.ts | 2 +- packages/global/core/app/utils.ts | 2 +- packages/global/core/chat/type.d.ts | 3 +- packages/global/core/dataset/api.d.ts | 24 +- .../dataset/{constant.ts => constants.ts} | 14 +- packages/global/core/dataset/type.d.ts | 4 +- packages/global/core/dataset/utils.ts | 22 +- packages/global/core/module/constants.ts | 11 + packages/global/core/module/node/constant.ts | 3 +- .../core/module/template/system/aiChat.ts | 41 +- .../module/template/system/assignedAnswer.ts | 12 +- .../template/system/classifyQuestion.ts | 18 +- .../module/template/system/contextExtract.ts | 6 +- .../module/template/system/datasetSearch.ts | 22 +- .../core/module/template/system/http.ts | 4 +- .../core/module/template/system/runApp.ts | 4 +- .../core/module/template/system/runPlugin.ts | 2 +- .../core/module/template/system/userGuide.ts | 10 +- .../core/module/template/system/userInput.ts | 8 +- packages/global/core/module/template/tip.ts | 11 +- packages/global/core/module/utils.ts | 11 +- packages/global/support/user/api.d.ts | 1 - packages/global/support/user/type.d.ts | 1 + packages/global/support/wallet/bill/api.d.ts | 3 +- packages/global/support/wallet/bill/type.d.ts | 3 +- .../service/common/file/gridfs/controller.ts | 41 +- packages/service/common/file/gridfs/schema.ts | 15 + .../service/common/file/image/controller.ts | 4 +- packages/service/common/file/image/schema.ts | 2 + packages/service/common/file/load/pdf.ts | 68 - packages/service/common/file/load/type.d.ts | 18 - packages/service/common/file/load/utils.ts | 50 - packages/service/common/file/load/word.ts | 22 - packages/service/common/file/multer.ts | 36 +- packages/service/common/file/utils.ts | 22 - packages/service/common/string/cheerio.ts | 62 +- .../common/vectorStore/controller.d.ts | 10 +- .../service/common/vectorStore/controller.ts | 30 +- .../service/common/vectorStore/pg/class.ts | 6 +- .../common/vectorStore/pg/controller.ts | 112 +- packages/service/common/vectorStore/type.d.ts | 1 - packages/service/core/ai/embedding/index.ts | 9 +- packages/service/core/chat/chatItemSchema.ts | 25 +- packages/service/core/chat/chatSchema.ts | 20 +- packages/service/core/chat/controller.ts | 9 +- .../core/dataset/collection/controller.ts | 72 +- .../service/core/dataset/collection/schema.ts | 20 +- .../service/core/dataset/collection/utils.ts | 35 +- packages/service/core/dataset/controller.ts | 66 +- .../service/core/dataset/data/controller.ts | 85 -- packages/service/core/dataset/data/schema.ts | 20 +- packages/service/core/dataset/schema.ts | 2 +- .../service/core/dataset/training/schema.ts | 8 +- packages/service/package.json | 2 - .../service/support/permission/auth/app.ts | 4 +- .../support/permission/auth/dataset.ts | 62 +- .../support/permission/auth/openapi.ts | 4 +- .../support/permission/auth/outLink.ts | 4 +- .../service/support/permission/auth/plugin.ts | 6 +- .../service/support/permission/auth/user.ts | 6 +- .../support/permission/limit/dataset.ts | 2 +- packages/service/support/user/controller.ts | 10 +- packages/service/support/user/schema.ts | 9 + .../service/support/user/team/controller.ts | 8 +- .../service/support/user/team/teamSchema.ts | 6 +- .../service/support/wallet/bill/controller.ts | 6 +- .../service/support/wallet/bill/schema.ts | 6 +- packages/web/common/file/read/html.ts | 2 +- .../web/components/common/Icon/constants.ts | 34 +- .../common/Icon/icons/common/closeLight.svg | 5 +- .../common/Icon/icons/common/errorFill.svg | 6 + .../common/Icon/icons/common/folderFill.svg | 5 + .../common/Icon/icons/common/gitInlight.svg | 4 + .../common/Icon/icons/common/importLight.svg | 7 +- .../common/Icon/icons/common/linkBlue.svg | 5 + .../Icon/icons/common/overviewLight.svg | 5 +- .../Icon/icons/common/questionLight.svg | 2 +- .../common/Icon/icons/common/saveFill.svg | 4 + .../common/Icon/icons/common/selectLight.svg | 6 + .../common/Icon/icons/common/settingLight.svg | 7 +- .../Icon/icons/common/uploadFileFill.svg | 5 + .../common/Icon/icons/common/viewLight.svg | 5 +- .../common/Icon/icons/core/app/logsLight.svg | 16 +- .../Icon/icons/core/app/questionGuide.svg | 6 +- .../Icon/icons/core/app/simpleMode/ai.svg | 5 + .../Icon/icons/core/app/simpleMode/chat.svg | 5 + .../icons/core/app/simpleMode/dataset.svg | 11 + .../icons/core/app/simpleMode/template.svg | 11 + .../Icon/icons/core/app/simpleMode/tts.svg | 5 + .../icons/core/app/simpleMode/variable.svg | 11 + .../common/Icon/icons/core/app/tts.svg | 3 - .../common/Icon/icons/core/chat/chatFill.svg | 4 +- .../icons/core/dataset/fileCollection.svg | 4 + .../Icon/icons/core/dataset/folderDataset.svg | 1 - .../icons/core/dataset/manualCollection.svg | 4 + .../icons/core/dataset/tableCollection.svg | 11 + .../Icon/icons/core/modules/flowLight.svg | 4 + .../web/components/common/Icon/icons/edit.svg | 9 +- .../components/common/Icon/icons/export.svg | 5 +- .../common/Icon/icons/file/fill}/csv.svg | 0 .../common/Icon/icons/file/fill}/doc.svg | 0 .../common/Icon/icons/file/fill}/file.svg | 0 .../common/Icon/icons/file/fill/folder.svg | 5 + .../common/Icon/icons/file/fill}/html.svg | 0 .../common/Icon/icons/file/fill/manual.svg | 5 + .../common/Icon/icons/file/fill}/markdown.svg | 0 .../common/Icon/icons/file/fill/pdf.svg | 11 + .../common/Icon/icons/file/fill}/txt.svg | 0 .../common/Icon/icons/file/markdown.svg | 12 +- .../components/common/Icon/icons/file/pdf.svg | 13 +- .../components/common/Icon/icons/kbTest.svg | 5 +- .../common/Icon/icons/modal/edit.svg | 11 + .../common/Icon/icons/modal/manualDataset.svg | 5 + .../common/Icon/icons/modal/selectSource.svg | 5 + .../common/Icon/icons/phoneTabbar/more.svg | 1 - .../common/Icon/icons/phoneTabbar/tool.svg | 4 + .../Icon/icons/phoneTabbar/toolFill.svg | 12 + .../Icon/icons/support/outlink/shareLight.svg | 8 +- .../icons/support/permission/privateLight.svg | 10 +- .../web/components/common/MyModal/index.tsx | 92 ++ .../web/components/common/MyTooltip/index.tsx | 31 + .../web/components/common/Radio/LeftRadio.tsx | 112 ++ .../web/components/common/Tabs/RowTabs.tsx | 54 + .../common/Textarea/PromptEditor/Editor.tsx | 135 ++ .../Textarea/PromptEditor/index.module.scss | 24 + .../common/Textarea/PromptEditor/index.tsx | 83 ++ .../PromptEditor/modules/ComfirmVar/index.tsx | 96 ++ .../plugins/OnBlurPlugin/index.tsx | 24 + .../plugins/VariablePickerPlugin/index.tsx | 143 ++ .../plugins/VariablePlugin/index.tsx | 49 + .../plugins/VariablePlugin/node.ts | 49 + .../plugins/VariablePlugin/utils.ts | 223 ++++ .../common/Textarea/PromptEditor/type.d.ts | 5 + .../common/Textarea/PromptEditor/utils.ts | 275 ++++ packages/web/hooks/useStep.tsx | 98 ++ packages/web/package.json | 6 +- pnpm-lock.yaml | 395 +++++- .../data/simpleTemplates/fastgpt-simple.json | 4 +- projects/app/next.config.js | 2 +- projects/app/package.json | 2 +- projects/app/public/docs/versionIntro.md | 21 +- projects/app/public/imgs/files/collection.svg | 1 - projects/app/public/imgs/files/folder.svg | 6 +- projects/app/public/imgs/files/link.svg | 1 - projects/app/public/imgs/files/manual.svg | 1 - projects/app/public/imgs/files/mark.svg | 1 - projects/app/public/imgs/files/pdf.svg | 1 - projects/app/public/imgs/modal/folder.svg | 1 - projects/app/public/imgs/module/ai.svg | 12 +- projects/app/public/imgs/module/templates.png | Bin 2444 -> 0 bytes projects/app/public/js/iframe.js | 2 +- projects/app/public/locales/en/common.json | 323 ++++- projects/app/public/locales/zh/common.json | 249 +++- .../src/components/ChatBox/FeedbackModal.tsx | 2 +- .../src/components/ChatBox/ResponseTags.tsx | 3 +- .../ChatBox/SelectMarkCollection.tsx | 4 +- .../components/ChatBox/WholeResponseModal.tsx | 3 +- projects/app/src/components/Layout/navbar.tsx | 4 +- .../app/src/components/Layout/navbarPhone.tsx | 35 +- projects/app/src/components/Loading/index.tsx | 1 + projects/app/src/components/MyMenu/index.tsx | 6 +- projects/app/src/components/MyModal/index.tsx | 72 +- projects/app/src/components/Tag/index.tsx | 10 +- .../src/components/common/MyRadio/index.tsx | 2 +- .../common/Textarea/MyTextarea/index.tsx | 95 ++ .../common/Textarea/PromptTextarea/index.tsx | 97 -- .../core/dataset/DatasetTypeTag.tsx | 4 +- .../src/components/core/dataset/QuoteItem.tsx | 4 +- .../components/core/dataset/RawSourceBox.tsx | 3 +- .../components/core/dataset/SelectModal.tsx | 2 +- .../core/module/AIChatSettingsModal.tsx | 94 +- .../core/module/DatasetParamsModal.tsx | 10 +- .../core/module/DatasetSelectModal.tsx | 2 +- .../core/module/Flow/SelectAppModal.tsx | 5 +- .../Flow/components/modules/ButtonEdge.tsx | 40 +- .../Flow/components/modules/QGSwitch.tsx | 2 +- .../Flow/components/modules/TTSSelect.tsx | 4 +- .../Flow/components/modules/VariableEdit.tsx | 14 +- .../Flow/components/nodes/NodeCQNode.tsx | 2 +- .../Flow/components/nodes/NodeUserGuide.tsx | 51 +- .../components/nodes/abandon/NodeVariable.tsx | 60 - .../components/render/RenderInput/index.tsx | 81 +- .../RenderInput/templates/SelectDataset.tsx | 120 +- .../templates/SelectDatasetParams.tsx | 83 +- .../render/RenderInput/templates/Textarea.tsx | 55 +- .../src/components/core/module/Flow/index.tsx | 96 +- .../src/components/support/apikey/Table.tsx | 38 +- .../src/components/support/wallet/Price.tsx | 20 +- .../app/src/global/core/api/datasetReq.d.ts | 4 +- projects/app/src/global/core/app/constants.ts | 6 +- projects/app/src/global/core/dataset/api.d.ts | 2 +- projects/app/src/global/core/prompt/AIChat.ts | 8 +- projects/app/src/pages/_app.tsx | 2 +- .../pages/account/components/BillDetail.tsx | 28 +- .../app/src/pages/account/components/Info.tsx | 4 +- projects/app/src/pages/api/admin/initv46-2.ts | 184 --- .../app/src/pages/api/admin/initv46-fix.ts | 173 --- projects/app/src/pages/api/admin/initv46.ts | 330 ----- projects/app/src/pages/api/admin/initv463.ts | 2 +- projects/app/src/pages/api/admin/initv467.ts | 106 ++ .../timeTasks/checkUnValidDatasetFiles.ts | 95 -- .../app/src/pages/api/common/file/upload.ts | 30 +- .../pages/api/common/system/getInitData.ts | 3 +- .../core/app/form2Modules/fastgpt-simple.ts | 40 +- .../app/form2Modules/fastgpt-universal.ts | 38 +- .../app/src/pages/api/core/app/getChatLogs.ts | 19 +- .../src/pages/api/core/chat/clearHistories.ts | 3 +- .../app/src/pages/api/core/chat/delHistory.ts | 2 + .../api/core/chat/feedback/adminUpdate.ts | 2 + .../api/core/chat/feedback/closeCustom.ts | 13 +- .../core/chat/feedback/updateUserFeedback.ts | 1 + .../src/pages/api/core/chat/getHistories.ts | 2 +- projects/app/src/pages/api/core/chat/init.ts | 3 +- .../src/pages/api/core/chat/item/delete.ts | 5 +- .../src/pages/api/core/chat/item/getSpeech.ts | 2 +- .../src/pages/api/core/chat/outLink/init.ts | 3 +- .../src/pages/api/core/chat/updateHistory.ts | 2 +- .../src/pages/api/core/dataset/allDataset.ts | 2 +- .../core/dataset/collection/create/file.ts | 85 ++ .../collection/{apiCreate => create}/link.ts | 5 +- .../collection/{apiCreate => create}/text.ts | 12 +- .../api/core/dataset/collection/delete.ts | 17 +- .../pages/api/core/dataset/collection/list.ts | 10 +- .../api/core/dataset/collection/sync/link.ts | 11 +- .../app/src/pages/api/core/dataset/create.ts | 2 +- .../src/pages/api/core/dataset/data/delete.ts | 20 +- .../pages/api/core/dataset/data/insertData.ts | 5 +- .../src/pages/api/core/dataset/data/list.ts | 10 +- .../pages/api/core/dataset/data/pushData.ts | 5 +- .../src/pages/api/core/dataset/data/update.ts | 4 +- .../app/src/pages/api/core/dataset/delete.ts | 19 +- .../src/pages/api/core/dataset/exportAll.ts | 11 +- .../app/src/pages/api/core/dataset/list.ts | 2 +- .../src/pages/api/core/dataset/searchTest.ts | 5 +- .../pages/api/plugins/customFeedback/index.ts | 2 + .../support/user/account/loginByPassword.ts | 11 +- .../user/team/limit/datasetSizeLimit.ts | 37 + .../support/wallet/bill/createTrainingBill.ts | 19 +- .../src/pages/api/v1/audio/transcriptions.ts | 8 +- .../app/src/pages/api/v1/chat/completions.ts | 7 +- projects/app/src/pages/api/v1/embeddings.ts | 8 +- .../app/detail/components/FlowEdit/Header.tsx | 45 +- .../app/detail/components/FlowEdit/index.tsx | 10 +- .../pages/app/detail/components/InfoModal.tsx | 31 +- .../app/detail/components/OutLink/Share.tsx | 46 +- .../app/detail/components/OutLink/index.tsx | 12 +- .../detail/components/SimpleEdit/AppCard.tsx | 8 +- .../components/SimpleEdit/CfrEditModal.tsx | 31 +- .../detail/components/SimpleEdit/ChatTest.tsx | 2 +- .../detail/components/SimpleEdit/EditForm.tsx | 118 +- projects/app/src/pages/app/detail/index.tsx | 38 +- .../pages/app/list/component/CreateModal.tsx | 27 +- projects/app/src/pages/chat/index.tsx | 1 - .../dataset/component/EditFolderModal.tsx | 7 +- .../detail/components/CollectionCard.tsx | 87 +- .../dataset/detail/components/DataCard.tsx | 415 +++--- .../detail/components/Import/Chunk.tsx | 166 --- .../components/Import/CreateFileModal.tsx | 72 - .../dataset/detail/components/Import/Csv.tsx | 90 -- .../detail/components/Import/FileSelect.tsx | 446 ------- .../detail/components/Import/ImportModal.tsx | 138 -- .../detail/components/Import/Provider.tsx | 629 +++------ .../dataset/detail/components/Import/QA.tsx | 112 -- .../components/Import/UrlFetchModal.tsx | 94 -- .../Import/commonProgress/DataProcess.tsx | 356 +++++ .../Import/commonProgress/PreviewData.tsx | 30 + .../Import/commonProgress/Upload.tsx | 298 +++++ .../components/Import/components/Preview.tsx | 134 ++ .../Import/components/PreviewRawText.tsx | 28 + .../Import/diffSource/FileCustomText.tsx | 103 ++ .../components/Import/diffSource/FileLink.tsx | 147 +++ .../Import/diffSource/FileLocal.tsx | 177 +++ .../Import/diffSource/TableLocal.tsx | 168 +++ .../detail/components/Import/index.tsx | 154 +++ .../sourceSelector/FileSourceSelector.tsx | 65 + .../pages/dataset/detail/components/Info.tsx | 61 +- .../detail/components/InputDataModal.tsx | 132 +- .../pages/dataset/detail/components/Test.tsx | 15 +- .../app/src/pages/dataset/detail/index.tsx | 25 +- .../dataset/list/component/CreateModal.tsx | 74 +- .../dataset/list/component/MoveModal.tsx | 2 +- projects/app/src/pages/dataset/list/index.tsx | 18 +- .../src/pages/login/components/LoginForm.tsx | 2 +- projects/app/src/pages/login/index.tsx | 2 +- .../service/core/dataset/data/controller.ts | 138 +- .../src/service/core/dataset/data/utils.ts | 7 +- projects/app/src/service/events/generateQA.ts | 11 +- .../app/src/service/events/generateVector.ts | 6 +- .../service/moduleDispatch/dataset/search.ts | 12 +- .../app/src/service/moduleDispatch/index.ts | 3 +- .../service/support/permission/auth/chat.ts | 1 + .../support/permission/auth/dataset.ts | 2 +- .../src/service/support/wallet/bill/push.ts | 41 +- .../app/src/service/utils/chat/saveChat.ts | 2 +- projects/app/src/utils/tools.ts | 10 +- .../app/src/web/common/file/controller.ts | 2 +- .../web/common/file/hooks/useSelectFile.tsx | 4 +- .../app/src/web/common/hooks/useConfirm.tsx | 4 +- .../app/src/web/common/hooks/useEditTitle.tsx | 16 +- .../app/src/web/common/system/staticData.ts | 10 +- projects/app/src/web/core/app/templates.ts | 106 +- projects/app/src/web/core/app/utils.ts | 2 +- projects/app/src/web/core/dataset/api.ts | 6 +- .../core/dataset/components/FileSelector.tsx | 200 +++ .../dataset/components/SelectCollections.tsx | 12 +- .../app/src/web/core/dataset/constants.ts | 4 + .../app/src/web/core/dataset/store/dataset.ts | 5 +- .../src/web/core/dataset/store/searchTest.ts | 2 +- projects/app/src/web/core/dataset/type.d.ts | 38 + projects/app/src/web/core/dataset/utils.ts | 52 +- .../src/web/core/modules/template/system.ts | 14 +- projects/app/src/web/styles/theme.ts | 44 +- projects/app/src/web/support/user/team/api.ts | 2 + 324 files changed, 8509 insertions(+), 4757 deletions(-) create mode 100644 docSite/content/docs/development/upgrading/467.md rename packages/global/core/dataset/{constant.ts => constants.ts} (91%) create mode 100644 packages/service/common/file/gridfs/schema.ts delete mode 100644 packages/service/common/file/load/pdf.ts delete mode 100644 packages/service/common/file/load/type.d.ts delete mode 100644 packages/service/common/file/load/utils.ts delete mode 100644 packages/service/common/file/load/word.ts create mode 100644 packages/web/components/common/Icon/icons/common/errorFill.svg create mode 100644 packages/web/components/common/Icon/icons/common/folderFill.svg create mode 100644 packages/web/components/common/Icon/icons/common/gitInlight.svg create mode 100644 packages/web/components/common/Icon/icons/common/linkBlue.svg create mode 100644 packages/web/components/common/Icon/icons/common/saveFill.svg create mode 100644 packages/web/components/common/Icon/icons/common/selectLight.svg create mode 100644 packages/web/components/common/Icon/icons/common/uploadFileFill.svg create mode 100644 packages/web/components/common/Icon/icons/core/app/simpleMode/ai.svg create mode 100644 packages/web/components/common/Icon/icons/core/app/simpleMode/chat.svg create mode 100644 packages/web/components/common/Icon/icons/core/app/simpleMode/dataset.svg create mode 100644 packages/web/components/common/Icon/icons/core/app/simpleMode/template.svg create mode 100644 packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg create mode 100644 packages/web/components/common/Icon/icons/core/app/simpleMode/variable.svg delete mode 100644 packages/web/components/common/Icon/icons/core/app/tts.svg create mode 100644 packages/web/components/common/Icon/icons/core/dataset/fileCollection.svg delete mode 100644 packages/web/components/common/Icon/icons/core/dataset/folderDataset.svg create mode 100644 packages/web/components/common/Icon/icons/core/dataset/manualCollection.svg create mode 100644 packages/web/components/common/Icon/icons/core/dataset/tableCollection.svg create mode 100644 packages/web/components/common/Icon/icons/core/modules/flowLight.svg rename {projects/app/public/imgs/files => packages/web/components/common/Icon/icons/file/fill}/csv.svg (100%) rename {projects/app/public/imgs/files => packages/web/components/common/Icon/icons/file/fill}/doc.svg (100%) rename {projects/app/public/imgs/files => packages/web/components/common/Icon/icons/file/fill}/file.svg (100%) create mode 100644 packages/web/components/common/Icon/icons/file/fill/folder.svg rename {projects/app/public/imgs/files => packages/web/components/common/Icon/icons/file/fill}/html.svg (100%) create mode 100644 packages/web/components/common/Icon/icons/file/fill/manual.svg rename {projects/app/public/imgs/files => packages/web/components/common/Icon/icons/file/fill}/markdown.svg (100%) create mode 100644 packages/web/components/common/Icon/icons/file/fill/pdf.svg rename {projects/app/public/imgs/files => packages/web/components/common/Icon/icons/file/fill}/txt.svg (100%) create mode 100644 packages/web/components/common/Icon/icons/modal/edit.svg create mode 100644 packages/web/components/common/Icon/icons/modal/manualDataset.svg create mode 100644 packages/web/components/common/Icon/icons/modal/selectSource.svg delete mode 100644 packages/web/components/common/Icon/icons/phoneTabbar/more.svg create mode 100644 packages/web/components/common/Icon/icons/phoneTabbar/tool.svg create mode 100644 packages/web/components/common/Icon/icons/phoneTabbar/toolFill.svg create mode 100644 packages/web/components/common/MyModal/index.tsx create mode 100644 packages/web/components/common/MyTooltip/index.tsx create mode 100644 packages/web/components/common/Radio/LeftRadio.tsx create mode 100644 packages/web/components/common/Tabs/RowTabs.tsx create mode 100644 packages/web/components/common/Textarea/PromptEditor/Editor.tsx create mode 100644 packages/web/components/common/Textarea/PromptEditor/index.module.scss create mode 100644 packages/web/components/common/Textarea/PromptEditor/index.tsx create mode 100644 packages/web/components/common/Textarea/PromptEditor/modules/ComfirmVar/index.tsx create mode 100644 packages/web/components/common/Textarea/PromptEditor/plugins/OnBlurPlugin/index.tsx create mode 100644 packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx create mode 100644 packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/index.tsx create mode 100644 packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/node.ts create mode 100644 packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/utils.ts create mode 100644 packages/web/components/common/Textarea/PromptEditor/type.d.ts create mode 100644 packages/web/components/common/Textarea/PromptEditor/utils.ts create mode 100644 packages/web/hooks/useStep.tsx delete mode 100644 projects/app/public/imgs/files/collection.svg delete mode 100644 projects/app/public/imgs/files/link.svg delete mode 100644 projects/app/public/imgs/files/manual.svg delete mode 100644 projects/app/public/imgs/files/mark.svg delete mode 100644 projects/app/public/imgs/files/pdf.svg delete mode 100644 projects/app/public/imgs/modal/folder.svg delete mode 100644 projects/app/public/imgs/module/templates.png create mode 100644 projects/app/src/components/common/Textarea/MyTextarea/index.tsx delete mode 100644 projects/app/src/components/common/Textarea/PromptTextarea/index.tsx delete mode 100644 projects/app/src/components/core/module/Flow/components/nodes/abandon/NodeVariable.tsx delete mode 100644 projects/app/src/pages/api/admin/initv46-2.ts delete mode 100644 projects/app/src/pages/api/admin/initv46-fix.ts delete mode 100644 projects/app/src/pages/api/admin/initv46.ts create mode 100644 projects/app/src/pages/api/admin/initv467.ts delete mode 100644 projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts create mode 100644 projects/app/src/pages/api/core/dataset/collection/create/file.ts rename projects/app/src/pages/api/core/dataset/collection/{apiCreate => create}/link.ts (96%) rename projects/app/src/pages/api/core/dataset/collection/{apiCreate => create}/text.ts (87%) create mode 100644 projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/CreateFileModal.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/Csv.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/QA.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/components/Preview.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/index.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/sourceSelector/FileSourceSelector.tsx create mode 100644 projects/app/src/web/core/dataset/components/FileSelector.tsx create mode 100644 projects/app/src/web/core/dataset/constants.ts create mode 100644 projects/app/src/web/core/dataset/type.d.ts diff --git a/docSite/content/docs/commercial/intro.md b/docSite/content/docs/commercial/intro.md index 53fc2ed21..cf9733d48 100644 --- a/docSite/content/docs/commercial/intro.md +++ b/docSite/content/docs/commercial/intro.md @@ -44,15 +44,19 @@ FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。 **特有服务** {{< table "table-hover table-striped-columns" >}} -| 部署方式 | 特有服务 | 上线时长 | 价格 | +| 部署方式 | 特有服务 | 上线时长 | 标品价格 | | ---- | ---- | ---- | ---- | | Sealos全托管 | 1. 有效期内免费升级。
2. 免运维服务&数据库。 | 半天 | 3000元起/月（3个月起）
或
30000元起/年 | | 自有服务器-单机版 | 1. 6个版本的升级服务。 | 14天内 | 60000元/套（不限时长） | -| 自有服务器-Sealos版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）| +| 自有服务器-高可用版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）| {{< /table >}} {{% alert icon="🤖 " context="success" %}} -6个版本的升级服务不是指只能用 6 个版本，而是指依赖 FastGPT 团队提供的升级服务。大部分时候，建议自行升级，也不麻烦。 +- 6个版本的升级服务不是指只能用 6 个版本，而是指依赖 FastGPT 团队提供的升级服务。大部分时候，建议自行升级，也不麻烦。 +- 全托管版本适合技术人员紧缺的团队，仅需关注业务推动，无需关心服务是否正常运行。 +- 单机版和高可用版可以完全部署在自己服务器中。 +- 单机版适合中小团队对内提供服务，需要自己维护数据库备份等。 +- 高可用版适合对外提供在线服务，包含可视化监控、多副本、负载均衡、数据库自动备份等生产环境的基础设施。 {{% /alert %}} diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md index 054204ff0..d082645d6 100644 --- a/docSite/content/docs/development/configuration.md +++ b/docSite/content/docs/development/configuration.md @@ -277,7 +277,7 @@ weight: 708 "maxContext": 1600, "maxResponse": 4000, "inputPrice": 0, - "outputPrice": 0, + "outputPrice": 0 } ], "vectorModels": [ // 向量模型 diff --git a/docSite/content/docs/development/openapi/dataset.md b/docSite/content/docs/development/openapi/dataset.md index bcad43f57..efa70009c 100644 --- a/docSite/content/docs/development/openapi/dataset.md +++ b/docSite/content/docs/development/openapi/dataset.md @@ -51,7 +51,7 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/data/pus --header 'Content-Type: application/json' \ --data-raw '{     "collectionId": "64663f451ba1676dbdef0499", - "mode": "chunk", + "trainingMode": "chunk", "prompt": "可选。qa 拆分引导词，chunk 模式下忽略", "billId": "可选。如果有这个值，本次的数据会被聚合到一个订单中，这个值可以重复使用。可以参考 [创建训练订单] 获取该值。",     "data": [ @@ -220,3 +220,1156 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe {{< /tab >}} {{< /tabs >}} + + +# 更多接口 + +目前未整理，简陋导出： + +## POST 知识库搜索测试 + +POST /core/dataset/searchTest + +> Body Parameters + +```json +{ + "datasetId": "656c2ccff7f114064daa72f6", + "text": "导演是谁", + "limit": 1500, + "searchMode": "embedding", + "usingReRank": true, + "similarity": 0.5 +} +``` + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|Authorization|header|string| no |none| +|body|body|object| no |none| +|» datasetId|body|string| yes |none| +|» text|body|string| yes |none| +|» limit|body|integer| no |none| +|» searchMode|body|[search mode](#schemasearch%20mode)| yes |none| +|» usingReRank|body|boolean| no |none| +|» similarity|body|[similary](#schemasimilary)| no |none| + +> Response Examples + +> 成功 + +```json +{ + "code": 200, + "statusText": "", + "message": "", + "data": { + "list": [ + { + "id": "65962b23f5fac58e46330dfd", + "q": "# 快速了解 FastGPT\nFastGPT 的能力与优势\n\nFastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的问答场景！\n\n🤖\n\nFastGPT 在线使用：[https://fastgpt.in](https://fastgpt.in)\n\n| | |\n| --- | --- |\n| ![](https://doc.fastgpt.in/imgs/intro1.png) | ![](https://doc.fastgpt.in/imgs/intro2.png) |\n| ![](https://doc.fastgpt.in/imgs/intro3.png) | ![](https://doc.fastgpt.in/imgs/intro4.png) |\n\n", + "a": "", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65962b2089642fd209da3b03", + "sourceName": "https://doc.fastgpt.in/docs/intro/", + "sourceId": "https://doc.fastgpt.in/docs/intro/", + "score": [ + { + "type": "embedding", + "value": 0.8036568760871887, + "index": 20 + }, + { + "type": "fullText", + "value": 1.168349443855932, + "index": 2 + }, + { + "type": "reRank", + "value": 0.9870296135626316, + "index": 0 + }, + { + "type": "rrf", + "value": 0.04366449476962486, + "index": 0 + } + ] + }, + { + "id": "65962b24f5fac58e46330dff", + "q": "# 快速了解 FastGPT\n## FastGPT 能力\n### 2. 简单易用的可视化界面\nFastGPT 采用直观的可视化界面设计，为各种应用场景提供了丰富实用的功能。通过简洁易懂的操作步骤，可以轻松完成 AI 客服的创建和训练流程。\n\n![](https://doc.fastgpt.in/imgs/ability5.png)\n\n", + "a": "", + "chunkIndex": 2, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65962b2089642fd209da3b03", + "sourceName": "https://doc.fastgpt.in/docs/intro/", + "sourceId": "https://doc.fastgpt.in/docs/intro/", + "score": [ + { + "type": "embedding", + "value": 0.8152669668197632, + "index": 3 + }, + { + "type": "fullText", + "value": 1.0511363636363635, + "index": 8 + }, + { + "type": "reRank", + "value": 0.9287972729281414, + "index": 14 + }, + { + "type": "rrf", + "value": 0.04265696347031964, + "index": 1 + } + ] + }, + { + "id": "65962b25f5fac58e46330e00", + "q": "# 快速了解 FastGPT\n## FastGPT 能力\n### 3. 自动数据预处理\n提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径，其中“直接分段”支持通过 PDF、WORD、Markdown 和 CSV 文档内容作为上下文。FastGPT 会自动对文本数据进行预处理、向量化和 QA 分割，节省手动训练时间，提升效能。\n\n![](https://doc.fastgpt.in/imgs/ability2.png)\n\n", + "a": "", + "chunkIndex": 3, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65962b2089642fd209da3b03", + "sourceName": "https://doc.fastgpt.in/docs/intro/", + "sourceId": "https://doc.fastgpt.in/docs/intro/", + "score": [ + { + "type": "embedding", + "value": 0.8158369064331055, + "index": 2 + }, + { + "type": "fullText", + "value": 1.014030612244898, + "index": 20 + }, + { + "type": "reRank", + "value": 0.9064876908461501, + "index": 17 + }, + { + "type": "rrf", + "value": 0.04045823457588163, + "index": 2 + } + ] + }, + { + "id": "65a7e1e8fc13bdf20fd46d41", + "q": "# 快速了解 FastGPT\n## FastGPT 能力\n### 5. 强大的 API 集成\nFastGPT 对外的 API 接口对齐了 OpenAI 官方接口，可以直接接入现有的 GPT 应用，也可以轻松集成到企业微信、公众号、飞书等平台。\n\n![](https://doc.fastgpt.in/imgs/ability4.png)", + "a": "", + "chunkIndex": 66, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7e1d4fc13bdf20fd46abe", + "sourceName": "dataset - 2024-01-04T151625.388.csv", + "sourceId": "65a7e1d2fc13bdf20fd46abc", + "score": [ + { + "type": "embedding", + "value": 0.803692102432251, + "index": 18 + }, + { + "type": "fullText", + "value": 1.0511363636363635, + "index": 7 + }, + { + "type": "reRank", + "value": 0.9177460552422909, + "index": 15 + }, + { + "type": "rrf", + "value": 0.03970501147383226, + "index": 3 + } + ] + }, + { + "id": "65a7be319d96e21823f69c9b", + "q": "FastGPT Flow 的工作流设计方案提供了哪些操作？", + "a": "FastGPT Flow 的工作流设计方案提供了数据预处理、各类 AI 应用设置、调试测试及结果反馈等操作。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8283981680870056, + "index": 0 + }, + { + "type": "reRank", + "value": 0.9620363047907355, + "index": 4 + }, + { + "type": "rrf", + "value": 0.03177805800756621, + "index": 4 + } + ] + }, + { + "id": "65a7be389d96e21823f69d58", + "q": "FastGPT Flow 的实验室预约示例中使用了哪些参数？", + "a": "FastGPT Flow 的实验室预约示例中使用了姓名、时间和实验室名称等参数。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8143455386161804, + "index": 9 + }, + { + "type": "reRank", + "value": 0.9806919138043485, + "index": 1 + }, + { + "type": "rrf", + "value": 0.0304147465437788, + "index": 5 + } + ] + }, + { + "id": "65a7be309d96e21823f69c78", + "q": "FastGPT Flow 是什么？", + "a": "FastGPT Flow 是一款基于大型语言模型的知识库问答系统，通过引入 Flow 可视化工作流编排技术，提供了一个即插即用的解决方案。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8115077018737793, + "index": 11 + }, + { + "type": "reRank", + "value": 0.9686195704870232, + "index": 3 + }, + { + "type": "rrf", + "value": 0.029513888888888888, + "index": 6 + } + ] + }, + { + "id": "65a7be389d96e21823f69d5e", + "q": "FastGPT Flow 的实验室预约示例中的代码实现了哪些功能？", + "a": "FastGPT Flow 的实验室预约示例中的代码实现了预约实验室、修改预约、查询预约和取消预约等功能。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8166953921318054, + "index": 1 + }, + { + "type": "reRank", + "value": 0.8350804533361768, + "index": 20 + }, + { + "type": "rrf", + "value": 0.028474711270410194, + "index": 8 + } + ] + }, + { + "id": "65a7be389d96e21823f69d4f", + "q": "FastGPT Flow 的联网搜索示例中使用了哪些参数？", + "a": "FastGPT Flow 的联网搜索示例中使用了搜索关键词、Google 搜索的 API 密钥和自定义搜索引擎 ID。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8025297522544861, + "index": 21 + }, + { + "type": "reRank", + "value": 0.9730876959261983, + "index": 2 + }, + { + "type": "rrf", + "value": 0.028068137824235385, + "index": 10 + } + ] + }, + { + "id": "65a7e1e8fc13bdf20fd46d55", + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7e1d4fc13bdf20fd46abe", + "sourceName": "dataset - 2024-01-04T151625.388.csv", + "sourceId": "65a7e1d2fc13bdf20fd46abc", + "q": "# 快速了解 FastGPT\n## FastGPT 特点\n1. **项目开源**\n \n FastGPT 遵循附加条件 Apache License 2.0 开源协议，你可以 [Fork](https://github.com/labring/FastGPT/fork) 之后进行二次开发和发布。FastGPT 社区版将保留核心功能，商业版仅在社区版基础上使用 API 的形式进行扩展，不影响学习使用。\n \n2. **独特的 QA 结构**\n \n 针对客服问答场景设计的 QA 结构，提高在大量数据场景中的问答准确性。\n \n3. **可视化工作流**\n \n 通过 Flow 模块展示了从问题输入到模型输出的完整流程，便于调试和设计复杂流程。\n \n4. **无限扩展**\n \n 基于 API 进行扩展，无需修改 FastGPT 源码，也可快速接入现有的程序中。\n \n5. **便于调试**\n \n 提供搜索测试、引用修改、完整对话预览等多种调试途径。\n \n6. **支持多种模型**\n \n 支持 GPT、Claude、文心一言等多种 LLM 模型，未来也将支持自定义的向量模型。", + "a": "", + "chunkIndex": 67, + "score": [ + { + "type": "fullText", + "value": 1.0340073529411764, + "index": 12 + }, + { + "type": "reRank", + "value": 0.9542227274192233, + "index": 9 + }, + { + "type": "rrf", + "value": 0.027272727272727275, + "index": 11 + } + ] + }, + { + "id": "65a7be319d96e21823f69c8f", + "q": "FastGPT Flow 的工作流设计中，模块之间如何进行组合和组装？", + "a": "FastGPT Flow 允许用户在核心工作流模块中进行自由组合和组装，从而衍生出一个新的模块。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8098832368850708, + "index": 13 + }, + { + "type": "reRank", + "value": 0.9478657435317039, + "index": 12 + }, + { + "type": "rrf", + "value": 0.027212143650499815, + "index": 12 + } + ] + }, + { + "id": "65a7be359d96e21823f69ce0", + "q": "FastGPT Flow 的模块的输入和输出如何连接？", + "a": "FastGPT Flow 的模块的输入和输出通过连接点进行连接，连接点的颜色代表了不同的数据类型。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.8060981035232544, + "index": 16 + }, + { + "type": "reRank", + "value": 0.9530133603823691, + "index": 10 + }, + { + "type": "rrf", + "value": 0.027071520029266508, + "index": 13 + } + ] + }, + { + "id": "65a7be319d96e21823f69c98", + "q": "FastGPT Flow 的工作流设计方案能够满足哪些问答场景？", + "a": "FastGPT Flow 的工作流设计方案能够满足基本的 AI 知识库问答需求，并适应各种复杂的问答场景，例如联网搜索、数据库操作、数据实时更新、消息通知等。", + "chunkIndex": 0, + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7be059d96e21823f69af5", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7be059d96e21823f69ae8", + "score": [ + { + "type": "embedding", + "value": 0.814436137676239, + "index": 8 + }, + { + "type": "reRank", + "value": 0.8814109034236719, + "index": 19 + }, + { + "type": "rrf", + "value": 0.026992753623188405, + "index": 16 + } + ] + }, + { + "id": "65a7e058fc13bdf20fd46577", + "datasetId": "6593e137231a2be9c5603ba7", + "collectionId": "65a7e01efc13bdf20fd45815", + "sourceName": "FastGPT软著.pdf", + "sourceId": "65a7e01dfc13bdf20fd457f3", + "q": "FastGPT Flow 工作流设计112312 3123213123 232321312 21312 23一、介绍FastGPT 作为一款基于大型语言模型（LLM）的知识库问答系统，旨在为用户提供一个即插即用的解决方案。它集成了数据处理、模型调用等多项功能，通过引入 Flow 可视化工作流编排技术，进一步增强了对复杂问答场景的支持能力。本文将重点介绍 FastGPT Flow工作流的设计方案和应用优势。\nFastGPT Flow 工 作 流 采 用 了 React Flow 框 架 作 为 UI 底 座 ， 结 合 自 研 的 FlowController 实现工作流的运行。FastGPT 使用 Flow 模块为用户呈现了一个直观、可视化的界面，从而简化了 AI 应用工作流程的设计和管理方式。React Flow 的应用使得用户能够以图形化的方式组织和编排工作流，这不仅使得工作流的创建过程更为直观，同时也为用户提供了强大且灵活的工作流编辑器。在 FastGPT Flow 工作流设计中，核心工作流模块包括用户引导、问题输入、知识库检索、AI 文本生成、问题分类、结构化内容提取、指定回复、应用调用和 HTTP 扩展，并允许用户在这类模块中进行自由组合和组装，从而衍生出一个新的模块。", + "a": "", + "chunkIndex": 0, + "score": [ + { + "type": "fullText", + "value": 1.0229779411764706, + "index": 15 + }, + { + "type": "reRank", + "value": 0.9577545043363116, + "index": 8 + }, + { + "type": "rrf", + "value": 0.026992753623188405, + "index": 17 + } + ] + } + ], + "duration": "2.978s", + "searchMode": "mixedRecall", + "limit": 1500, + "similarity": 0.1, + "usingReRank": true, + "usingSimilarityFilter": true + } +} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +HTTP Status Code **200** + +|Name|Type|Required|Restrictions|Title|description| +|---|---|---|---|---|---| +|» code|integer|true|none||none| +|» statusText|string|true|none||none| +|» message|string|true|none||none| +|» data|object|true|none||none| +|»» list|[object]|true|none||none| +|»»» id|string|true|none||none| +|»»» q|string|true|none||none| +|»»» a|string|true|none||none| +|»»» chunkIndex|integer|true|none||none| +|»»» datasetId|string|true|none||none| +|»»» collectionId|string|true|none||none| +|»»» sourceName|string|true|none||none| +|»»» sourceId|string|true|none||none| +|»»» score|[object]|true|none||none| +|»»»» type|string|true|none||none| +|»»»» value|number|true|none||none| +|»»»» index|integer|true|none||none| +|»» duration|string|true|none||none| +|»» searchMode|string|true|none||none| +|»» limit|integer|true|none||none| +|»» similarity|number|true|none||none| +|»» usingReRank|boolean|true|none||none| +|»» usingSimilarityFilter|boolean|true|none||none| + +# openapi/知识库/知识库crud + +## GET 获取知识库列表 + +GET /core/dataset/list + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|parentId|query|string| no |父级的ID| +|Authorization|header|string| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## GET 获取知识库详情 + +GET /core/dataset/detail + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|id|query|string| no |知识库id| +|Authorization|header|string| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +# openapi/知识库/集合crud + +## POST 获取知识库集合列表 + +POST /core/dataset/collection/list + +> Body Parameters + +```json +{ + "pageNum": 1, + "pageSize": 10, + "datasetId": "6597ca43e26f2a90a1501414", + "parentId": null, + "searchText": "", + "simple": true +} +``` + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|Authorization|header|string| no |none| +|body|body|object| no |none| +|» pageNum|body|integer| no |none| +|» pageSize|body|integer| no |none| +|» datasetId|body|string| yes |none| +|» parentId|body|null| no |none| +|» searchText|body|string| no |none| +|» simple|body|boolean| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## GET 获取集合详情 + +GET /core/dataset/collection/detail + +### Params + +|Name|Location|Type|Required|Description| +|---|---|---|---|---| +|id|query|string| no |知识库id| +|Authorization|header|string| no |none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## PUT 更新集合 + +PUT /core/dataset/collection/update + +> Body Parameters + +```json +{ + "id": "6597ce094e10ee661f0891c8", + "parentId": null, + "name": "222" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» id|body|string| yes ||none| +|» parentId|body|null| no | 父级的id|none| +|» name|body|string| no | 名称|none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 创建空集合（文件夹或者一个空集合） + +POST /core/dataset/collection/create + +> Body Parameters + +```json +{ + "datasetId": "6597ca43e26f2a90a1501414", + "parentId": null, + "name": "集合名", + "type": "folder", + "metadata": {} +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» datasetId|body|string| yes ||none| +|» parentId|body|null| no ||none| +|» name|body|string| yes ||none| +|» type|body|[collection type](#schemacollection%20type)| yes ||none| +|» metadata|body|object| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 创建文本集合 + +POST /core/dataset/collection/create/text + +> Body Parameters + +```json +{ + "text": "xxxxxxxxxxxxxx", + "datasetId": "6593e137231a2be9c5603ba7", + "parentId": null, + "name": "测试", + "trainingType": "qa", + "chunkSize": 8000, + "chunkSplitter": "", + "qaPrompt": "", + "metadata": {} +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» datasetId|body|string| no ||none| +|» parentId|body|null| no ||none| +|» name|body|string| yes ||none| +|» text|body|string| yes | 原文本|none| +|» trainingType|body|[training type](#schematraining%20type)| yes ||none| +|» chunkSize|body|integer| no | 分块大小|none| +|» chunkSplitter|body|string| no | 自定义最高优先级的分段符号|none| +|» qaPrompt|body|string| no ||none| +|» metadata|body|object| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 创建网络链接集合 + +POST /core/dataset/collection/create/link + +> Body Parameters + +```json +{ + "link": "https://doc.fastgpt.in/docs/course/quick-start/", + "datasetId": "6593e137231a2be9c5603ba7", + "parentId": null, + "trainingType": "chunk", + "chunkSize": 512, + "chunkSplitter": "", + "qaPrompt": "", + "metadata": { + "webPageSelector": ".docs-content" + } +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» datasetId|body|string| yes ||none| +|» parentId|body|null| no ||none| +|» link|body|string| yes ||none| +|» trainingType|body|[training type](#schematraining%20type)| yes ||none| +|» chunkSize|body|integer| no ||none| +|» chunkSplitter|body|string| no ||none| +|» qaPrompt|body|string| no ||none| +|» metadata|body|object| no ||none| +|»» webPageSelector|body|string| no | web选择器|none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## DELETE 删除一个集合 + +DELETE /core/dataset/collection/delete + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|id|query|string| no ||知识库id| +|Authorization|header|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +# openapi/知识库/数据crud + +## POST 获取数据列表 + +POST /core/dataset/data/list + +> Body Parameters + +```json +{ + "pageNum": 1, + "pageSize": 10, + "collectionId": "65a8d2700d70d3de0bf09186", + "searchText": "" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» pageNum|body|integer| yes ||none| +|» pageSize|body|integer| yes ||none| +|» searchText|body|string| yes ||none| +|» collectionId|body|string| yes ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## GET 获取数据详情 + +GET /core/dataset/data/detail + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|id|query|string| yes ||none| +|Authorization|header|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## DELETE 删除一条数据 + +DELETE /core/dataset/data/delete + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|id|query|string| no ||none| +|Authorization|header|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## PUT 更新数据 + +PUT /core/dataset/data/update + +> Body Parameters + +```json +{ + "id": "6597ce094e10ee661f0891c8", + "parentId": null, + "name": "222" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» id|body|string| yes ||none| +|» q|body|string| yes ||none| +|» a|body|string| no ||none| +|» indexes|body|[[数据自定义向量](#schema%e6%95%b0%e6%8d%ae%e8%87%aa%e5%ae%9a%e4%b9%89%e5%90%91%e9%87%8f)]| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +## POST 知识库插入记录（批量插入） + +POST /core/dataset/data/pushData + +> Body Parameters + +```json +{ + "collectionId": "string", + "data": [ + { + "a": "string", + "q": "string", + "chunkIndex": 1 + } + ], + "trainingMode": "string", + "promot": "string", + "billId": "" +} +``` + +### Params + +|Name|Location|Type|Required|Title|Description| +|---|---|---|---|---|---| +|Authorization|header|string| no ||none| +|body|body|object| no ||none| +|» collectionId|body|string| yes ||none| +|» data|body|[object]| yes ||none| +|»» a|body|string| no ||none| +|»» q|body|string| no ||none| +|»» chunkIndex|body|integer| no ||none| +|» trainingMode|body|[training type](#schematraining%20type)| no ||none| +|» promot|body|string| no ||none| +|» billId|body|string| no ||none| + +> Response Examples + +> 200 Response + +```json +{} +``` + +### Responses + +|HTTP Status Code |Meaning|Description|Data schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|成功|Inline| + +### Responses Data Schema + +# Data Schema + +

similary

+ + + + + + +```json +1 + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|integer|false|none||none| + + + + + + + + +```json +"embedding" + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|string|false|none||none| + +#### Enum + +|Name|Value| +|---|---| +|*anonymous*|embedding| +|*anonymous*|fullTextRecall| +|*anonymous*|mixedRecall| + +

training type

+ + + + + + +```json +"chunk" + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|string|false|none||none| + +#### Enum + +|Name|Value| +|---|---| +|*anonymous*|chunk| +|*anonymous*|qa| + +

collection type

+ + + + + + +```json +"folder" + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|*anonymous*|string|false|none||none| + +#### Enum + +|Name|Value| +|---|---| +|*anonymous*|folder| +|*anonymous*|virtual| +|*anonymous*|link| +|*anonymous*|file| + +

数据自定义向量

+ + + + + + +```json +{ + "defaultIndex": true, + "type": "string", + "text": "string" +} + +``` + +### Attribute + +|Name|Type|Required|Restrictions|Title|Description| +|---|---|---|---|---|---| +|defaultIndex|boolean|false|none||是否为默认| +|type|string|true|none||none| +|text|string|true|none||索引文本| + diff --git a/docSite/content/docs/development/qa.md b/docSite/content/docs/development/qa.md index 00bd08b36..3784c3337 100644 --- a/docSite/content/docs/development/qa.md +++ b/docSite/content/docs/development/qa.md @@ -25,7 +25,7 @@ OneAPI 账号的余额不足，默认 root 用户只有 200 刀，可以手动 ### xxx渠道找不到 -OneAPI 中没有配置该模型渠道。 +OneAPI 中没有配置该模型渠道。或者是修改了配置文件中一部分的模型，但没有全部修改。 ### 页面中可以正常回复，API 报错 @@ -35,6 +35,15 @@ OneAPI 中没有配置该模型渠道。 OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并重启容器（先 stop 然后 rm 掉，最后再 up -d 运行一次）。可以`exec`进入容器，`env`查看环境变量是否生效。 +### 其他模型没法进行问题分类/内容提取 + +需要给其他模型配置`toolChoice=false`，就会默认走提示词模式。目前内置提示词仅针对了商业模型API进行测试，国内外的商业模型基本都可用。 + +### 页面崩溃 + +1. 关闭翻译 +2. 检查配置文件是否正常加载，如果没有正常加载会导致缺失系统信息，在某些操作下会导致空指针。 + ## Docker 部署常见问题 ### 如何更新？ diff --git a/docSite/content/docs/development/upgrading/467.md b/docSite/content/docs/development/upgrading/467.md new file mode 100644 index 000000000..c1ae38654 --- /dev/null +++ b/docSite/content/docs/development/upgrading/467.md @@ -0,0 +1,33 @@ +--- +title: 'V4.6.7（需要初始化）' +description: 'FastGPT V4.6.7' +icon: 'upgrade' +draft: false +toc: true +weight: 829 +--- + +## 1。执行初始化 API + +发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名) + +1. https://xxxxx/api/admin/initv464 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv467' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + +初始化说明： +1. 将 images 重新关联到数据集（不初始化也问题不大，就是可能会留下永久脏数据） + + +## V4.6.7 更新说明 + +1. 修改了知识库UI及新的导入交互方式。 +2. 优化知识库和对话的数据索引。 +3. 知识库 openAPI，支持通过 API 操作知识库。（文档待补充） +4. 新增 - 输入框变量提示。输入 { 号后将会获得可用变量提示。根据社区针对高级编排的反馈，我们计划于 2 月份的版本中，优化变量内容，支持模块的局部变量以及更多全局变量写入。 +5. 修复 - API 对话时，chatId 冲突问题。 +6. 修复 - Iframe 嵌入网页可能导致的 window.onLoad 冲突。 \ No newline at end of file diff --git a/docSite/content/docs/use-cases/datasetEngine.md b/docSite/content/docs/use-cases/datasetEngine.md index 6a1f16152..26f672381 100644 --- a/docSite/content/docs/use-cases/datasetEngine.md +++ b/docSite/content/docs/use-cases/datasetEngine.md @@ -25,7 +25,9 @@ FastGPT 采用了 RAG 中的 Embedding 方案构建知识库，要使用好 Fast FastGPT 采用了 `PostgresSQL` 的 `PG Vector` 插件作为向量检索器，索引为`HNSW`。且`PostgresSQL`仅用于向量检索，`MongoDB`用于其他数据的存取。 -在`PostgresSQL`的表中，设置一个 `index` 字段用于存储向量，以及一个`data_id`用于在`MongoDB`中寻找对应的映射值。多个`index`可以对应一组`data_id`，也就是说，一组向量可以对应多组数据。在进行检索时，相同数据会进行合并。 +在`MongoDB`的`dataset.datas`表中，会存储向量原数据的信息，同时有一个`indexes`字段，会记录其对应的向量ID，这是一个数组，也就是说，一组向量可以对应多组数据。 + +在`PostgresSQL`的表中，设置一个 `index` 字段用于存储向量。在检索时，会先召回向量，再根据向量的ID，去`MongoDB`中寻找原数据内容，如果对应了同一组原数据，则进行合并，向量得分取最高得分。 ![](/imgs/datasetSetting1.png) diff --git a/packages/global/common/file/icon.ts b/packages/global/common/file/icon.ts index aeee3cca1..9e1444f64 100644 --- a/packages/global/common/file/icon.ts +++ b/packages/global/common/file/icon.ts @@ -1,9 +1,9 @@ export const fileImgs = [ - { suffix: 'pdf', src: '/imgs/files/pdf.svg' }, - { suffix: 'csv', src: '/imgs/files/csv.svg' }, - { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' }, - { suffix: 'txt', src: '/imgs/files/txt.svg' }, - { suffix: 'md', src: '/imgs/files/markdown.svg' } + { suffix: 'pdf', src: 'file/fill/pdf' }, + { suffix: 'csv', src: 'file/fill/csv' }, + { suffix: '(doc|docs)', src: 'file/fill/doc' }, + { suffix: 'txt', src: 'file/fill/txt' }, + { suffix: 'md', src: 'file/fill/markdown' } // { suffix: '.', src: '/imgs/files/file.svg' } ]; diff --git a/packages/global/common/file/image/constants.ts b/packages/global/common/file/image/constants.ts index 7136f8da4..643d6c21b 100644 --- a/packages/global/common/file/image/constants.ts +++ b/packages/global/common/file/image/constants.ts @@ -9,7 +9,7 @@ export enum MongoImageTypeEnum { teamAvatar = 'teamAvatar', chatImage = 'chatImage', - docImage = 'docImage' + collectionImage = 'collectionImage' } export const mongoImageTypeMap = { [MongoImageTypeEnum.systemAvatar]: { @@ -41,8 +41,8 @@ export const mongoImageTypeMap = { label: 'common.file.type.chatImage', unique: false }, - [MongoImageTypeEnum.docImage]: { - label: 'common.file.type.docImage', + [MongoImageTypeEnum.collectionImage]: { + label: 'common.file.type.collectionImage', unique: false } }; diff --git a/packages/global/common/file/image/type.d.ts b/packages/global/common/file/image/type.d.ts index 79228d3d5..9bb4d28cb 100644 --- a/packages/global/common/file/image/type.d.ts +++ b/packages/global/common/file/image/type.d.ts @@ -1,11 +1,14 @@ import { MongoImageTypeEnum } from './constants'; export type MongoImageSchemaType = { + _id: string; teamId: string; binary: Buffer; createTime: Date; expiredTime?: Date; type: `${MongoImageTypeEnum}`; - metadata?: { fileId?: string }; + metadata?: { + relatedId?: string; // This id is associated with a set of images + }; }; diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts index 45f13aaec..5fb21ae57 100644 --- a/packages/global/common/string/textSplitter.ts +++ b/packages/global/common/string/textSplitter.ts @@ -13,13 +13,12 @@ export const splitText2Chunks = (props: { chunkLen: number; overlapRatio?: number; customReg?: string[]; - countTokens?: boolean; }): { chunks: string[]; - tokens: number; + chars: number; overlapRatio?: number; } => { - let { text = '', chunkLen, overlapRatio = 0.2, customReg = [], countTokens = true } = props; + let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props; const splitMarker = 'SPLIT_HERE_SPLIT_HERE'; const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER'; const overlapLen = Math.round(chunkLen * overlapRatio); @@ -240,13 +239,11 @@ export const splitText2Chunks = (props: { mdTitle: '' }).map((chunk) => chunk?.replaceAll(codeBlockMarker, '\n') || ''); // restore code block - const tokens = countTokens - ? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0) - : 0; + const chars = chunks.reduce((sum, chunk) => sum + chunk.length, 0); return { chunks, - tokens + chars }; } catch (err) { throw new Error(getErrText(err)); diff --git a/packages/global/common/system/types/index.d.ts b/packages/global/common/system/types/index.d.ts index 2dd489f0a..57e759962 100644 --- a/packages/global/common/system/types/index.d.ts +++ b/packages/global/common/system/types/index.d.ts @@ -55,6 +55,8 @@ export type FastGPTFeConfigsType = { datasetStoreFreeSize?: number; datasetStorePrice?: number; }; + + uploadFileMaxSize?: number; }; export type SystemEnvType = { diff --git a/packages/global/core/app/type.d.ts b/packages/global/core/app/type.d.ts index 8e29426c0..7b00561e4 100644 --- a/packages/global/core/app/type.d.ts +++ b/packages/global/core/app/type.d.ts @@ -4,7 +4,7 @@ import { PermissionTypeEnum } from '../../support/permission/constant'; import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.d'; import { VariableInputEnum } from '../module/constants'; import { SelectedDatasetType } from '../module/api'; -import { DatasetSearchModeEnum } from '../dataset/constant'; +import { DatasetSearchModeEnum } from '../dataset/constants'; export interface AppSchema { _id: string; diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts index 48f22335a..3b4bbcfa2 100644 --- a/packages/global/core/app/utils.ts +++ b/packages/global/core/app/utils.ts @@ -4,7 +4,7 @@ import { ModuleOutputKeyEnum, ModuleInputKeyEnum } from '../module/constants'; import type { FlowNodeInputItemType } from '../module/node/type.d'; import { getGuideModule, splitGuideModule } from '../module/utils'; import { ModuleItemType } from '../module/type.d'; -import { DatasetSearchModeEnum } from '../dataset/constant'; +import { DatasetSearchModeEnum } from '../dataset/constants'; export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEditFormType => { return { diff --git a/packages/global/core/chat/type.d.ts b/packages/global/core/chat/type.d.ts index 8c8cb9c88..05b421c10 100644 --- a/packages/global/core/chat/type.d.ts +++ b/packages/global/core/chat/type.d.ts @@ -4,7 +4,7 @@ import { ChatRoleEnum, ChatSourceEnum, ChatStatusEnum } from './constants'; import { FlowNodeTypeEnum } from '../module/node/constant'; import { ModuleOutputKeyEnum } from '../module/constants'; import { AppSchema } from '../app/type'; -import { DatasetSearchModeEnum } from '../dataset/constant'; +import { DatasetSearchModeEnum } from '../dataset/constants'; export type ChatSchema = { _id: string; @@ -92,6 +92,7 @@ export type moduleDispatchResType = { runningTime?: number; inputTokens?: number; outputTokens?: number; + charsLength?: number; model?: string; query?: string; contextTotalLen?: number; diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 0b3f18697..43a15f75a 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -1,5 +1,5 @@ import { DatasetDataIndexItemType, DatasetSchemaType } from './type'; -import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constants'; import type { LLMModelItemType } from '../ai/model.d'; /* ================= dataset ===================== */ @@ -17,27 +17,25 @@ export type DatasetUpdateBody = { /* ================= collection ===================== */ export type DatasetCollectionChunkMetadataType = { + parentId?: string; trainingType?: `${TrainingModeEnum}`; chunkSize?: number; chunkSplitter?: string; qaPrompt?: string; + metadata?: Record; }; export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { datasetId: string; - parentId?: string; name: string; type: `${DatasetCollectionTypeEnum}`; fileId?: string; rawLink?: string; rawTextLength?: number; hashRawText?: string; - metadata?: Record; }; export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { datasetId: string; - parentId?: string; - metadata?: Record; }; export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { name: string; @@ -45,16 +43,24 @@ export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams }; export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { link: string; - chunkSplitter?: string; +}; +export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { + name: string; + rawTextLength: number; + hashRawText: string; + trainingType: `${TrainingModeEnum}`; + chunkSize: number; + chunkSplitter: string; + qaPrompt: string; + + fileMetadata?: Record; + collectionMetadata?: Record; }; /* ================= data ===================== */ export type PgSearchRawType = { id: string; - team_id: string; - tmb_id: string; collection_id: string; - data_id: string; score: number; }; export type PushDatasetDataChunkProps = { diff --git a/packages/global/core/dataset/constant.ts b/packages/global/core/dataset/constants.ts similarity index 91% rename from packages/global/core/dataset/constant.ts rename to packages/global/core/dataset/constants.ts index 8c6aaf38d..005fcf155 100644 --- a/packages/global/core/dataset/constant.ts +++ b/packages/global/core/dataset/constants.ts @@ -6,7 +6,7 @@ export enum DatasetTypeEnum { } export const DatasetTypeMap = { [DatasetTypeEnum.folder]: { - icon: 'core/dataset/folderDataset', + icon: 'common/folderFill', label: 'core.dataset.Folder Dataset', collectionLabel: 'common.Folder' }, @@ -104,10 +104,12 @@ export enum TrainingModeEnum { export const TrainingTypeMap = { [TrainingModeEnum.chunk]: { - label: 'core.dataset.training.type chunk' + label: 'core.dataset.training.Chunk mode', + tooltip: 'core.dataset.import.Chunk Split Tip' }, [TrainingModeEnum.qa]: { - label: 'core.dataset.training.type qa' + label: 'core.dataset.training.QA mode', + tooltip: 'core.dataset.import.QA Import Tip' } }; @@ -168,4 +170,8 @@ export const SearchScoreTypeMap = { } }; -export const FolderAvatarSrc = '/imgs/files/folder.svg'; +export const FolderIcon = 'file/fill/folder'; +export const FolderImgUrl = '/imgs/files/folder.svg'; + +export const CustomCollectionIcon = 'common/linkBlue'; +export const LinkCollectionIcon = 'common/linkBlue'; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 8e99af6dd..bac51bd41 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -8,7 +8,7 @@ import { DatasetTypeEnum, SearchScoreTypeEnum, TrainingModeEnum -} from './constant'; +} from './constants'; /* schema */ export type DatasetSchemaType = { @@ -55,6 +55,8 @@ export type DatasetCollectionSchemaType = { hashRawText?: string; metadata?: { webPageSelector?: string; + relatedImgId?: string; // The id of the associated image collections + [key: string]: any; }; }; diff --git a/packages/global/core/dataset/utils.ts b/packages/global/core/dataset/utils.ts index 577d009b4..e8626e1c8 100644 --- a/packages/global/core/dataset/utils.ts +++ b/packages/global/core/dataset/utils.ts @@ -1,4 +1,4 @@ -import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constants'; import { getFileIcon } from '../../common/file/icon'; import { strIsLink } from '../../common/string/tools'; @@ -7,18 +7,13 @@ export function getCollectionIcon( name = '' ) { if (type === DatasetCollectionTypeEnum.folder) { - return '/imgs/files/folder.svg'; + return 'common/folderFill'; } if (type === DatasetCollectionTypeEnum.link) { - return '/imgs/files/link.svg'; + return 'common/linkBlue'; } if (type === DatasetCollectionTypeEnum.virtual) { - if (name === '手动录入') { - return '/imgs/files/manual.svg'; - } else if (name === '手动标注') { - return '/imgs/files/mark.svg'; - } - return '/imgs/files/collection.svg'; + return 'file/fill/manual'; } return getFileIcon(name); } @@ -30,19 +25,14 @@ export function getSourceNameIcon({ sourceId?: string; }) { if (strIsLink(sourceId)) { - return '/imgs/files/link.svg'; + return 'common/linkBlue'; } const fileIcon = getFileIcon(sourceName, ''); if (fileIcon) { return fileIcon; } - if (sourceName === '手动录入') { - return '/imgs/files/manual.svg'; - } else if (sourceName === '手动标注') { - return '/imgs/files/mark.svg'; - } - return '/imgs/files/collection.svg'; + return 'file/fill/manual'; } export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: string }) { diff --git a/packages/global/core/module/constants.ts b/packages/global/core/module/constants.ts index a9bf7db72..003f8006a 100644 --- a/packages/global/core/module/constants.ts +++ b/packages/global/core/module/constants.ts @@ -113,5 +113,16 @@ export enum VariableInputEnum { textarea = 'textarea', select = 'select' } +export const variableMap = { + [VariableInputEnum.input]: { + icon: 'core/app/variable/input' + }, + [VariableInputEnum.textarea]: { + icon: 'core/app/variable/textarea' + }, + [VariableInputEnum.select]: { + icon: 'core/app/variable/select' + } +}; export const DYNAMIC_INPUT_KEY = 'DYNAMIC_INPUT_KEY'; diff --git a/packages/global/core/module/node/constant.ts b/packages/global/core/module/node/constant.ts index 17db8dd87..312719193 100644 --- a/packages/global/core/module/node/constant.ts +++ b/packages/global/core/module/node/constant.ts @@ -54,10 +54,9 @@ export enum FlowNodeTypeEnum { pluginModule = 'pluginModule', pluginInput = 'pluginInput', pluginOutput = 'pluginOutput', - cfr = 'cfr', + cfr = 'cfr' // abandon - variable = 'variable' } export const EDGE_TYPE = 'default'; diff --git a/packages/global/core/module/template/system/aiChat.ts b/packages/global/core/module/template/system/aiChat.ts index 363c1dd74..3c548df1e 100644 --- a/packages/global/core/module/template/system/aiChat.ts +++ b/packages/global/core/module/template/system/aiChat.ts @@ -23,15 +23,15 @@ export const AiChatModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.textAnswer, flowType: FlowNodeTypeEnum.chatNode, avatar: '/imgs/module/AI.png', - name: 'AI 对话', - intro: 'AI 大模型对话', + name: 'core.module.template.Ai chat', + intro: 'core.module.template.Ai chat intro', showStatus: true, inputs: [ Input_Template_Switch, { key: ModuleInputKeyEnum.aiModel, type: FlowNodeInputTypeEnum.selectChatModel, - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: ModuleIOValueTypeEnum.string, showTargetInApp: false, @@ -41,42 +41,31 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatTemperature, type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window - label: '温度', + label: '', value: 0, valueType: ModuleIOValueTypeEnum.number, min: 0, max: 10, step: 1, - markList: [ - { label: '严谨', value: 0 }, - { label: '发散', value: 10 } - ], showTargetInApp: false, showTargetInPlugin: false }, { key: ModuleInputKeyEnum.aiChatMaxToken, type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window - label: '回复上限', + label: '', value: 2000, valueType: ModuleIOValueTypeEnum.number, min: 100, max: 4000, step: 50, - markList: [ - { label: '100', value: 100 }, - { - label: `${4000}`, - value: 4000 - } - ], showTargetInApp: false, showTargetInPlugin: false }, { key: ModuleInputKeyEnum.aiChatIsResponseText, type: FlowNodeInputTypeEnum.hidden, - label: '返回AI内容', + label: '', value: true, valueType: ModuleIOValueTypeEnum.boolean, showTargetInApp: false, @@ -85,7 +74,7 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatQuoteTemplate, type: FlowNodeInputTypeEnum.hidden, - label: '引用内容模板', + label: '', valueType: ModuleIOValueTypeEnum.string, showTargetInApp: false, showTargetInPlugin: false @@ -93,7 +82,7 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatQuotePrompt, type: FlowNodeInputTypeEnum.hidden, - label: '引用内容提示词', + label: '', valueType: ModuleIOValueTypeEnum.string, showTargetInApp: false, showTargetInPlugin: false @@ -110,7 +99,7 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiSystemPrompt, type: FlowNodeInputTypeEnum.textarea, - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: ModuleIOValueTypeEnum.string, description: chatNodeSystemPromptTip, @@ -122,8 +111,8 @@ export const AiChatModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.aiChatDatasetQuote, type: FlowNodeInputTypeEnum.target, - label: '引用内容', - description: "对象数组格式，结构：\n [{q:'问题',a:'回答'}]", + label: 'core.module.input.label.Quote', + description: 'core.module.input.description.Quote', valueType: ModuleIOValueTypeEnum.datasetQuote, showTargetInApp: true, showTargetInPlugin: true @@ -134,16 +123,16 @@ export const AiChatModule: FlowModuleTemplateType = { Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.history, - label: '新的上下文', - description: '将本次回复内容拼接上历史记录，作为新的上下文返回', + label: 'core.module.output.label.New context', + description: 'core.module.output.description.New context', valueType: ModuleIOValueTypeEnum.chatHistory, type: FlowNodeOutputTypeEnum.source, targets: [] }, { key: ModuleOutputKeyEnum.answerText, - label: 'AI回复内容', - description: '将在 stream 回复完毕后触发', + label: 'core.module.output.label.Ai response content', + description: 'core.module.output.description.Ai response content', valueType: ModuleIOValueTypeEnum.string, type: FlowNodeOutputTypeEnum.source, targets: [] diff --git a/packages/global/core/module/template/system/assignedAnswer.ts b/packages/global/core/module/template/system/assignedAnswer.ts index cc48fa066..fb2a3d431 100644 --- a/packages/global/core/module/template/system/assignedAnswer.ts +++ b/packages/global/core/module/template/system/assignedAnswer.ts @@ -9,19 +9,17 @@ export const AssignedAnswerModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.textAnswer, flowType: FlowNodeTypeEnum.answerNode, avatar: '/imgs/module/reply.png', - name: '指定回复', - intro: '该模块可以直接回复一段指定的内容。常用于引导、提示', + name: 'core.module.template.Assigned reply', + intro: 'core.module.template.Assigned reply intro', inputs: [ Input_Template_Switch, { key: ModuleInputKeyEnum.answerText, type: FlowNodeInputTypeEnum.textarea, valueType: ModuleIOValueTypeEnum.any, - label: '回复的内容', - description: - '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串', - placeholder: - '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串', + label: 'core.module.input.label.Response content', + description: 'core.module.input.description.Response content', + placeholder: 'core.module.input.description.Response content', showTargetInApp: true, showTargetInPlugin: true } diff --git a/packages/global/core/module/template/system/classifyQuestion.ts b/packages/global/core/module/template/system/classifyQuestion.ts index a31e3f07d..4e0d1a141 100644 --- a/packages/global/core/module/template/system/classifyQuestion.ts +++ b/packages/global/core/module/template/system/classifyQuestion.ts @@ -17,12 +17,8 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.functionCall, flowType: FlowNodeTypeEnum.classifyQuestion, avatar: '/imgs/module/cq.png', - name: '问题分类', - intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子： -类型1: 打招呼 -类型2: 关于商品“使用”问题 -类型3: 关于商品“购买”问题 -类型4: 其他问题`, + name: 'core.module.template.Classify question', + intro: `core.module.template.Classify question intro`, showStatus: true, inputs: [ Input_Template_Switch, @@ -30,7 +26,7 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.aiModel, type: FlowNodeInputTypeEnum.selectCQModel, valueType: ModuleIOValueTypeEnum.string, - label: '分类模型', + label: 'core.module.input.label.Classify model', required: true, showTargetInApp: false, showTargetInPlugin: false @@ -39,11 +35,9 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.aiSystemPrompt, type: FlowNodeInputTypeEnum.textarea, valueType: ModuleIOValueTypeEnum.string, - label: '背景知识', - description: - '你可以添加一些特定内容的介绍，从而更好的识别用户的问题类型。这个内容通常是给模型介绍一个它不知道的内容。', - placeholder: - '例如: \n1. AIGC（人工智能生成内容）是指使用人工智能技术自动或半自动地生成数字内容，如文本、图像、音乐、视频等。\n2. AIGC技术包括但不限于自然语言处理、计算机视觉、机器学习和深度学习。这些技术可以创建新内容或修改现有内容，以满足特定的创意、教育、娱乐或信息需求。', + label: 'core.module.input.label.Background', + description: 'core.module.input.description.Background', + placeholder: 'core.module.input.placeholder.Classify background', showTargetInApp: true, showTargetInPlugin: true }, diff --git a/packages/global/core/module/template/system/contextExtract.ts b/packages/global/core/module/template/system/contextExtract.ts index 83b2ae87b..5616ffa40 100644 --- a/packages/global/core/module/template/system/contextExtract.ts +++ b/packages/global/core/module/template/system/contextExtract.ts @@ -17,8 +17,8 @@ export const ContextExtractModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.functionCall, flowType: FlowNodeTypeEnum.contentExtract, avatar: '/imgs/module/extract.png', - name: '文本内容提取', - intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等', + name: 'core.module.template.Extract field', + intro: 'core.module.template.Extract field intro', showStatus: true, inputs: [ Input_Template_Switch, @@ -26,7 +26,7 @@ export const ContextExtractModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.aiModel, type: FlowNodeInputTypeEnum.selectExtractModel, valueType: ModuleIOValueTypeEnum.string, - label: '提取模型', + label: 'core.module.input.label.LLM', required: true, showTargetInApp: false, showTargetInPlugin: false diff --git a/packages/global/core/module/template/system/datasetSearch.ts b/packages/global/core/module/template/system/datasetSearch.ts index e6ed8c8bf..c06c03727 100644 --- a/packages/global/core/module/template/system/datasetSearch.ts +++ b/packages/global/core/module/template/system/datasetSearch.ts @@ -12,22 +12,22 @@ import { } from '../../constants'; import { Input_Template_Switch, Input_Template_UserChatInput } from '../input'; import { Output_Template_Finish, Output_Template_UserChatInput } from '../output'; -import { DatasetSearchModeEnum } from '../../../dataset/constant'; +import { DatasetSearchModeEnum } from '../../../dataset/constants'; export const DatasetSearchModule: FlowModuleTemplateType = { id: FlowNodeTypeEnum.datasetSearchNode, templateType: ModuleTemplateTypeEnum.functionCall, flowType: FlowNodeTypeEnum.datasetSearchNode, avatar: '/imgs/module/db.png', - name: '知识库搜索', - intro: '去知识库中搜索对应的答案。可作为 AI 对话引用参考。', + name: 'core.module.template.Dataset search', + intro: 'core.module.template.Dataset search intro', showStatus: true, inputs: [ Input_Template_Switch, { key: ModuleInputKeyEnum.datasetSelectList, type: FlowNodeInputTypeEnum.selectDataset, - label: '关联的知识库', + label: 'core.module.input.label.Select dataset', value: [], valueType: ModuleIOValueTypeEnum.selectDataset, list: [], @@ -38,7 +38,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.datasetSimilarity, type: FlowNodeInputTypeEnum.hidden, - label: '最低相关性', + label: '', value: 0.4, valueType: ModuleIOValueTypeEnum.number, min: 0, @@ -54,8 +54,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.datasetLimit, type: FlowNodeInputTypeEnum.hidden, - label: '引用上限', - description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens', + label: '', value: 1500, valueType: ModuleIOValueTypeEnum.number, showTargetInApp: false, @@ -93,23 +92,22 @@ export const DatasetSearchModule: FlowModuleTemplateType = { Output_Template_UserChatInput, { key: ModuleOutputKeyEnum.datasetIsEmpty, - label: '搜索结果为空', + label: 'core.module.output.label.Search result empty', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.boolean, targets: [] }, { key: ModuleOutputKeyEnum.datasetUnEmpty, - label: '搜索结果不为空', + label: 'core.module.output.label.Search result not empty', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.boolean, targets: [] }, { key: ModuleOutputKeyEnum.datasetQuoteQA, - label: '引用内容', - description: - '始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器', + label: 'core.module.output.label.Quote', + description: 'core.module.output.label.Quote intro', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.datasetQuote, targets: [] diff --git a/packages/global/core/module/template/system/http.ts b/packages/global/core/module/template/system/http.ts index 22d6ada02..4397a4936 100644 --- a/packages/global/core/module/template/system/http.ts +++ b/packages/global/core/module/template/system/http.ts @@ -17,8 +17,8 @@ export const HttpModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.externalCall, flowType: FlowNodeTypeEnum.httpRequest, avatar: '/imgs/module/http.png', - name: 'HTTP模块', - intro: '可以发出一个 HTTP POST 请求，实现更为复杂的操作（联网搜索、数据库查询等）', + name: 'core.module.template.Http request', + intro: 'core.module.template.Http request intro', showStatus: true, inputs: [ Input_Template_Switch, diff --git a/packages/global/core/module/template/system/runApp.ts b/packages/global/core/module/template/system/runApp.ts index ce740dfdd..ef4cdcecc 100644 --- a/packages/global/core/module/template/system/runApp.ts +++ b/packages/global/core/module/template/system/runApp.ts @@ -22,8 +22,8 @@ export const RunAppModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.externalCall, flowType: FlowNodeTypeEnum.runApp, avatar: '/imgs/module/app.png', - name: '应用调用', - intro: '可以选择一个其他应用进行调用', + name: 'core.module.template.Running app', + intro: 'core.module.template.Running app intro', showStatus: true, inputs: [ Input_Template_Switch, diff --git a/packages/global/core/module/template/system/runPlugin.ts b/packages/global/core/module/template/system/runPlugin.ts index 51b563b52..710a0fa9a 100644 --- a/packages/global/core/module/template/system/runPlugin.ts +++ b/packages/global/core/module/template/system/runPlugin.ts @@ -8,7 +8,7 @@ export const RunPluginModule: FlowModuleTemplateType = { flowType: FlowNodeTypeEnum.pluginModule, avatar: '/imgs/module/custom.png', intro: '', - name: '自定义模块', + name: '', showStatus: false, inputs: [], // [{key:'pluginId'},...] outputs: [] diff --git a/packages/global/core/module/template/system/userGuide.ts b/packages/global/core/module/template/system/userGuide.ts index 1401cbbf4..95c63ff68 100644 --- a/packages/global/core/module/template/system/userGuide.ts +++ b/packages/global/core/module/template/system/userGuide.ts @@ -8,14 +8,14 @@ export const UserGuideModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.userGuide, flowType: FlowNodeTypeEnum.userGuide, avatar: '/imgs/module/userGuide.png', - name: '用户引导', + name: 'core.module.template.User guide', intro: userGuideTip, inputs: [ { key: ModuleInputKeyEnum.welcomeText, type: FlowNodeInputTypeEnum.hidden, valueType: ModuleIOValueTypeEnum.string, - label: '开场白', + label: 'core.app.Welcome Text', showTargetInApp: false, showTargetInPlugin: false }, @@ -23,7 +23,7 @@ export const UserGuideModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.variables, type: FlowNodeInputTypeEnum.hidden, valueType: ModuleIOValueTypeEnum.any, - label: '对话框变量', + label: 'core.module.Variable', value: [], showTargetInApp: false, showTargetInPlugin: false @@ -32,7 +32,7 @@ export const UserGuideModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.questionGuide, valueType: ModuleIOValueTypeEnum.boolean, type: FlowNodeInputTypeEnum.switch, - label: '问题引导', + label: '', showTargetInApp: false, showTargetInPlugin: false }, @@ -40,7 +40,7 @@ export const UserGuideModule: FlowModuleTemplateType = { key: ModuleInputKeyEnum.tts, type: FlowNodeInputTypeEnum.hidden, valueType: ModuleIOValueTypeEnum.any, - label: '语音播报', + label: '', showTargetInApp: false, showTargetInPlugin: false } diff --git a/packages/global/core/module/template/system/userInput.ts b/packages/global/core/module/template/system/userInput.ts index 3005bf33c..289943e03 100644 --- a/packages/global/core/module/template/system/userInput.ts +++ b/packages/global/core/module/template/system/userInput.ts @@ -16,14 +16,14 @@ export const UserInputModule: FlowModuleTemplateType = { templateType: ModuleTemplateTypeEnum.systemInput, flowType: FlowNodeTypeEnum.questionInput, avatar: '/imgs/module/userChatInput.png', - name: '用户问题(入口)', - intro: '用户输入的内容。该模块通常作为应用的入口，用户在发送消息后会首先执行该模块。', + name: 'core.module.template.Chat entrance', + intro: 'core.module.template.Chat entrance intro', inputs: [ { key: ModuleInputKeyEnum.userChatInput, type: FlowNodeInputTypeEnum.systemInput, valueType: ModuleIOValueTypeEnum.string, - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false } @@ -31,7 +31,7 @@ export const UserInputModule: FlowModuleTemplateType = { outputs: [ { key: ModuleOutputKeyEnum.userChatInput, - label: '用户问题', + label: 'core.module.input.label.user question', type: FlowNodeOutputTypeEnum.source, valueType: ModuleIOValueTypeEnum.string, targets: [] diff --git a/packages/global/core/module/template/tip.ts b/packages/global/core/module/template/tip.ts index e64ea65e2..2aa9dd405 100644 --- a/packages/global/core/module/template/tip.ts +++ b/packages/global/core/module/template/tip.ts @@ -1,7 +1,4 @@ -export const chatNodeSystemPromptTip = - '模型固定的引导词，通过调整该内容，可以引导模型聊天方向。该内容会被固定在上下文的开头。可使用变量，例如 {{language}}'; -export const userGuideTip = '可以在对话前设置引导语，设置全局变量，设置下一步指引'; -export const welcomeTextTip = - '每次对话开始前，发送一个初始内容。支持标准 Markdown 语法，可使用的额外标记:\n[快捷按键]: 用户点击后可以直接发送该问题'; -export const variableTip = - '可以在对话开始前，要求用户填写一些内容作为本轮对话的特定变量。该模块位于开场引导之后。\n变量可以通过 {{变量key}} 的形式注入到其他模块 string 类型的输入中，例如：提示词、限定词等'; +export const chatNodeSystemPromptTip = 'core.app.tip.chatNodeSystemPromptTip'; +export const userGuideTip = 'core.app.tip.userGuideTip'; +export const welcomeTextTip = 'core.app.tip.welcomeTextTip'; +export const variableTip = 'core.app.tip.variableTip'; diff --git a/packages/global/core/module/utils.ts b/packages/global/core/module/utils.ts index 476e0e2d2..a066df945 100644 --- a/packages/global/core/module/utils.ts +++ b/packages/global/core/module/utils.ts @@ -1,5 +1,5 @@ import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from './node/constant'; -import { ModuleIOValueTypeEnum, ModuleInputKeyEnum } from './constants'; +import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, variableMap } from './constants'; import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type'; import { AppTTSConfigType, ModuleItemType, VariableItemType } from './type'; import { Input_Template_Switch } from './template/input'; @@ -94,3 +94,12 @@ export function plugin2ModuleIO( : [] }; } + +export const formatVariablesIcon = ( + variables: VariableItemType[] +): (VariableItemType & { icon: string })[] => { + return variables.map((item) => ({ + ...item, + icon: variableMap[item.type]?.icon + })); +}; diff --git a/packages/global/support/user/api.d.ts b/packages/global/support/user/api.d.ts index dd256039d..5f7d8e212 100644 --- a/packages/global/support/user/api.d.ts +++ b/packages/global/support/user/api.d.ts @@ -3,7 +3,6 @@ import { OAuthEnum } from './constant'; export type PostLoginProps = { username: string; password: string; - tmbId?: string; }; export type OauthLoginProps = { diff --git a/packages/global/support/user/type.d.ts b/packages/global/support/user/type.d.ts index b8f7a64a7..3cb978810 100644 --- a/packages/global/support/user/type.d.ts +++ b/packages/global/support/user/type.d.ts @@ -13,6 +13,7 @@ export type UserModelSchema = { createTime: number; timezone: string; status: `${UserStatusEnum}`; + lastLoginTmbId?: string; openaiAccount?: { key: string; baseUrl: string; diff --git a/packages/global/support/wallet/bill/api.d.ts b/packages/global/support/wallet/bill/api.d.ts index 6314ab164..dcc421ecf 100644 --- a/packages/global/support/wallet/bill/api.d.ts +++ b/packages/global/support/wallet/bill/api.d.ts @@ -3,8 +3,7 @@ import { BillListItemCountType, BillListItemType } from './type'; export type CreateTrainingBillProps = { name: string; - vectorModel?: string; - agentModel?: string; + datasetId: string; }; export type ConcatBillProps = BillListItemCountType & { diff --git a/packages/global/support/wallet/bill/type.d.ts b/packages/global/support/wallet/bill/type.d.ts index 5b0606c8a..b80d7afef 100644 --- a/packages/global/support/wallet/bill/type.d.ts +++ b/packages/global/support/wallet/bill/type.d.ts @@ -4,9 +4,8 @@ import { BillSourceEnum } from './constants'; export type BillListItemCountType = { inputTokens?: number; outputTokens?: number; - textLen?: number; + charsLength?: number; duration?: number; - dataLen?: number; // abandon tokenLen?: number; diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index 84a05d631..729c9fb02 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -3,9 +3,10 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; import fsp from 'fs/promises'; import fs from 'fs'; import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type'; -import { delImgByFileIdList } from '../image/controller'; +import { MongoFileSchema } from './schema'; export function getGFSCollection(bucket: `${BucketNameEnum}`) { + MongoFileSchema; return connectionMongo.connection.db.collection(`${bucket}.files`); } export function getGridBucket(bucket: `${BucketNameEnum}`) { @@ -21,6 +22,7 @@ export async function uploadFile({ tmbId, path, filename, + contentType, metadata = {} }: { bucketName: `${BucketNameEnum}`; @@ -28,6 +30,7 @@ export async function uploadFile({ tmbId: string; path: string; filename: string; + contentType?: string; metadata?: Record; }) { if (!path) return Promise.reject(`filePath is empty`); @@ -44,7 +47,7 @@ export async function uploadFile({ const stream = bucket.openUploadStream(filename, { metadata, - contentType: metadata?.contentType + contentType }); // save to gridfs @@ -96,40 +99,6 @@ export async function delFileByFileIdList({ } } } -// delete file by metadata(datasetId) -export async function delFileByMetadata({ - bucketName, - datasetId -}: { - bucketName: `${BucketNameEnum}`; - datasetId?: string; -}) { - const bucket = getGridBucket(bucketName); - - const files = await bucket - .find( - { - ...(datasetId && { 'metadata.datasetId': datasetId }) - }, - { - projection: { - _id: 1 - } - } - ) - .toArray(); - - const idList = files.map((item) => String(item._id)); - - // delete img - await delImgByFileIdList(idList); - - // delete file - await delFileByFileIdList({ - bucketName, - fileIdList: idList - }); -} export async function getDownloadStream({ bucketName, diff --git a/packages/service/common/file/gridfs/schema.ts b/packages/service/common/file/gridfs/schema.ts new file mode 100644 index 000000000..457447d7f --- /dev/null +++ b/packages/service/common/file/gridfs/schema.ts @@ -0,0 +1,15 @@ +import { connectionMongo, type Model } from '../../mongo'; +const { Schema, model, models } = connectionMongo; + +const FileSchema = new Schema({}); + +try { + FileSchema.index({ 'metadata.teamId': 1 }); + FileSchema.index({ 'metadata.uploadDate': -1 }); +} catch (error) { + console.log(error); +} + +export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema); + +MongoFileSchema.syncIndexes(); diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index c8da371c8..ecb079666 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -46,8 +46,8 @@ export async function readMongoImg({ id }: { id: string }) { return data?.binary; } -export async function delImgByFileIdList(fileIds: string[]) { +export async function delImgByRelatedId(relateIds: string[]) { return MongoImage.deleteMany({ - 'metadata.fileId': { $in: fileIds.map((item) => String(item)) } + 'metadata.relatedId': { $in: relateIds.map((id) => String(id)) } }); } diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index 0b4ff832a..baa799ac9 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -35,6 +35,8 @@ try { ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 }); ImageSchema.index({ type: 1 }); ImageSchema.index({ teamId: 1 }); + ImageSchema.index({ createTime: 1 }); + ImageSchema.index({ 'metadata.relatedId': 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/common/file/load/pdf.ts b/packages/service/common/file/load/pdf.ts deleted file mode 100644 index 4cbb4673d..000000000 --- a/packages/service/common/file/load/pdf.ts +++ /dev/null @@ -1,68 +0,0 @@ -import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'; -// @ts-ignore -import('pdfjs-dist/legacy/build/pdf.worker.min.mjs'); -import { ReadFileParams } from './type'; - -type TokenType = { - str: string; - dir: string; - width: number; - height: number; - transform: number[]; - fontName: string; - hasEOL: boolean; -}; - -export const readPdfFile = async ({ path }: ReadFileParams) => { - const readPDFPage = async (doc: any, pageNo: number) => { - const page = await doc.getPage(pageNo); - const tokenizedText = await page.getTextContent(); - - const viewport = page.getViewport({ scale: 1 }); - const pageHeight = viewport.height; - const headerThreshold = pageHeight * 0.95; - const footerThreshold = pageHeight * 0.05; - - const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => { - return ( - !token.transform || - (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold) - ); - }); - - // concat empty string 'hasEOL' - for (let i = 0; i < pageTexts.length; i++) { - const item = pageTexts[i]; - if (item.str === '' && pageTexts[i - 1]) { - pageTexts[i - 1].hasEOL = item.hasEOL; - pageTexts.splice(i, 1); - i--; - } - } - - page.cleanup(); - - return pageTexts - .map((token) => { - const paragraphEnd = token.hasEOL && /([。？！.?!\n\r]|(\r\n))$/.test(token.str); - - return paragraphEnd ? `${token.str}\n` : token.str; - }) - .join(''); - }; - - const loadingTask = pdfjs.getDocument(path); - const doc = await loadingTask.promise; - - const pageTextPromises = []; - for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) { - pageTextPromises.push(readPDFPage(doc, pageNo)); - } - const pageTexts = await Promise.all(pageTextPromises); - - loadingTask.destroy(); - - return { - rawText: pageTexts.join('') - }; -}; diff --git a/packages/service/common/file/load/type.d.ts b/packages/service/common/file/load/type.d.ts deleted file mode 100644 index ffb9f3ee0..000000000 --- a/packages/service/common/file/load/type.d.ts +++ /dev/null @@ -1,18 +0,0 @@ -export type ReadFileParams = { - preview: boolean; - teamId: string; - path: string; - metadata?: Record; -}; - -export type ReadFileResponse = { - rawText: string; -}; - -export type ReadFileBufferItemType = ReadFileParams & { - rawText: string; -}; - -declare global { - var readFileBuffers: ReadFileBufferItemType[]; -} diff --git a/packages/service/common/file/load/utils.ts b/packages/service/common/file/load/utils.ts deleted file mode 100644 index 760bc610e..000000000 --- a/packages/service/common/file/load/utils.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { readPdfFile } from './pdf'; -import { readDocFle } from './word'; -import { ReadFileBufferItemType, ReadFileParams } from './type'; - -global.readFileBuffers = global.readFileBuffers || []; - -const bufferMaxSize = 200; - -export const pushFileReadBuffer = (params: ReadFileBufferItemType) => { - global.readFileBuffers.push(params); - - if (global.readFileBuffers.length > bufferMaxSize) { - global.readFileBuffers.shift(); - } -}; -export const getReadFileBuffer = ({ path, teamId }: ReadFileParams) => - global.readFileBuffers.find((item) => item.path === path && item.teamId === teamId); - -export const readFileContent = async (params: ReadFileParams) => { - const { path } = params; - - const buffer = getReadFileBuffer(params); - - if (buffer) { - return buffer; - } - - const extension = path?.split('.')?.pop()?.toLowerCase() || ''; - - const { rawText } = await (async () => { - switch (extension) { - case 'pdf': - return readPdfFile(params); - case 'docx': - return readDocFle(params); - default: - return Promise.reject('Only support .pdf, .docx'); - } - })(); - - pushFileReadBuffer({ - ...params, - rawText - }); - - return { - ...params, - rawText - }; -}; diff --git a/packages/service/common/file/load/word.ts b/packages/service/common/file/load/word.ts deleted file mode 100644 index 8842a6fd9..000000000 --- a/packages/service/common/file/load/word.ts +++ /dev/null @@ -1,22 +0,0 @@ -import mammoth from 'mammoth'; -import { htmlToMarkdown } from '../../string/markdown'; -import { ReadFileParams } from './type'; -/** - * read docx to markdown - */ -export const readDocFle = async ({ path, metadata = {} }: ReadFileParams) => { - try { - const { value: html } = await mammoth.convertToHtml({ - path - }); - - const md = await htmlToMarkdown(html); - - return { - rawText: md - }; - } catch (error) { - console.log('error doc read:', error); - return Promise.reject('Can not read doc file, please convert to PDF'); - } -}; diff --git a/packages/service/common/file/multer.ts b/packages/service/common/file/multer.ts index 62ff53313..90f378923 100644 --- a/packages/service/common/file/multer.ts +++ b/packages/service/common/file/multer.ts @@ -3,7 +3,6 @@ import multer from 'multer'; import path from 'path'; import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants'; import { getNanoid } from '@fastgpt/global/common/string/tools'; -import { tmpFileDirPath } from './constants'; type FileType = { fieldname: string; @@ -15,8 +14,6 @@ type FileType = { size: number; }; -const expiredTime = 30 * 60 * 1000; - export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { maxSize *= 1024 * 1024; class UploadModel { @@ -31,15 +28,16 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { // }, filename: async (req, file, cb) => { const { ext } = path.parse(decodeURIComponent(file.originalname)); - cb(null, `${Date.now() + expiredTime}-${getNanoid(32)}${ext}`); + cb(null, `${getNanoid(32)}${ext}`); } }) - }).any(); + }).single('file'); async doUpload>(req: NextApiRequest, res: NextApiResponse) { return new Promise<{ - files: FileType[]; - metadata: T; + file: FileType; + metadata: Record; + data: T; bucketName?: `${BucketNameEnum}`; }>((resolve, reject) => { // @ts-ignore @@ -54,20 +52,28 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { return reject('BucketName is invalid'); } + // @ts-ignore + const file = req.file as FileType; + resolve({ - ...req.body, - files: - // @ts-ignore - req.files?.map((file) => ({ - ...file, - originalname: decodeURIComponent(file.originalname) - })) || [], + file: { + ...file, + originalname: decodeURIComponent(file.originalname) + }, + bucketName, metadata: (() => { if (!req.body?.metadata) return {}; try { return JSON.parse(req.body.metadata); } catch (error) { - console.log(error); + return {}; + } + })(), + data: (() => { + if (!req.body?.data) return {}; + try { + return JSON.parse(req.body.data); + } catch (error) { return {}; } })() diff --git a/packages/service/common/file/utils.ts b/packages/service/common/file/utils.ts index 641aed8f9..f3214056f 100644 --- a/packages/service/common/file/utils.ts +++ b/packages/service/common/file/utils.ts @@ -1,5 +1,4 @@ import fs from 'fs'; -import { tmpFileDirPath } from './constants'; export const removeFilesByPaths = (paths: string[]) => { paths.forEach((path) => { @@ -10,24 +9,3 @@ export const removeFilesByPaths = (paths: string[]) => { }); }); }; - -/* cron job. check expired tmp files */ -export const checkExpiredTmpFiles = () => { - // get all file name - const files = fs.readdirSync(tmpFileDirPath).map((name) => { - const timestampStr = name.split('-')[0]; - const expiredTimestamp = timestampStr ? Number(timestampStr) : 0; - - return { - filename: name, - expiredTimestamp, - path: `${tmpFileDirPath}/${name}` - }; - }); - - // count expiredFiles - const expiredFiles = files.filter((item) => item.expiredTimestamp < Date.now()); - - // remove expiredFiles - removeFilesByPaths(expiredFiles.map((item) => item.path)); -}; diff --git a/packages/service/common/string/cheerio.ts b/packages/service/common/string/cheerio.ts index 722e77c46..5bb56495b 100644 --- a/packages/service/common/string/cheerio.ts +++ b/packages/service/common/string/cheerio.ts @@ -64,41 +64,39 @@ export const urlsFetch = async ({ }: UrlFetchParams): Promise => { urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url)); - const response = ( - await Promise.all( - urlList.map(async (url) => { - try { - const fetchRes = await axios.get(url, { - timeout: 30000 - }); + const response = await Promise.all( + urlList.map(async (url) => { + try { + const fetchRes = await axios.get(url, { + timeout: 30000 + }); - const $ = cheerio.load(fetchRes.data); - const { title, html, usedSelector } = cheerioToHtml({ - fetchUrl: url, - $, - selector - }); - const md = await htmlToMarkdown(html); + const $ = cheerio.load(fetchRes.data); + const { title, html, usedSelector } = cheerioToHtml({ + fetchUrl: url, + $, + selector + }); + const md = await htmlToMarkdown(html); - return { - url, - title, - content: md, - selector: usedSelector - }; - } catch (error) { - console.log(error, 'fetch error'); + return { + url, + title, + content: md, + selector: usedSelector + }; + } catch (error) { + console.log(error, 'fetch error'); - return { - url, - title: '', - content: '', - selector: '' - }; - } - }) - ) - ).filter((item) => item.content); + return { + url, + title: '', + content: '', + selector: '' + }; + } + }) + ); return response; }; diff --git a/packages/service/common/vectorStore/controller.d.ts b/packages/service/common/vectorStore/controller.d.ts index 671230299..c395939b4 100644 --- a/packages/service/common/vectorStore/controller.d.ts +++ b/packages/service/common/vectorStore/controller.d.ts @@ -1,21 +1,19 @@ export type DeleteDatasetVectorProps = { + teamId: string; + id?: string; datasetIds?: string[]; collectionIds?: string[]; - - collectionId?: string; - dataIds?: string[]; + idList?: string[]; }; export type InsertVectorProps = { teamId: string; - tmbId: string; datasetId: string; collectionId: string; - dataId: string; }; export type EmbeddingRecallProps = { - similarity?: number; datasetIds: string[]; + similarity?: number; }; diff --git a/packages/service/common/vectorStore/controller.ts b/packages/service/common/vectorStore/controller.ts index 9d9ef5660..423c4093c 100644 --- a/packages/service/common/vectorStore/controller.ts +++ b/packages/service/common/vectorStore/controller.ts @@ -10,6 +10,7 @@ const getVectorObj = () => { export const initVectorStore = getVectorObj().init; export const deleteDatasetDataVector = getVectorObj().delete; export const recallFromVectorStore = getVectorObj().recall; +export const checkVectorDataExist = getVectorObj().checkDataExist; export const getVectorDataByTime = getVectorObj().getVectorDataByTime; export const getVectorCountByTeamId = getVectorObj().getVectorCountByTeamId; @@ -21,7 +22,7 @@ export const insertDatasetDataVector = async ({ query: string; model: string; }) => { - const { vectors, tokens } = await getVectorsByText({ + const { vectors, charsLength } = await getVectorsByText({ model, input: query }); @@ -31,32 +32,27 @@ export const insertDatasetDataVector = async ({ }); return { - tokens, + charsLength, insertId }; }; export const updateDatasetDataVector = async ({ id, - query, - model -}: { + ...props +}: InsertVectorProps & { id: string; query: string; model: string; }) => { - // get vector - const { vectors, tokens } = await getVectorsByText({ - model, - input: query + // insert new vector + const { charsLength, insertId } = await insertDatasetDataVector(props); + + // delete old vector + await deleteDatasetDataVector({ + teamId: props.teamId, + id }); - await getVectorObj().update({ - id, - vectors - }); - - return { - tokens - }; + return { charsLength, insertId }; }; diff --git a/packages/service/common/vectorStore/pg/class.ts b/packages/service/common/vectorStore/pg/class.ts index 3685bf41f..11a8480a9 100644 --- a/packages/service/common/vectorStore/pg/class.ts +++ b/packages/service/common/vectorStore/pg/class.ts @@ -1,20 +1,20 @@ import { initPg, insertDatasetDataVector, - updateDatasetDataVector, deleteDatasetDataVector, embeddingRecall, getVectorDataByTime, - getVectorCountByTeamId + getVectorCountByTeamId, + checkDataExist } from './controller'; export class PgVector { constructor() {} init = initPg; insert = insertDatasetDataVector; - update = updateDatasetDataVector; delete = deleteDatasetDataVector; recall = embeddingRecall; + checkDataExist = checkDataExist; getVectorCountByTeamId = getVectorCountByTeamId; getVectorDataByTime = getVectorDataByTime; } diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index a2e044d6f..f91da1efb 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -4,7 +4,7 @@ import { delay } from '@fastgpt/global/common/system/utils'; import { PgClient, connectPg } from './index'; import { PgSearchRawType } from '@fastgpt/global/core/dataset/api'; import { EmbeddingRecallItemType } from '../type'; -import { DeleteDatasetVectorProps, EmbeddingRecallProps } from '../controller.d'; +import { DeleteDatasetVectorProps, EmbeddingRecallProps, InsertVectorProps } from '../controller.d'; import dayjs from 'dayjs'; export async function initPg() { @@ -16,11 +16,9 @@ export async function initPg() { id BIGSERIAL PRIMARY KEY, vector VECTOR(1536) NOT NULL, team_id VARCHAR(50) NOT NULL, - tmb_id VARCHAR(50) NOT NULL, dataset_id VARCHAR(50) NOT NULL, collection_id VARCHAR(50) NOT NULL, - data_id VARCHAR(50) NOT NULL, - createTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP + createtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); `); @@ -34,26 +32,21 @@ export async function initPg() { } } -export const insertDatasetDataVector = async (props: { - teamId: string; - tmbId: string; - datasetId: string; - collectionId: string; - dataId: string; - vectors: number[][]; - retry?: number; -}): Promise<{ insertId: string }> => { - const { dataId, teamId, tmbId, datasetId, collectionId, vectors, retry = 3 } = props; +export const insertDatasetDataVector = async ( + props: InsertVectorProps & { + vectors: number[][]; + retry?: number; + } +): Promise<{ insertId: string }> => { + const { teamId, datasetId, collectionId, vectors, retry = 3 } = props; try { const { rows } = await PgClient.insert(PgDatasetTableName, { values: [ [ { key: 'vector', value: `[${vectors[0]}]` }, { key: 'team_id', value: String(teamId) }, - { key: 'tmb_id', value: String(tmbId) }, { key: 'dataset_id', value: datasetId }, - { key: 'collection_id', value: collectionId }, - { key: 'data_id', value: String(dataId) } + { key: 'collection_id', value: collectionId } ] ] }); @@ -72,48 +65,33 @@ export const insertDatasetDataVector = async (props: { } }; -export const updateDatasetDataVector = async (props: { - id: string; - vectors: number[][]; - retry?: number; -}): Promise => { - const { id, vectors, retry = 2 } = props; - try { - // update pg - await PgClient.update(PgDatasetTableName, { - where: [['id', id]], - values: [{ key: 'vector', value: `[${vectors[0]}]` }] - }); - } catch (error) { - if (retry <= 0) { - return Promise.reject(error); - } - await delay(500); - return updateDatasetDataVector({ - ...props, - retry: retry - 1 - }); - } -}; - export const deleteDatasetDataVector = async ( props: DeleteDatasetVectorProps & { retry?: number; } ): Promise => { - const { id, datasetIds, collectionIds, collectionId, dataIds, retry = 2 } = props; + const { teamId, id, datasetIds, collectionIds, idList, retry = 2 } = props; + + const teamIdWhere = `team_id='${String(teamId)}' AND`; const where = await (() => { - if (id) return `id=${id}`; - if (datasetIds) return `dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})`; - if (collectionIds) { - return `collection_id IN (${collectionIds.map((id) => `'${String(id)}'`).join(',')})`; - } - if (collectionId && dataIds) { - return `collection_id='${String(collectionId)}' and data_id IN (${dataIds + if (id) return `${teamIdWhere} id=${id}`; + + if (datasetIds) { + return `${teamIdWhere} dataset_id IN (${datasetIds .map((id) => `'${String(id)}'`) .join(',')})`; } + + if (collectionIds) { + return `${teamIdWhere} collection_id IN (${collectionIds + .map((id) => `'${String(id)}'`) + .join(',')})`; + } + + if (idList) { + return `${teamIdWhere} id IN (${idList.map((id) => `'${String(id)}'`).join(',')})`; + } return Promise.reject('deleteDatasetData: no where'); })(); @@ -142,13 +120,13 @@ export const embeddingRecall = async ( ): Promise<{ results: EmbeddingRecallItemType[]; }> => { - const { vectors, limit, similarity = 0, datasetIds, retry = 2 } = props; + const { datasetIds, vectors, limit, similarity = 0, retry = 2 } = props; try { const results: any = await PgClient.query( `BEGIN; SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 100}; - select id, collection_id, data_id, (vector <#> '[${vectors[0]}]') * -1 AS score + select id, collection_id, (vector <#> '[${vectors[0]}]') * -1 AS score from ${PgDatasetTableName} where dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} @@ -158,21 +136,10 @@ export const embeddingRecall = async ( const rows = results?.[2]?.rows as PgSearchRawType[]; - // concat same data_id - const filterRows: PgSearchRawType[] = []; - let set = new Set(); - for (const row of rows) { - if (!set.has(row.data_id)) { - filterRows.push(row); - set.add(row.data_id); - } - } - return { - results: filterRows.map((item) => ({ + results: rows.map((item) => ({ id: item.id, collectionId: item.collection_id, - dataId: item.data_id, score: item.score })) }; @@ -184,7 +151,11 @@ export const embeddingRecall = async ( } }; -// bill +export const checkDataExist = async (id: string) => { + const { rows } = await PgClient.query(`SELECT id FROM ${PgDatasetTableName} WHERE id=${id};`); + + return rows.length > 0; +}; export const getVectorCountByTeamId = async (teamId: string) => { const total = await PgClient.count(PgDatasetTableName, { where: [['team_id', String(teamId)]] @@ -193,15 +164,20 @@ export const getVectorCountByTeamId = async (teamId: string) => { return total; }; export const getVectorDataByTime = async (start: Date, end: Date) => { - const { rows } = await PgClient.query<{ id: string; data_id: string }>(`SELECT id, data_id + const { rows } = await PgClient.query<{ + id: string; + team_id: string; + dataset_id: string; + }>(`SELECT id, team_id, dataset_id FROM ${PgDatasetTableName} - WHERE createTime BETWEEN '${dayjs(start).format('YYYY-MM-DD')}' AND '${dayjs(end).format( - 'YYYY-MM-DD 23:59:59' + WHERE createtime BETWEEN '${dayjs(start).format('YYYY-MM-DD HH:mm:ss')}' AND '${dayjs(end).format( + 'YYYY-MM-DD HH:mm:ss' )}'; `); return rows.map((item) => ({ id: item.id, - dataId: item.data_id + datasetId: item.dataset_id, + teamId: item.team_id })); }; diff --git a/packages/service/common/vectorStore/type.d.ts b/packages/service/common/vectorStore/type.d.ts index 99807dcee..0f7624662 100644 --- a/packages/service/common/vectorStore/type.d.ts +++ b/packages/service/common/vectorStore/type.d.ts @@ -7,6 +7,5 @@ declare global { export type EmbeddingRecallItemType = { id: string; collectionId: string; - dataId: string; score: number; }; diff --git a/packages/service/core/ai/embedding/index.ts b/packages/service/core/ai/embedding/index.ts index 2b785ab0a..7c8e86b48 100644 --- a/packages/service/core/ai/embedding/index.ts +++ b/packages/service/core/ai/embedding/index.ts @@ -18,10 +18,9 @@ export async function getVectorsByText({ } try { - // 获取 chatAPI const ai = getAIApi(); - // 把输入的内容转成向量 + // input text to vector const result = await ai.embeddings .create({ model, @@ -38,7 +37,7 @@ export async function getVectorsByText({ } return { - tokens: res.usage.total_tokens || 0, + charsLength: input.length, vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding))) }; }); @@ -53,7 +52,9 @@ export async function getVectorsByText({ function unityDimensional(vector: number[]) { if (vector.length > 1536) { - console.log(`当前向量维度为: ${vector.length}, 向量维度不能超过 1536, 已自动截取前 1536 维度`); + console.log( + `The current vector dimension is ${vector.length}, and the vector dimension cannot exceed 1536. The first 1536 dimensions are automatically captured` + ); return vector.slice(0, 1536); } let resultVector = vector; diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index aefbd7d01..bf71c8d65 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -11,6 +11,8 @@ import { appCollectionName } from '../app/schema'; import { userCollectionName } from '../../support/user/schema'; import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants'; +export const ChatItemCollectionName = 'chatitems'; + const ChatItemSchema = new Schema({ teamId: { type: Schema.Types.ObjectId, @@ -79,20 +81,23 @@ const ChatItemSchema = new Schema({ }); try { - ChatItemSchema.index({ teamId: 1 }); - ChatItemSchema.index({ time: -1 }); - ChatItemSchema.index({ appId: 1 }); - ChatItemSchema.index({ chatId: 1 }); - ChatItemSchema.index({ obj: 1 }); - ChatItemSchema.index({ userGoodFeedback: 1 }); - ChatItemSchema.index({ userBadFeedback: 1 }); - ChatItemSchema.index({ customFeedbacks: 1 }); - ChatItemSchema.index({ adminFeedback: 1 }); + ChatItemSchema.index({ dataId: 1 }, { background: true }); + /* delete by app; + delete by chat id; + get chat list; + get chat logs; + close custom feedback; + */ + ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true }); + ChatItemSchema.index({ userGoodFeedback: 1 }, { background: true }); + ChatItemSchema.index({ userBadFeedback: 1 }, { background: true }); + ChatItemSchema.index({ customFeedbacks: 1 }, { background: true }); + ChatItemSchema.index({ adminFeedback: 1 }, { background: true }); } catch (error) { console.log(error); } export const MongoChatItem: Model = - models['chatItem'] || model('chatItem', ChatItemSchema); + models[ChatItemCollectionName] || model(ChatItemCollectionName, ChatItemSchema); MongoChatItem.syncIndexes(); diff --git a/packages/service/core/chat/chatSchema.ts b/packages/service/core/chat/chatSchema.ts index 637e69550..52cc92526 100644 --- a/packages/service/core/chat/chatSchema.ts +++ b/packages/service/core/chat/chatSchema.ts @@ -1,13 +1,12 @@ import { connectionMongo, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d'; -import { ChatRoleMap, ChatSourceMap } from '@fastgpt/global/core/chat/constants'; +import { ChatSourceMap } from '@fastgpt/global/core/chat/constants'; import { TeamCollectionName, TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; import { appCollectionName } from '../app/schema'; -import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants'; export const chatCollectionName = 'chat'; @@ -48,7 +47,8 @@ const ChatSchema = new Schema({ default: '' }, top: { - type: Boolean + type: Boolean, + default: false }, source: { type: String, @@ -73,10 +73,16 @@ const ChatSchema = new Schema({ }); try { - ChatSchema.index({ appId: 1 }); - ChatSchema.index({ tmbId: 1 }); - ChatSchema.index({ shareId: 1 }); - ChatSchema.index({ updateTime: -1 }); + ChatSchema.index({ chatId: 1 }, { background: true }); + // get user history + ChatSchema.index({ tmbId: 1, appId: 1, top: -1, updateTime: -1 }, { background: true }); + // delete by appid; clear history; init chat; update chat; auth chat; + ChatSchema.index({ appId: 1, chatId: 1 }, { background: true }); + + // get chat logs; + ChatSchema.index({ teamId: 1, appId: 1, updateTime: -1 }, { background: true }); + // get share chat history + ChatSchema.index({ shareId: 1, outLinkUid: 1 }, { background: true }); } catch (error) { console.log(error); } diff --git a/packages/service/core/chat/controller.ts b/packages/service/core/chat/controller.ts index 59d465f8d..094fc11bf 100644 --- a/packages/service/core/chat/controller.ts +++ b/packages/service/core/chat/controller.ts @@ -3,10 +3,12 @@ import { MongoChatItem } from './chatItemSchema'; import { addLog } from '../../common/system/log'; export async function getChatItems({ + appId, chatId, limit = 30, field }: { + appId: string; chatId?: string; limit?: number; field: string; @@ -15,7 +17,10 @@ export async function getChatItems({ return { history: [] }; } - const history = await MongoChatItem.find({ chatId }, field).sort({ _id: -1 }).limit(limit).lean(); + const history = await MongoChatItem.find({ appId, chatId }, field) + .sort({ _id: -1 }) + .limit(limit) + .lean(); history.reverse(); @@ -23,10 +28,12 @@ export async function getChatItems({ } export const addCustomFeedbacks = async ({ + appId, chatId, chatItemId, feedbacks }: { + appId: string; chatId?: string; chatItemId?: string; feedbacks: string[]; diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index d0e7c14d2..4a609088f 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -1,6 +1,20 @@ -import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + TrainingModeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { MongoDatasetCollection } from './schema'; +import { + CollectionWithDatasetType, + DatasetCollectionSchemaType +} from '@fastgpt/global/core/dataset/type'; +import { MongoDatasetTraining } from '../training/schema'; +import { delay } from '@fastgpt/global/common/system/utils'; +import { MongoDatasetData } from '../data/schema'; +import { delImgByRelatedId } from '../../../common/file/image/controller'; +import { deleteDatasetDataVector } from '../../../common/vectorStore/controller'; +import { delFileByFileIdList } from '../../../common/file/gridfs/controller'; +import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; export async function createOneCollection({ teamId, @@ -85,20 +99,50 @@ export function createDefaultCollection({ }); } -// check same collection -export const getSameRawTextCollection = async ({ - datasetId, - hashRawText +/** + * delete collection and it related data + */ +export async function delCollectionAndRelatedSources({ + collections }: { - datasetId: string; - hashRawText?: string; -}) => { - if (!hashRawText) return undefined; + collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[]; +}) { + if (collections.length === 0) return; - const collection = await MongoDatasetCollection.findOne({ - datasetId, - hashRawText + const teamId = collections[0].teamId; + + if (!teamId) return Promise.reject('teamId is not exist'); + + const collectionIds = collections.map((item) => String(item._id)); + const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean); + const relatedImageIds = collections + .map((item) => item?.metadata?.relatedImgId || '') + .filter(Boolean); + + // delete training data + await MongoDatasetTraining.deleteMany({ + teamId, + collectionId: { $in: collectionIds } }); - return collection; -}; + await delay(2000); + + // delete dataset.datas + await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }); + // delete pg data + await deleteDatasetDataVector({ teamId, collectionIds }); + + // delete file and imgs + await Promise.all([ + delImgByRelatedId(relatedImageIds), + delFileByFileIdList({ + bucketName: BucketNameEnum.dataset, + fileIdList + }) + ]); + + // delete collections + await MongoDatasetCollection.deleteMany({ + _id: { $in: collectionIds } + }); +} diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index f2b5f8def..4b3d4ca8d 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -1,7 +1,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; -import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant'; +import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants'; import { DatasetCollectionName } from '../schema'; import { TeamCollectionName, @@ -91,11 +91,19 @@ const DatasetCollectionSchema = new Schema({ }); try { - DatasetCollectionSchema.index({ teamId: 1 }); - DatasetCollectionSchema.index({ datasetId: 1 }); - DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, parentId: 1 }); - DatasetCollectionSchema.index({ updateTime: -1 }); - DatasetCollectionSchema.index({ hashRawText: -1 }); + // auth file + DatasetCollectionSchema.index({ teamId: 1, fileId: 1 }, { background: true }); + + // list collection; deep find collections + DatasetCollectionSchema.index( + { + teamId: 1, + datasetId: 1, + parentId: 1, + updateTime: -1 + }, + { background: true } + ); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 225e01b22..9fc3cdf21 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -4,16 +4,32 @@ import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { MongoDatasetTraining } from '../training/schema'; import { urlsFetch } from '../../../common/string/cheerio'; -import { DatasetCollectionTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + DatasetCollectionTypeEnum, + TrainingModeEnum +} from '@fastgpt/global/core/dataset/constants'; import { hashStr } from '@fastgpt/global/common/string/tools'; /** * get all collection by top collectionId */ -export async function findCollectionAndChild(id: string, fields = '_id parentId name metadata') { +export async function findCollectionAndChild({ + teamId, + datasetId, + collectionId, + fields = '_id parentId name metadata' +}: { + teamId: string; + datasetId: string; + collectionId: string; + fields?: string; +}) { async function find(id: string) { // find children - const children = await MongoDatasetCollection.find({ parentId: id }, fields); + const children = await MongoDatasetCollection.find( + { teamId, datasetId, parentId: id }, + fields + ).lean(); let collections = children; @@ -25,8 +41,8 @@ export async function findCollectionAndChild(id: string, fields = '_id parentId return collections; } const [collection, childCollections] = await Promise.all([ - MongoDatasetCollection.findById(id, fields), - find(id) + MongoDatasetCollection.findById(collectionId, fields), + find(collectionId) ]); if (!collection) { @@ -107,8 +123,8 @@ export const getCollectionAndRawText = async ({ }); return { - title: result[0].title, - rawText: result[0].content + title: result[0]?.title, + rawText: result[0]?.content }; } @@ -121,7 +137,7 @@ export const getCollectionAndRawText = async ({ })(); const hashRawText = hashStr(rawText); - const isSameRawText = col.hashRawText === hashRawText; + const isSameRawText = rawText && col.hashRawText === hashRawText; return { collection: col, @@ -161,8 +177,7 @@ export const reloadCollectionChunks = async ({ // split data const { chunks } = splitText2Chunks({ text: newRawText, - chunkLen: col.chunkSize || 512, - countTokens: false + chunkLen: col.chunkSize || 512 }); // insert to training queue diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index 20faa55ae..65f9ed876 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -1,24 +1,47 @@ -import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type'; +import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from './collection/schema'; import { MongoDataset } from './schema'; +import { delCollectionAndRelatedSources } from './collection/controller'; /* ============= dataset ========== */ /* find all datasetId by top datasetId */ -export async function findDatasetIdTreeByTopDatasetId( - id: string, - result: string[] = [] -): Promise { - let allChildrenIds = [...result]; +export async function findDatasetAndAllChildren({ + teamId, + datasetId, + fields +}: { + teamId: string; + datasetId: string; + fields?: string; +}): Promise { + const find = async (id: string) => { + const children = await MongoDataset.find( + { + teamId, + parentId: id + }, + fields + ).lean(); - // find children - const children = await MongoDataset.find({ parentId: id }); + let datasets = children; - for (const child of children) { - const grandChildrenIds = await findDatasetIdTreeByTopDatasetId(child._id, result); - allChildrenIds = allChildrenIds.concat(grandChildrenIds); + for (const child of children) { + const grandChildrenIds = await find(child._id); + datasets = datasets.concat(grandChildrenIds); + } + + return datasets; + }; + const [dataset, childDatasets] = await Promise.all([ + MongoDataset.findById(datasetId), + find(datasetId) + ]); + + if (!dataset) { + return Promise.reject('Dataset not found'); } - return [String(id), ...allChildrenIds]; + return [dataset, ...childDatasets]; } export async function getCollectionWithDataset(collectionId: string) { @@ -30,3 +53,22 @@ export async function getCollectionWithDataset(collectionId: string) { } return data; } + +/* delete all data by datasetIds */ +export async function delDatasetRelevantData({ datasets }: { datasets: DatasetSchemaType[] }) { + if (!datasets.length) return; + + const teamId = datasets[0].teamId; + const datasetIds = datasets.map((item) => String(item._id)); + + // Get _id, teamId, fileId, metadata.relatedImgId for all collections + const collections = await MongoDatasetCollection.find( + { + teamId, + datasetId: { $in: datasetIds } + }, + '_id teamId fileId metadata' + ).lean(); + + await delCollectionAndRelatedSources({ collections }); +} diff --git a/packages/service/core/dataset/data/controller.ts b/packages/service/core/dataset/data/controller.ts index e1af0ad0b..ebb463c1e 100644 --- a/packages/service/core/dataset/data/controller.ts +++ b/packages/service/core/dataset/data/controller.ts @@ -1,87 +1,2 @@ import { MongoDatasetData } from './schema'; -import { MongoDatasetTraining } from '../training/schema'; -import { delFileByFileIdList, delFileByMetadata } from '../../../common/file/gridfs/controller'; -import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { MongoDatasetCollection } from '../collection/schema'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { delImgByFileIdList } from '../../../common/file/image/controller'; import { deleteDatasetDataVector } from '../../../common/vectorStore/controller'; - -/* delete all data by datasetIds */ -export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) { - datasetIds = datasetIds.map((item) => String(item)); - - // delete training data(There could be a training mission) - await MongoDatasetTraining.deleteMany({ - datasetId: { $in: datasetIds } - }); - - await delay(2000); - - // delete dataset.datas - await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } }); - // delete pg data - await deleteDatasetDataVector({ datasetIds }); - - // delete collections - await MongoDatasetCollection.deleteMany({ - datasetId: { $in: datasetIds } - }); - - // delete related files - await Promise.all( - datasetIds.map((id) => delFileByMetadata({ bucketName: BucketNameEnum.dataset, datasetId: id })) - ); -} -/** - * delete all data by collectionIds - */ -export async function delCollectionRelevantData({ - collectionIds, - fileIds -}: { - collectionIds: string[]; - fileIds: string[]; -}) { - collectionIds = collectionIds.filter(Boolean).map((item) => String(item)); - const filterFileIds = fileIds.filter(Boolean).map((item) => String(item)); - - // delete training data - await MongoDatasetTraining.deleteMany({ - collectionId: { $in: collectionIds } - }); - - await delay(2000); - - // delete dataset.datas - await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } }); - // delete pg data - await deleteDatasetDataVector({ collectionIds }); - - // delete collections - await MongoDatasetCollection.deleteMany({ - _id: { $in: collectionIds } - }); - - // delete file and imgs - await Promise.all([ - delImgByFileIdList(filterFileIds), - delFileByFileIdList({ - bucketName: BucketNameEnum.dataset, - fileIdList: filterFileIds - }) - ]); -} -/** - * delete one data by mongoDataId - */ -export async function delDatasetDataByDataId({ - collectionId, - mongoDataId -}: { - collectionId: string; - mongoDataId: string; -}) { - await deleteDatasetDataVector({ collectionId, dataIds: [mongoDataId] }); - await MongoDatasetData.findByIdAndDelete(mongoDataId); -} diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index 8681cc9fc..8209ae786 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -10,7 +10,7 @@ import { DatasetColCollectionName } from '../collection/schema'; import { DatasetDataIndexTypeEnum, DatasetDataIndexTypeMap -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; export const DatasetDataCollectionName = 'dataset.datas'; @@ -71,6 +71,7 @@ const DatasetDataSchema = new Schema({ ], default: [] }, + updateTime: { type: Date, default: () => new Date() @@ -85,13 +86,18 @@ const DatasetDataSchema = new Schema({ }); try { - DatasetDataSchema.index({ teamId: 1 }); - DatasetDataSchema.index({ datasetId: 1 }); - DatasetDataSchema.index({ collectionId: 1 }); - DatasetDataSchema.index({ updateTime: -1 }); - DatasetDataSchema.index({ collectionId: 1, q: 1, a: 1 }); + // same data check + DatasetDataSchema.index({ teamId: 1, collectionId: 1, q: 1, a: 1 }, { background: true }); + // list collection and count data; list data + DatasetDataSchema.index( + { teamId: 1, datasetId: 1, collectionId: 1, chunkIndex: 1, updateTime: -1 }, + { background: true } + ); // full text index - DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' }); + DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }, { background: true }); + // Recall vectors after data matching + DatasetDataSchema.index({ teamId: 1, datasetId: 1, 'indexes.dataId': 1 }, { background: true }); + DatasetDataSchema.index({ updateTime: 1 }, { background: true }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 827487a23..fc7135037 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -5,7 +5,7 @@ import { DatasetStatusEnum, DatasetStatusMap, DatasetTypeMap -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import { TeamCollectionName, TeamMemberCollectionName diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index 8d7742763..410103571 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type'; -import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; +import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants'; import { DatasetColCollectionName } from '../collection/schema'; import { DatasetCollectionName } from '../schema'; import { @@ -102,11 +102,11 @@ const TrainingDataSchema = new Schema({ }); try { - TrainingDataSchema.index({ teamId: 1 }); + // lock training data; delete training data + TrainingDataSchema.index({ teamId: 1, collectionId: 1 }); + // get training data and sort TrainingDataSchema.index({ weight: -1 }); TrainingDataSchema.index({ lockTime: 1 }); - TrainingDataSchema.index({ datasetId: 1 }); - TrainingDataSchema.index({ collectionId: 1 }); TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 }); } catch (error) { console.log(error); diff --git a/packages/service/package.json b/packages/service/package.json index baa926fb5..997c343cf 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -9,13 +9,11 @@ "dayjs": "^1.11.7", "encoding": "^0.1.13", "jsonwebtoken": "^9.0.2", - "mammoth": "^1.6.0", "mongoose": "^7.0.2", "multer": "1.4.5-lts.1", "next": "13.5.2", "nextjs-cors": "^2.1.2", "node-cron": "^3.0.3", - "pdfjs-dist": "^4.0.269", "pg": "^8.10.0", "tunnel": "^0.0.6" }, diff --git a/packages/service/support/permission/auth/app.ts b/packages/service/support/permission/auth/app.ts index 5970a0a56..14c72e233 100644 --- a/packages/service/support/permission/auth/app.ts +++ b/packages/service/support/permission/auth/app.ts @@ -6,7 +6,7 @@ import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { parseHeaderCert } from '../controller'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import { AppErrEnum } from '@fastgpt/global/common/error/code/app'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; // 模型使用权校验 export async function authApp({ @@ -24,7 +24,7 @@ export async function authApp({ > { const result = await parseHeaderCert(props); const { teamId, tmbId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { app, isOwner, canWrite } = await (async () => { // get app diff --git a/packages/service/support/permission/auth/dataset.ts b/packages/service/support/permission/auth/dataset.ts index 3b92ec8cd..c793714eb 100644 --- a/packages/service/support/permission/auth/dataset.ts +++ b/packages/service/support/permission/auth/dataset.ts @@ -13,8 +13,9 @@ import { } from '@fastgpt/global/core/dataset/type'; import { getFileById } from '../../../common/file/gridfs/controller'; import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { MongoDatasetCollection } from '../../../core/dataset/collection/schema'; export async function authDatasetByTmbId({ teamId, @@ -27,7 +28,7 @@ export async function authDatasetByTmbId({ datasetId: string; per: AuthModeType['per']; }) { - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { dataset, isOwner, canWrite } = await (async () => { const dataset = await MongoDataset.findOne({ _id: datasetId, teamId }).lean(); @@ -107,7 +108,7 @@ export async function authDatasetCollection({ } > { const { userId, teamId, tmbId } = await parseHeaderCert(props); - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { collection, isOwner, canWrite } = await (async () => { const collection = await getCollectionWithDataset(collectionId); @@ -163,47 +164,40 @@ export async function authDatasetFile({ } > { const { userId, teamId, tmbId } = await parseHeaderCert(props); - const { role } = await getTeamInfoByTmbId({ tmbId }); - const file = await getFileById({ bucketName: BucketNameEnum.dataset, fileId }); + const [file, collection] = await Promise.all([ + getFileById({ bucketName: BucketNameEnum.dataset, fileId }), + MongoDatasetCollection.findOne({ + teamId, + fileId + }) + ]); if (!file) { return Promise.reject(CommonErrEnum.fileNotFound); } - if (file.metadata.teamId !== teamId) { + if (!collection) { return Promise.reject(DatasetErrEnum.unAuthDatasetFile); } - const { dataset } = await authDataset({ - ...props, - datasetId: file.metadata.datasetId, - per - }); - const isOwner = - role !== TeamMemberRoleEnum.visitor && - (String(dataset.tmbId) === tmbId || role === TeamMemberRoleEnum.owner); + // file role = collection role + try { + const { isOwner, canWrite } = await authDatasetCollection({ + ...props, + collectionId: collection._id, + per + }); - const canWrite = - isOwner || - (role !== TeamMemberRoleEnum.visitor && dataset.permission === PermissionTypeEnum.public); - - if (per === 'r' && !isOwner && dataset.permission !== PermissionTypeEnum.public) { + return { + userId, + teamId, + tmbId, + file, + isOwner, + canWrite + }; + } catch (error) { return Promise.reject(DatasetErrEnum.unAuthDatasetFile); } - if (per === 'w' && !canWrite) { - return Promise.reject(DatasetErrEnum.unAuthDatasetFile); - } - if (per === 'owner' && !isOwner) { - return Promise.reject(DatasetErrEnum.unAuthDatasetFile); - } - - return { - userId, - teamId, - tmbId, - file, - isOwner, - canWrite - }; } diff --git a/packages/service/support/permission/auth/openapi.ts b/packages/service/support/permission/auth/openapi.ts index 4fd444ef9..736cecc9c 100644 --- a/packages/service/support/permission/auth/openapi.ts +++ b/packages/service/support/permission/auth/openapi.ts @@ -2,7 +2,7 @@ import { AuthResponseType } from '@fastgpt/global/support/permission/type'; import { AuthModeType } from '../type'; import { OpenApiSchema } from '@fastgpt/global/support/openapi/type'; import { parseHeaderCert } from '../controller'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { MongoOpenApi } from '../../openapi/schema'; import { OpenApiErrEnum } from '@fastgpt/global/common/error/code/openapi'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; @@ -21,7 +21,7 @@ export async function authOpenApiKeyCrud({ const result = await parseHeaderCert(props); const { tmbId, teamId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { openapi, isOwner, canWrite } = await (async () => { const openapi = await MongoOpenApi.findOne({ _id: id, teamId }); diff --git a/packages/service/support/permission/auth/outLink.ts b/packages/service/support/permission/auth/outLink.ts index 030f5d90f..f13c921f0 100644 --- a/packages/service/support/permission/auth/outLink.ts +++ b/packages/service/support/permission/auth/outLink.ts @@ -9,7 +9,7 @@ import { MongoApp } from '../../../core/app/schema'; import { OutLinkErrEnum } from '@fastgpt/global/common/error/code/outLink'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import { AppErrEnum } from '@fastgpt/global/common/error/code/app'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; /* crud outlink permission */ export async function authOutLinkCrud({ @@ -27,7 +27,7 @@ export async function authOutLinkCrud({ const result = await parseHeaderCert(props); const { tmbId, teamId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { app, outLink, isOwner, canWrite } = await (async () => { const outLink = await MongoOutLink.findOne({ _id: outLinkId, teamId }); diff --git a/packages/service/support/permission/auth/plugin.ts b/packages/service/support/permission/auth/plugin.ts index 6bbedd008..f6a53785c 100644 --- a/packages/service/support/permission/auth/plugin.ts +++ b/packages/service/support/permission/auth/plugin.ts @@ -1,7 +1,7 @@ import { AuthResponseType } from '@fastgpt/global/support/permission/type'; import { AuthModeType } from '../type'; import { parseHeaderCert } from '../controller'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { MongoPlugin } from '../../../core/plugin/schema'; import { PluginErrEnum } from '@fastgpt/global/common/error/code/plugin'; @@ -23,7 +23,7 @@ export async function authPluginCrud({ const result = await parseHeaderCert(props); const { tmbId, teamId } = result; - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const { plugin, isOwner, canWrite } = await (async () => { const plugin = await MongoPlugin.findOne({ _id: id, teamId }); @@ -73,7 +73,7 @@ export async function authPluginCanUse({ } if (source === PluginSourceEnum.personal) { - const { role } = await getTeamInfoByTmbId({ tmbId }); + const { role } = await getTmbInfoByTmbId({ tmbId }); const plugin = await MongoPlugin.findOne({ _id: pluginId, teamId }); if (!plugin) { return Promise.reject(PluginErrEnum.unExist); diff --git a/packages/service/support/permission/auth/user.ts b/packages/service/support/permission/auth/user.ts index bbcb2a1d1..e8da32072 100644 --- a/packages/service/support/permission/auth/user.ts +++ b/packages/service/support/permission/auth/user.ts @@ -3,7 +3,7 @@ import { AuthModeType } from '../type'; import { TeamItemType } from '@fastgpt/global/support/user/team/type'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { parseHeaderCert } from '../controller'; -import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { getTmbInfoByTmbId } from '../../user/team/controller'; import { UserErrEnum } from '../../../../global/common/error/code/user'; export async function authUserNotVisitor(props: AuthModeType): Promise< @@ -13,7 +13,7 @@ export async function authUserNotVisitor(props: AuthModeType): Promise< } > { const { userId, teamId, tmbId } = await parseHeaderCert(props); - const team = await getTeamInfoByTmbId({ tmbId }); + const team = await getTmbInfoByTmbId({ tmbId }); if (team.role === TeamMemberRoleEnum.visitor) { return Promise.reject(UserErrEnum.binVisitor); @@ -38,7 +38,7 @@ export async function authUserRole(props: AuthModeType): Promise< } > { const result = await parseHeaderCert(props); - const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId: result.tmbId }); + const { role: userRole, canWrite } = await getTmbInfoByTmbId({ tmbId: result.tmbId }); return { ...result, diff --git a/packages/service/support/permission/limit/dataset.ts b/packages/service/support/permission/limit/dataset.ts index a214465e2..796587e23 100644 --- a/packages/service/support/permission/limit/dataset.ts +++ b/packages/service/support/permission/limit/dataset.ts @@ -14,7 +14,7 @@ export const checkDatasetLimit = async ({ const usedSize = await getVectorCountByTeamId(teamId); if (usedSize + insertLen >= maxSize) { - return Promise.reject(`数据库容量已满，无法继续添加。可以在账号页面进行扩容。`); + return Promise.reject(`数据库容量不足，无法继续添加。可以在账号页面进行扩容。`); } return; }; diff --git a/packages/service/support/user/controller.ts b/packages/service/support/user/controller.ts index b2020d193..361a9029b 100644 --- a/packages/service/support/user/controller.ts +++ b/packages/service/support/user/controller.ts @@ -1,6 +1,6 @@ import { UserType } from '@fastgpt/global/support/user/type'; import { MongoUser } from './schema'; -import { getTeamInfoByTmbId, getUserDefaultTeam } from './team/controller'; +import { getTmbInfoByTmbId, getUserDefaultTeam } from './team/controller'; import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode'; import { UserErrEnum } from '@fastgpt/global/common/error/code/user'; @@ -21,16 +21,16 @@ export async function getUserDetail({ tmbId?: string; userId?: string; }): Promise { - const team = await (async () => { + const tmb = await (async () => { if (tmbId) { - return getTeamInfoByTmbId({ tmbId }); + return getTmbInfoByTmbId({ tmbId }); } if (userId) { return getUserDefaultTeam({ userId }); } return Promise.reject(ERROR_ENUM.unAuthorization); })(); - const user = await MongoUser.findById(team.userId); + const user = await MongoUser.findById(tmb.userId); if (!user) { return Promise.reject(ERROR_ENUM.unAuthorization); @@ -44,7 +44,7 @@ export async function getUserDetail({ timezone: user.timezone, promotionRate: user.promotionRate, openaiAccount: user.openaiAccount, - team + team: tmb }; } diff --git a/packages/service/support/user/schema.ts b/packages/service/support/user/schema.ts index a3e4d6515..15e75c491 100644 --- a/packages/service/support/user/schema.ts +++ b/packages/service/support/user/schema.ts @@ -56,9 +56,18 @@ const UserSchema = new Schema({ timezone: { type: String, default: 'Asia/Shanghai' + }, + lastLoginTmbId: { + type: Schema.Types.ObjectId } }); +try { + UserSchema.index({ createTime: -1 }); +} catch (error) { + console.log(error); +} + export const MongoUser: Model = models[userCollectionName] || model(userCollectionName, UserSchema); MongoUser.syncIndexes(); diff --git a/packages/service/support/user/team/controller.ts b/packages/service/support/user/team/controller.ts index 9efd97009..212f8db63 100644 --- a/packages/service/support/user/team/controller.ts +++ b/packages/service/support/user/team/controller.ts @@ -8,7 +8,7 @@ import { import { MongoTeamMember } from './teamMemberSchema'; import { MongoTeam } from './teamSchema'; -async function getTeam(match: Record): Promise { +async function getTeamMember(match: Record): Promise { const tmb = (await MongoTeamMember.findOne(match).populate('teamId')) as TeamMemberWithTeamSchema; if (!tmb) { @@ -31,11 +31,11 @@ async function getTeam(match: Record): Promise { }; } -export async function getTeamInfoByTmbId({ tmbId }: { tmbId: string }) { +export async function getTmbInfoByTmbId({ tmbId }: { tmbId: string }) { if (!tmbId) { return Promise.reject('tmbId or userId is required'); } - return getTeam({ + return getTeamMember({ _id: new Types.ObjectId(tmbId), status: notLeaveStatus }); @@ -45,7 +45,7 @@ export async function getUserDefaultTeam({ userId }: { userId: string }) { if (!userId) { return Promise.reject('tmbId or userId is required'); } - return getTeam({ + return getTeamMember({ userId: new Types.ObjectId(userId), defaultTeam: true }); diff --git a/packages/service/support/user/team/teamSchema.ts b/packages/service/support/user/team/teamSchema.ts index eaeb9fba9..7a58783b8 100644 --- a/packages/service/support/user/team/teamSchema.ts +++ b/packages/service/support/user/team/teamSchema.ts @@ -24,11 +24,11 @@ const TeamSchema = new Schema({ }, balance: { type: Number, - default: 2 * PRICE_SCALE + default: 0 }, maxSize: { type: Number, - default: 5 + default: 3 }, limit: { lastExportDatasetTime: { @@ -41,7 +41,7 @@ const TeamSchema = new Schema({ }); try { - TeamSchema.index({ lastDatasetBillTime: -1 }); + // TeamSchema.index({ createTime: -1 }); } catch (error) { console.log(error); } diff --git a/packages/service/support/wallet/bill/controller.ts b/packages/service/support/wallet/bill/controller.ts index 9f9f10488..2e9c9f705 100644 --- a/packages/service/support/wallet/bill/controller.ts +++ b/packages/service/support/wallet/bill/controller.ts @@ -25,15 +25,13 @@ export const createTrainingBill = async ({ { moduleName: 'wallet.moduleName.index', model: vectorModel, - inputTokens: 0, - outputTokens: 0, + charsLength: 0, amount: 0 }, { moduleName: 'wallet.moduleName.qa', model: agentModel, - inputTokens: 0, - outputTokens: 0, + charsLength: 0, amount: 0 } ], diff --git a/packages/service/support/wallet/bill/schema.ts b/packages/service/support/wallet/bill/schema.ts index 25c8b00c1..b6795d0bb 100644 --- a/packages/service/support/wallet/bill/schema.ts +++ b/packages/service/support/wallet/bill/schema.ts @@ -52,10 +52,8 @@ const BillSchema = new Schema({ }); try { - BillSchema.index({ teamId: 1 }); - BillSchema.index({ tmbId: 1 }); - BillSchema.index({ tmbId: 1, time: 1 }); - BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 }); + BillSchema.index({ teamId: 1, tmbId: 1, time: -1 }); + BillSchema.index({ time: 1 }, { expireAfterSeconds: 180 * 24 * 60 * 60 }); } catch (error) { console.log(error); } diff --git a/packages/web/common/file/read/html.ts b/packages/web/common/file/read/html.ts index 04eb911ab..403d4a5d9 100644 --- a/packages/web/common/file/read/html.ts +++ b/packages/web/common/file/read/html.ts @@ -17,5 +17,5 @@ export const readHtmlFile = async ({ uploadImgController }); - return { rawText: rawText }; + return { rawText: simpleMd }; }; diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 15a935dd6..d464f5e67 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -15,15 +15,19 @@ export const iconPaths = { 'common/courseLight': () => import('./icons/common/courseLight.svg'), 'common/customTitleLight': () => import('./icons/common/customTitleLight.svg'), 'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'), + 'common/errorFill': () => import('./icons/common/errorFill.svg'), 'common/file/move': () => import('./icons/common/file/move.svg'), + 'common/folderFill': () => import('./icons/common/folderFill.svg'), 'common/fullScreenLight': () => import('./icons/common/fullScreenLight.svg'), 'common/gitFill': () => import('./icons/common/gitFill.svg'), + 'common/gitInlight': () => import('./icons/common/gitInlight.svg'), 'common/gitLight': () => import('./icons/common/gitLight.svg'), 'common/googleFill': () => import('./icons/common/googleFill.svg'), 'common/importLight': () => import('./icons/common/importLight.svg'), 'common/inviteLight': () => import('./icons/common/inviteLight.svg'), 'common/language/en': () => import('./icons/common/language/en.svg'), 'common/language/zh': () => import('./icons/common/language/zh.svg'), + 'common/linkBlue': () => import('./icons/common/linkBlue.svg'), 'common/loading': () => import('./icons/common/loading.svg'), 'common/navbar/pluginFill': () => import('./icons/common/navbar/pluginFill.svg'), 'common/navbar/pluginLight': () => import('./icons/common/navbar/pluginLight.svg'), @@ -37,10 +41,13 @@ export const iconPaths = { 'common/retryLight': () => import('./icons/common/retryLight.svg'), 'common/rightArrowLight': () => import('./icons/common/rightArrowLight.svg'), 'common/routePushLight': () => import('./icons/common/routePushLight.svg'), + 'common/saveFill': () => import('./icons/common/saveFill.svg'), 'common/searchLight': () => import('./icons/common/searchLight.svg'), + 'common/selectLight': () => import('./icons/common/selectLight.svg'), 'common/settingLight': () => import('./icons/common/settingLight.svg'), 'common/text/t': () => import('./icons/common/text/t.svg'), 'common/tickFill': () => import('./icons/common/tickFill.svg'), + 'common/uploadFileFill': () => import('./icons/common/uploadFileFill.svg'), 'common/viewLight': () => import('./icons/common/viewLight.svg'), 'common/voiceLight': () => import('./icons/common/voiceLight.svg'), copy: () => import('./icons/copy.svg'), @@ -52,7 +59,12 @@ export const iconPaths = { 'core/app/logsLight': () => import('./icons/core/app/logsLight.svg'), 'core/app/markLight': () => import('./icons/core/app/markLight.svg'), 'core/app/questionGuide': () => import('./icons/core/app/questionGuide.svg'), - 'core/app/tts': () => import('./icons/core/app/tts.svg'), + 'core/app/simpleMode/ai': () => import('./icons/core/app/simpleMode/ai.svg'), + 'core/app/simpleMode/chat': () => import('./icons/core/app/simpleMode/chat.svg'), + 'core/app/simpleMode/dataset': () => import('./icons/core/app/simpleMode/dataset.svg'), + 'core/app/simpleMode/template': () => import('./icons/core/app/simpleMode/template.svg'), + 'core/app/simpleMode/tts': () => import('./icons/core/app/simpleMode/tts.svg'), + 'core/app/simpleMode/variable': () => import('./icons/core/app/simpleMode/variable.svg'), 'core/app/ttsFill': () => import('./icons/core/app/ttsFill.svg'), 'core/app/variable/input': () => import('./icons/core/app/variable/input.svg'), 'core/app/variable/select': () => import('./icons/core/app/variable/select.svg'), @@ -76,12 +88,15 @@ export const iconPaths = { 'core/dataset/commonDataset': () => import('./icons/core/dataset/commonDataset.svg'), 'core/dataset/datasetFill': () => import('./icons/core/dataset/datasetFill.svg'), 'core/dataset/datasetLight': () => import('./icons/core/dataset/datasetLight.svg'), - 'core/dataset/folderDataset': () => import('./icons/core/dataset/folderDataset.svg'), + 'core/dataset/fileCollection': () => import('./icons/core/dataset/fileCollection.svg'), 'core/dataset/fullTextRecall': () => import('./icons/core/dataset/fullTextRecall.svg'), + 'core/dataset/manualCollection': () => import('./icons/core/dataset/manualCollection.svg'), 'core/dataset/mixedRecall': () => import('./icons/core/dataset/mixedRecall.svg'), 'core/dataset/modeEmbedding': () => import('./icons/core/dataset/modeEmbedding.svg'), 'core/dataset/rerank': () => import('./icons/core/dataset/rerank.svg'), + 'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'), 'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'), + 'core/modules/flowLight': () => import('./icons/core/modules/flowLight.svg'), 'core/modules/previewLight': () => import('./icons/core/modules/previewLight.svg'), 'core/modules/variable': () => import('./icons/core/modules/variable.svg'), 'core/modules/welcomeText': () => import('./icons/core/modules/welcomeText.svg'), @@ -91,6 +106,15 @@ export const iconPaths = { empty: () => import('./icons/empty.svg'), export: () => import('./icons/export.svg'), 'file/csv': () => import('./icons/file/csv.svg'), + 'file/fill/csv': () => import('./icons/file/fill/csv.svg'), + 'file/fill/doc': () => import('./icons/file/fill/doc.svg'), + 'file/fill/file': () => import('./icons/file/fill/file.svg'), + 'file/fill/folder': () => import('./icons/file/fill/folder.svg'), + 'file/fill/html': () => import('./icons/file/fill/html.svg'), + 'file/fill/manual': () => import('./icons/file/fill/manual.svg'), + 'file/fill/markdown': () => import('./icons/file/fill/markdown.svg'), + 'file/fill/pdf': () => import('./icons/file/fill/pdf.svg'), + 'file/fill/txt': () => import('./icons/file/fill/txt.svg'), 'file/html': () => import('./icons/file/html.svg'), 'file/indexImport': () => import('./icons/file/indexImport.svg'), 'file/manualImport': () => import('./icons/file/manualImport.svg'), @@ -103,10 +127,14 @@ export const iconPaths = { kbTest: () => import('./icons/kbTest.svg'), menu: () => import('./icons/menu.svg'), minus: () => import('./icons/minus.svg'), + 'modal/edit': () => import('./icons/modal/edit.svg'), + 'modal/manualDataset': () => import('./icons/modal/manualDataset.svg'), + 'modal/selectSource': () => import('./icons/modal/selectSource.svg'), more: () => import('./icons/more.svg'), out: () => import('./icons/out.svg'), 'phoneTabbar/me': () => import('./icons/phoneTabbar/me.svg'), - 'phoneTabbar/more': () => import('./icons/phoneTabbar/more.svg'), + 'phoneTabbar/tool': () => import('./icons/phoneTabbar/tool.svg'), + 'phoneTabbar/toolFill': () => import('./icons/phoneTabbar/toolFill.svg'), save: () => import('./icons/save.svg'), stop: () => import('./icons/stop.svg'), 'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'), diff --git a/packages/web/components/common/Icon/icons/common/closeLight.svg b/packages/web/components/common/Icon/icons/common/closeLight.svg index 9fc9b8537..2f09ccdee 100644 --- a/packages/web/components/common/Icon/icons/common/closeLight.svg +++ b/packages/web/components/common/Icon/icons/common/closeLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/errorFill.svg b/packages/web/components/common/Icon/icons/common/errorFill.svg new file mode 100644 index 000000000..c73b125d9 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/errorFill.svg @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/folderFill.svg b/packages/web/components/common/Icon/icons/common/folderFill.svg new file mode 100644 index 000000000..7e8e49578 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/folderFill.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/gitInlight.svg b/packages/web/components/common/Icon/icons/common/gitInlight.svg new file mode 100644 index 000000000..354a96496 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/gitInlight.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/importLight.svg b/packages/web/components/common/Icon/icons/common/importLight.svg index e62871c3b..8e3f5b4d6 100644 --- a/packages/web/components/common/Icon/icons/common/importLight.svg +++ b/packages/web/components/common/Icon/icons/common/importLight.svg @@ -1 +1,6 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/linkBlue.svg b/packages/web/components/common/Icon/icons/common/linkBlue.svg new file mode 100644 index 000000000..c28f59906 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/linkBlue.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/overviewLight.svg b/packages/web/components/common/Icon/icons/common/overviewLight.svg index 3d107058e..a56fb7fd3 100644 --- a/packages/web/components/common/Icon/icons/common/overviewLight.svg +++ b/packages/web/components/common/Icon/icons/common/overviewLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/questionLight.svg b/packages/web/components/common/Icon/icons/common/questionLight.svg index f6fc6095e..f09e09270 100644 --- a/packages/web/components/common/Icon/icons/common/questionLight.svg +++ b/packages/web/components/common/Icon/icons/common/questionLight.svg @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/saveFill.svg b/packages/web/components/common/Icon/icons/common/saveFill.svg new file mode 100644 index 000000000..12e305f89 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/saveFill.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/selectLight.svg b/packages/web/components/common/Icon/icons/common/selectLight.svg new file mode 100644 index 000000000..f8ed98e79 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/selectLight.svg @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/settingLight.svg b/packages/web/components/common/Icon/icons/common/settingLight.svg index a4de19777..c74dd2022 100644 --- a/packages/web/components/common/Icon/icons/common/settingLight.svg +++ b/packages/web/components/common/Icon/icons/common/settingLight.svg @@ -1 +1,6 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/uploadFileFill.svg b/packages/web/components/common/Icon/icons/common/uploadFileFill.svg new file mode 100644 index 000000000..f10644c29 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/uploadFileFill.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/viewLight.svg b/packages/web/components/common/Icon/icons/common/viewLight.svg index 29f97bbc0..2ed165b24 100644 --- a/packages/web/components/common/Icon/icons/common/viewLight.svg +++ b/packages/web/components/common/Icon/icons/common/viewLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/logsLight.svg b/packages/web/components/common/Icon/icons/core/app/logsLight.svg index bcb7caaa1..00496ba98 100644 --- a/packages/web/components/common/Icon/icons/core/app/logsLight.svg +++ b/packages/web/components/common/Icon/icons/core/app/logsLight.svg @@ -1,14 +1,4 @@ - - - - - + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/questionGuide.svg b/packages/web/components/common/Icon/icons/core/app/questionGuide.svg index 4903f93dc..fba55289b 100644 --- a/packages/web/components/common/Icon/icons/core/app/questionGuide.svg +++ b/packages/web/components/common/Icon/icons/core/app/questionGuide.svg @@ -1,5 +1,5 @@ - + + d="M1.66675 7.97028C1.66675 6.0661 1.66675 5.11401 2.03733 4.38671C2.3633 3.74695 2.88343 3.22682 3.52318 2.90085C4.25048 2.53027 5.20257 2.53027 7.10675 2.53027H12.8934C14.7976 2.53027 15.7497 2.53027 16.477 2.90085C17.1167 3.22682 17.6369 3.74695 17.9628 4.38671C18.3334 5.11401 18.3334 6.06609 18.3334 7.97027V10.9413C18.3334 12.8455 18.3334 13.7976 17.9628 14.5249C17.6369 15.1647 17.1167 15.6848 16.477 16.0108C15.7497 16.3814 14.7976 16.3814 12.8934 16.3814H10.0744L7.42003 18.5835C6.99384 18.937 6.34598 18.7611 6.15717 18.2405L5.4777 16.3672C4.58619 16.3392 4.0053 16.2564 3.52318 16.0108C2.88343 15.6848 2.3633 15.1647 2.03733 14.5249C1.66675 13.7976 1.66675 12.8455 1.66675 10.9414V7.97028ZM6.38847 9.48176C6.98678 9.48176 7.4718 8.99673 7.4718 8.39843C7.4718 7.80012 6.98678 7.31509 6.38847 7.31509C5.79016 7.31509 5.30514 7.80012 5.30514 8.39843C5.30514 8.99673 5.79016 9.48176 6.38847 9.48176ZM14.695 8.39843C14.695 8.99673 14.21 9.48176 13.6117 9.48176C13.0134 9.48176 12.5284 8.99673 12.5284 8.39843C12.5284 7.80012 13.0134 7.31509 13.6117 7.31509C14.21 7.31509 14.695 7.80012 14.695 8.39843ZM12.7802 11.6057C13.0281 11.3091 12.9887 10.8677 12.692 10.6197C12.3954 10.3718 11.954 10.4112 11.706 10.7078C11.3434 11.1416 10.7759 11.4343 10.0383 11.4343C9.29561 11.4343 8.59499 11.0648 8.29232 10.7057C8.04317 10.4101 7.60156 10.3724 7.30595 10.6216C7.01034 10.8707 6.97268 11.3123 7.22183 11.6079C7.80834 12.3038 8.91134 12.8343 10.0383 12.8343C11.1704 12.8343 12.1389 12.3729 12.7802 11.6057Z" + fill="#00A9A6" /> \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/ai.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/ai.svg new file mode 100644 index 000000000..f06e1a3cc --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/ai.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/chat.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/chat.svg new file mode 100644 index 000000000..0a14859f0 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/chat.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/dataset.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/dataset.svg new file mode 100644 index 000000000..353761b43 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/dataset.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/template.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/template.svg new file mode 100644 index 000000000..d47abbdbd --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/template.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg new file mode 100644 index 000000000..4b9be223c --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/variable.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/variable.svg new file mode 100644 index 000000000..a4b5a181d --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/app/simpleMode/variable.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/tts.svg b/packages/web/components/common/Icon/icons/core/app/tts.svg deleted file mode 100644 index 2679946e8..000000000 --- a/packages/web/components/common/Icon/icons/core/app/tts.svg +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg index dc84253cb..0e81dfad8 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg @@ -1,4 +1,4 @@ - + + d="M18.0527 6.60938C17.6133 5.56055 16.9805 4.61914 16.1758 3.81055C15.3711 3.00391 14.4316 2.36914 13.3828 1.92969C12.3105 1.47852 11.1738 1.25 9.99999 1.25H9.96093C8.77928 1.25586 7.63671 1.49023 6.56053 1.95117C5.52147 2.39648 4.58983 3.0293 3.79296 3.83594C2.99608 4.64258 2.37108 5.58008 1.93749 6.625C1.49022 7.70703 1.26366 8.85742 1.26952 10.041C1.27538 11.3965 1.5996 12.7422 2.20507 13.9453V16.9141C2.20507 17.4102 2.60741 17.8125 3.10155 17.8125H6.06639C7.26757 18.4199 8.60936 18.7441 9.96288 18.75H10.0039C11.1719 18.75 12.3027 18.5234 13.3691 18.0801C14.4121 17.6445 15.3496 17.0195 16.1523 16.2207C16.957 15.4219 17.5898 14.4883 18.0332 13.4473C18.4922 12.3691 18.7265 11.2227 18.7324 10.0391C18.7363 8.84961 18.5058 7.69531 18.0527 6.60938ZM6.10155 10.9375C5.58593 10.9375 5.166 10.5176 5.166 10C5.166 9.48242 5.58593 9.0625 6.10155 9.0625C6.61718 9.0625 7.0371 9.48242 7.0371 10C7.0371 10.5176 6.61913 10.9375 6.10155 10.9375ZM9.99999 10.9375C9.48436 10.9375 9.06444 10.5176 9.06444 10C9.06444 9.48242 9.48436 9.0625 9.99999 9.0625C10.5156 9.0625 10.9355 9.48242 10.9355 10C10.9355 10.5176 10.5156 10.9375 9.99999 10.9375ZM13.8984 10.9375C13.3828 10.9375 12.9629 10.5176 12.9629 10C12.9629 9.48242 13.3828 9.0625 13.8984 9.0625C14.414 9.0625 14.834 9.48242 14.834 10C14.834 10.5176 14.414 10.9375 13.8984 10.9375Z" /> \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/fileCollection.svg b/packages/web/components/common/Icon/icons/core/dataset/fileCollection.svg new file mode 100644 index 000000000..8d6950d62 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/fileCollection.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/folderDataset.svg b/packages/web/components/common/Icon/icons/core/dataset/folderDataset.svg deleted file mode 100644 index 602393396..000000000 --- a/packages/web/components/common/Icon/icons/core/dataset/folderDataset.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/manualCollection.svg b/packages/web/components/common/Icon/icons/core/dataset/manualCollection.svg new file mode 100644 index 000000000..98f100724 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/manualCollection.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/tableCollection.svg b/packages/web/components/common/Icon/icons/core/dataset/tableCollection.svg new file mode 100644 index 000000000..7fa919420 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/tableCollection.svg @@ -0,0 +1,11 @@ + + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/modules/flowLight.svg b/packages/web/components/common/Icon/icons/core/modules/flowLight.svg new file mode 100644 index 000000000..3394ba30d --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/modules/flowLight.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/edit.svg b/packages/web/components/common/Icon/icons/edit.svg index f2a5c97ea..1fff0787e 100644 --- a/packages/web/components/common/Icon/icons/edit.svg +++ b/packages/web/components/common/Icon/icons/edit.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/export.svg b/packages/web/components/common/Icon/icons/export.svg index cf4cb222e..5570c2de7 100644 --- a/packages/web/components/common/Icon/icons/export.svg +++ b/packages/web/components/common/Icon/icons/export.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/csv.svg b/packages/web/components/common/Icon/icons/file/fill/csv.svg similarity index 100% rename from projects/app/public/imgs/files/csv.svg rename to packages/web/components/common/Icon/icons/file/fill/csv.svg diff --git a/projects/app/public/imgs/files/doc.svg b/packages/web/components/common/Icon/icons/file/fill/doc.svg similarity index 100% rename from projects/app/public/imgs/files/doc.svg rename to packages/web/components/common/Icon/icons/file/fill/doc.svg diff --git a/projects/app/public/imgs/files/file.svg b/packages/web/components/common/Icon/icons/file/fill/file.svg similarity index 100% rename from projects/app/public/imgs/files/file.svg rename to packages/web/components/common/Icon/icons/file/fill/file.svg diff --git a/packages/web/components/common/Icon/icons/file/fill/folder.svg b/packages/web/components/common/Icon/icons/file/fill/folder.svg new file mode 100644 index 000000000..7e8e49578 --- /dev/null +++ b/packages/web/components/common/Icon/icons/file/fill/folder.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/html.svg b/packages/web/components/common/Icon/icons/file/fill/html.svg similarity index 100% rename from projects/app/public/imgs/files/html.svg rename to packages/web/components/common/Icon/icons/file/fill/html.svg diff --git a/packages/web/components/common/Icon/icons/file/fill/manual.svg b/packages/web/components/common/Icon/icons/file/fill/manual.svg new file mode 100644 index 000000000..e5a941b52 --- /dev/null +++ b/packages/web/components/common/Icon/icons/file/fill/manual.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/markdown.svg b/packages/web/components/common/Icon/icons/file/fill/markdown.svg similarity index 100% rename from projects/app/public/imgs/files/markdown.svg rename to packages/web/components/common/Icon/icons/file/fill/markdown.svg diff --git a/packages/web/components/common/Icon/icons/file/fill/pdf.svg b/packages/web/components/common/Icon/icons/file/fill/pdf.svg new file mode 100644 index 000000000..80bb8ac0f --- /dev/null +++ b/packages/web/components/common/Icon/icons/file/fill/pdf.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/txt.svg b/packages/web/components/common/Icon/icons/file/fill/txt.svg similarity index 100% rename from projects/app/public/imgs/files/txt.svg rename to packages/web/components/common/Icon/icons/file/fill/txt.svg diff --git a/packages/web/components/common/Icon/icons/file/markdown.svg b/packages/web/components/common/Icon/icons/file/markdown.svg index bc4b63601..bc6c862f1 100644 --- a/packages/web/components/common/Icon/icons/file/markdown.svg +++ b/packages/web/components/common/Icon/icons/file/markdown.svg @@ -1 +1,11 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/file/pdf.svg b/packages/web/components/common/Icon/icons/file/pdf.svg index 3767414c6..c89b1326f 100644 --- a/packages/web/components/common/Icon/icons/file/pdf.svg +++ b/packages/web/components/common/Icon/icons/file/pdf.svg @@ -1 +1,12 @@ - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/kbTest.svg b/packages/web/components/common/Icon/icons/kbTest.svg index e4e4fa901..f57202895 100644 --- a/packages/web/components/common/Icon/icons/kbTest.svg +++ b/packages/web/components/common/Icon/icons/kbTest.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/modal/edit.svg b/packages/web/components/common/Icon/icons/modal/edit.svg new file mode 100644 index 000000000..284100b4f --- /dev/null +++ b/packages/web/components/common/Icon/icons/modal/edit.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/modal/manualDataset.svg b/packages/web/components/common/Icon/icons/modal/manualDataset.svg new file mode 100644 index 000000000..5f72aafd5 --- /dev/null +++ b/packages/web/components/common/Icon/icons/modal/manualDataset.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/modal/selectSource.svg b/packages/web/components/common/Icon/icons/modal/selectSource.svg new file mode 100644 index 000000000..43d81ac75 --- /dev/null +++ b/packages/web/components/common/Icon/icons/modal/selectSource.svg @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/phoneTabbar/more.svg b/packages/web/components/common/Icon/icons/phoneTabbar/more.svg deleted file mode 100644 index 19608a0f9..000000000 --- a/packages/web/components/common/Icon/icons/phoneTabbar/more.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg b/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg new file mode 100644 index 000000000..b77170d86 --- /dev/null +++ b/packages/web/components/common/Icon/icons/phoneTabbar/tool.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/phoneTabbar/toolFill.svg b/packages/web/components/common/Icon/icons/phoneTabbar/toolFill.svg new file mode 100644 index 000000000..e7cfcad60 --- /dev/null +++ b/packages/web/components/common/Icon/icons/phoneTabbar/toolFill.svg @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg b/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg index 4ab652201..f9c852323 100644 --- a/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg +++ b/packages/web/components/common/Icon/icons/support/outlink/shareLight.svg @@ -1,6 +1,4 @@ - - + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/permission/privateLight.svg b/packages/web/components/common/Icon/icons/support/permission/privateLight.svg index a72b6f44a..12be6e9de 100644 --- a/packages/web/components/common/Icon/icons/support/permission/privateLight.svg +++ b/packages/web/components/common/Icon/icons/support/permission/privateLight.svg @@ -1,8 +1,4 @@ - - - + + \ No newline at end of file diff --git a/packages/web/components/common/MyModal/index.tsx b/packages/web/components/common/MyModal/index.tsx new file mode 100644 index 000000000..1549965f3 --- /dev/null +++ b/packages/web/components/common/MyModal/index.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import { + Modal, + ModalOverlay, + ModalContent, + ModalHeader, + ModalCloseButton, + ModalContentProps, + Box, + Image +} from '@chakra-ui/react'; +import MyIcon from '../Icon'; + +export interface MyModalProps extends ModalContentProps { + iconSrc?: string; + title?: any; + isCentered?: boolean; + isOpen: boolean; + onClose?: () => void; + isPc?: boolean; +} + +const CustomModal = ({ + isOpen, + onClose, + iconSrc, + title, + children, + isCentered, + w = 'auto', + maxW = ['90vw', '600px'], + ...props +}: MyModalProps) => { + return ( + onClose && onClose()} + autoFocus={false} + isCentered={isCentered} + > + + + {!title && onClose && } + {!!title && ( + + {iconSrc && ( + <> + {iconSrc.startsWith('/') ? ( + + ) : ( + + )} + + )} + {title} + + {onClose && ( + + )} + + )} + + + {children} + + + + ); +}; + +export default CustomModal; diff --git a/packages/web/components/common/MyTooltip/index.tsx b/packages/web/components/common/MyTooltip/index.tsx new file mode 100644 index 000000000..90f0ee128 --- /dev/null +++ b/packages/web/components/common/MyTooltip/index.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import { Tooltip, TooltipProps } from '@chakra-ui/react'; + +interface Props extends TooltipProps { + forceShow?: boolean; +} + +const MyTooltip = ({ children, shouldWrapChildren = true, ...props }: Props) => { + return ( + + {children} + + ); +}; + +export default MyTooltip; diff --git a/packages/web/components/common/Radio/LeftRadio.tsx b/packages/web/components/common/Radio/LeftRadio.tsx new file mode 100644 index 000000000..6d97ba84c --- /dev/null +++ b/packages/web/components/common/Radio/LeftRadio.tsx @@ -0,0 +1,112 @@ +import React from 'react'; +import { Box, Flex, useTheme, Grid, type GridProps } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import MyTooltip from '../MyTooltip'; + +// @ts-ignore +interface Props extends GridProps { + list: { + title: string; + desc?: string; + value: any; + children?: React.ReactNode; + tooltip?: string; + }[]; + align?: 'flex-top' | 'center'; + value: any; + defaultBg?: string; + activeBg?: string; + onChange: (e: any) => void; +} + +const LeftRadio = ({ + list, + value, + align = 'flex-top', + px = 3, + py = 4, + defaultBg = 'myGray.50', + activeBg = 'primary.50', + onChange, + ...props +}: Props) => { + const { t } = useTranslation(); + const theme = useTheme(); + + return ( + + {list.map((item) => ( + + onChange(item.value)} + > + + + + + + + + {typeof item.title === 'string' ? t(item.title) : item.title} + + {!!item.desc && ( + + {t(item.desc)} + + )} + {item?.children} + + + + ))} + + ); +}; + +export default LeftRadio; diff --git a/packages/web/components/common/Tabs/RowTabs.tsx b/packages/web/components/common/Tabs/RowTabs.tsx new file mode 100644 index 000000000..339e534d2 --- /dev/null +++ b/packages/web/components/common/Tabs/RowTabs.tsx @@ -0,0 +1,54 @@ +import React from 'react'; +import { Flex, Box } from '@chakra-ui/react'; +import MyIcon from '../Icon'; + +type Props = { + list: { + icon?: string; + label: string | React.ReactNode; + value: string; + }[]; + value: string; + onChange: (e: string) => void; +}; + +const RowTabs = ({ list, value, onChange }: Props) => { + return ( + + {list.map((item) => ( + onChange(item.value) + })} + > + {item.icon && } + {item.label} + + ))} + + ); +}; + +export default RowTabs; diff --git a/packages/web/components/common/Textarea/PromptEditor/Editor.tsx b/packages/web/components/common/Textarea/PromptEditor/Editor.tsx new file mode 100644 index 000000000..3b5ee9063 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/Editor.tsx @@ -0,0 +1,135 @@ +import { useState, useRef } from 'react'; +import { LexicalComposer } from '@lexical/react/LexicalComposer'; +import { PlainTextPlugin } from '@lexical/react/LexicalPlainTextPlugin'; +import { ContentEditable } from '@lexical/react/LexicalContentEditable'; +import { HistoryPlugin } from '@lexical/react/LexicalHistoryPlugin'; +import { OnChangePlugin } from '@lexical/react/LexicalOnChangePlugin'; +import LexicalErrorBoundary from '@lexical/react/LexicalErrorBoundary'; +import VariablePickerPlugin from './plugins/VariablePickerPlugin'; +import { Box } from '@chakra-ui/react'; +import styles from './index.module.scss'; +import VariablePlugin from './plugins/VariablePlugin'; +import { VariableNode } from './plugins/VariablePlugin/node'; +import { EditorState, LexicalEditor } from 'lexical'; +import { textToEditorState } from './utils'; +import OnBlurPlugin from './plugins/OnBlurPlugin'; +import MyIcon from '../../Icon'; +import { PickerMenuItemType } from './type.d'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; + +export default function Editor({ + h = 200, + showResize = true, + showOpenModal = true, + onOpenModal, + variables, + onChange, + onBlur, + defaultValue, + placeholder = '' +}: { + h?: number; + showResize?: boolean; + showOpenModal?: boolean; + onOpenModal?: () => void; + variables: PickerMenuItemType[]; + onChange?: (editorState: EditorState) => void; + onBlur?: (editor: LexicalEditor) => void; + defaultValue?: string; + placeholder?: string; +}) { + const key = useRef(getNanoid(6)); + const [height, setHeight] = useState(h); + const [initialConfig, setInitialConfig] = useState({ + namespace: 'promptEditor', + nodes: [VariableNode], + editorState: textToEditorState(defaultValue), + onError: (error: Error) => { + throw error; + } + }); + const initialY = useRef(0); + + const handleMouseDown = (e: React.MouseEvent) => { + initialY.current = e.clientY; + + const handleMouseMove = (e: MouseEvent) => { + const deltaY = e.clientY - initialY.current; + setHeight((prevHeight) => (prevHeight + deltaY < h * 0.5 ? h * 0.5 : prevHeight + deltaY)); + initialY.current = e.clientY; + }; + + const handleMouseUp = () => { + document.removeEventListener('mousemove', handleMouseMove); + document.removeEventListener('mouseup', handleMouseUp); + }; + + document.addEventListener('mousemove', handleMouseMove); + document.addEventListener('mouseup', handleMouseUp); + }; + + return ( + + + } + placeholder={ + + + {placeholder} + + + } + ErrorBoundary={LexicalErrorBoundary} + /> + + onChange?.(e)} /> + + + + + {showResize && ( + + + + )} + {showOpenModal && ( + + + + )} + + ); +} diff --git a/packages/web/components/common/Textarea/PromptEditor/index.module.scss b/packages/web/components/common/Textarea/PromptEditor/index.module.scss new file mode 100644 index 000000000..7cba8bdc5 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/index.module.scss @@ -0,0 +1,24 @@ +.contentEditable { + position: relative; + height: 100%; + width: 100%; + border: 1px solid var(--chakra-colors-borderColor-base); + border-radius: var(--chakra-radii-md); + padding: 8px 12px; + background: var(--chakra-colors-gray-50); + font-size: 13px; + overflow-y: auto; +} + +.contentEditable:focus { + outline: none; + border: 1px solid; + border-color: var(--chakra-colors-primary-600); + background: #fff; + box-shadow: 0px 0px 0px 2.4px rgba(51, 112, 255, 0.15); +} + +.variable { + color: var(--chakra-colors-primary-600); + padding: 0 2px; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/index.tsx b/packages/web/components/common/Textarea/PromptEditor/index.tsx new file mode 100644 index 000000000..a13d297bd --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/index.tsx @@ -0,0 +1,83 @@ +import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react'; +import React from 'react'; +import { editorStateToText } from './utils'; +import Editor from './Editor'; +import MyModal from '../../MyModal'; +import { useTranslation } from 'next-i18next'; +import { $getRoot, EditorState, type LexicalEditor } from 'lexical'; +import { PickerMenuItemType } from './type.d'; +import { useCallback, useTransition } from 'react'; + +const PromptEditor = ({ + showOpenModal = true, + variables = [], + defaultValue, + onChange, + onBlur, + h, + placeholder, + title +}: { + showOpenModal?: boolean; + variables?: PickerMenuItemType[]; + defaultValue?: string; + onChange?: (text: string) => void; + onBlur?: (text: string) => void; + h?: number; + placeholder?: string; + title?: string; +}) => { + const { isOpen, onOpen, onClose } = useDisclosure(); + + const [, startSts] = useTransition(); + + const { t } = useTranslation(); + + const onChangeInput = useCallback((editorState: EditorState) => { + const text = editorState.read(() => $getRoot().getTextContent()); + const formatValue = text.replaceAll('\n\n', '\n').replaceAll('}}{{', '}} {{'); + onChange?.(formatValue); + }, []); + const onBlurInput = useCallback((editor: LexicalEditor) => { + startSts(() => { + const text = editorStateToText(editor).replaceAll('\n\n', '\n').replaceAll('}}{{', '}} {{'); + onBlur?.(text); + }); + }, []); + + return ( + <> + + + + + + + + + + + ); +}; +export default React.memo(PromptEditor); diff --git a/packages/web/components/common/Textarea/PromptEditor/modules/ComfirmVar/index.tsx b/packages/web/components/common/Textarea/PromptEditor/modules/ComfirmVar/index.tsx new file mode 100644 index 000000000..9b85f33e0 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/modules/ComfirmVar/index.tsx @@ -0,0 +1,96 @@ +import { Box, Button, Image } from '@chakra-ui/react'; + +export default function ComfirmVar({ + newVariables, + onCancel, + onConfirm +}: { + newVariables: string[]; + onCancel: () => void; + onConfirm: () => void; +}) { + return ( + <> + + + + + + + 引用了未定义的变量，是否自动添加？ + + + {newVariables.map((item, index) => ( + + + {`{{`} + {item} + {`}}`} + + + ))} + + + + + + + + + + ); +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/OnBlurPlugin/index.tsx b/packages/web/components/common/Textarea/PromptEditor/plugins/OnBlurPlugin/index.tsx new file mode 100644 index 000000000..e5bcc2594 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/OnBlurPlugin/index.tsx @@ -0,0 +1,24 @@ +import { useEffect } from 'react'; +import { BLUR_COMMAND, COMMAND_PRIORITY_EDITOR, LexicalEditor } from 'lexical'; +import { mergeRegister } from '@lexical/utils'; +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext'; + +export default function OnBlurPlugin({ onBlur }: { onBlur?: (editor: LexicalEditor) => void }) { + const [editor] = useLexicalComposerContext(); + + useEffect(() => { + return mergeRegister( + editor.registerCommand( + BLUR_COMMAND, + () => { + if (onBlur) onBlur(editor); + + return true; + }, + COMMAND_PRIORITY_EDITOR + ) + ); + }, [editor, onBlur]); + + return null; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx new file mode 100644 index 000000000..8ab9ce975 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePickerPlugin/index.tsx @@ -0,0 +1,143 @@ +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext'; +import { LexicalTypeaheadMenuPlugin } from '@lexical/react/LexicalTypeaheadMenuPlugin'; +import { $createTextNode, $getSelection, $isRangeSelection, TextNode } from 'lexical'; +import * as React from 'react'; +import { useCallback, useMemo, useState } from 'react'; +import * as ReactDOM from 'react-dom'; +import { VariableInputEnum } from '@fastgpt/global/core/module/constants'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '../../../../Icon'; +import { Box, Flex } from '@chakra-ui/react'; +import { useBasicTypeaheadTriggerMatch } from '../../utils'; +import { PickerMenuItemType } from '../../type.d'; + +export default function VariablePickerPlugin({ variables }: { variables: PickerMenuItemType[] }) { + const [editor] = useLexicalComposerContext(); + const [queryString, setQueryString] = useState(null); + const { t } = useTranslation(); + + const checkForTriggerMatch = useBasicTypeaheadTriggerMatch('{', { + minLength: 0 + }); + + const VariableTypeList = useMemo( + () => [ + { + title: t('core.module.variable.input type'), + icon: 'core/app/variable/input', + value: VariableInputEnum.input + }, + { + title: t('core.module.variable.textarea type'), + icon: 'core/app/variable/textarea', + value: VariableInputEnum.textarea + }, + { + title: t('core.module.variable.select type'), + icon: 'core/app/variable/select', + value: VariableInputEnum.select + } + ], + [t] + ); + + // const options: Array = useMemo(() => { + // // const newVariableOption = { + // // label: t('common.Add New') + "变量", + // // key: 'new_variable', + // // icon: 'core/modules/variable' + // // }; + // return [ + // ...variables.map((item) => ({ + // ...item, + // icon: VariableTypeList.find((type) => type.value === item.type)?.icon + // })) + // // newVariableOption + // ]; + // }, [VariableTypeList, t, variables]); + + const onSelectOption = useCallback( + (selectedOption: any, nodeToRemove: TextNode | null, closeMenu: () => void) => { + editor.update(() => { + const selection = $getSelection(); + if (!$isRangeSelection(selection) || selectedOption == null) { + return; + } + if (nodeToRemove) { + nodeToRemove.remove(); + } + selection.insertNodes([$createTextNode(`{{${selectedOption.key}}}`)]); + closeMenu(); + }); + }, + [editor] + ); + + return ( + { + if (anchorElementRef.current == null) { + return null; + } + return anchorElementRef.current && variables.length + ? ReactDOM.createPortal( + + {variables.map((item, index) => ( + { + setHighlightedIndex(index); + selectOptionAndCleanUp(item); + }} + onMouseEnter={() => { + setHighlightedIndex(index); + }} + > + + {`${item.key}(${item.label})`} + + ))} + , + anchorElementRef.current + ) + : null; + }} + /> + ); +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/index.tsx b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/index.tsx new file mode 100644 index 000000000..3927c7bc7 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/index.tsx @@ -0,0 +1,49 @@ +import { TextNode } from 'lexical'; +import { mergeRegister } from '@lexical/utils'; +import { useLexicalComposerContext } from '@lexical/react/LexicalComposerContext'; +import { useCallback, useEffect, useMemo } from 'react'; + +import { getHashtagRegexString } from './utils'; +import { registerLexicalTextEntity } from '../../utils'; +import { $createVariableNode, VariableNode } from './node'; +import { PickerMenuItemType } from '../../type'; + +const REGEX = new RegExp(getHashtagRegexString(), 'i'); + +export default function VariablePlugin({ variables }: { variables: PickerMenuItemType[] }) { + const [editor] = useLexicalComposerContext(); + useEffect(() => { + if (!editor.hasNodes([VariableNode])) + throw new Error('VariablePlugin: VariableNode not registered on editor'); + }, [editor]); + + const variableKeys: Array = useMemo(() => { + return variables.map((item) => item.key); + }, [variables]); + + const createVariableNode = useCallback((textNode: TextNode): VariableNode => { + return $createVariableNode(textNode.getTextContent()); + }, []); + + const getVariableMatch = useCallback((text: string) => { + const matches = REGEX.exec(text); + if (!matches) return null; + + if (variableKeys.indexOf(matches[3]) === -1) return null; + const hashtagLength = matches[3].length + 4; + const startOffset = matches.index; + const endOffset = startOffset + hashtagLength; + return { + end: endOffset, + start: startOffset + }; + }, []); + + useEffect(() => { + mergeRegister( + ...registerLexicalTextEntity(editor, getVariableMatch, VariableNode, createVariableNode) + ); + }, [createVariableNode, editor, getVariableMatch]); + + return null; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/node.ts b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/node.ts new file mode 100644 index 000000000..536f25b90 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/node.ts @@ -0,0 +1,49 @@ +import type { NodeKey, EditorConfig, LexicalNode, SerializedTextNode } from 'lexical'; + +import { TextNode, $applyNodeReplacement } from 'lexical'; +import { addClassNamesToElement } from '@lexical/utils'; +import styles from '../../index.module.scss'; + +export class VariableNode extends TextNode { + static getType(): string { + return 'variable'; + } + + static clone(node: VariableNode): VariableNode { + return new VariableNode(node.__text, node.__key); + } + + constructor(text: string, key?: NodeKey) { + super(text, key); + } + + createDOM(config: EditorConfig): HTMLElement { + const element = super.createDOM(config); + addClassNamesToElement(element, styles.variable); + return element; + } + + static importJSON(serializedNode: SerializedTextNode): TextNode { + const node = $createVariableNode(serializedNode.text); + node.setFormat(serializedNode.format); + node.setDetail(serializedNode.detail); + node.setMode(serializedNode.mode); + node.setStyle(serializedNode.style); + return node; + } + + exportJSON(): SerializedTextNode { + return { + ...super.exportJSON(), + type: 'variable' + }; + } +} + +export function $createVariableNode(text: string): VariableNode { + return $applyNodeReplacement(new VariableNode(text)); +} + +export function $isVariableNode(node: LexicalNode | null | undefined): node is VariableNode { + return node instanceof VariableNode; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/utils.ts b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/utils.ts new file mode 100644 index 000000000..5ec8230ad --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/plugins/VariablePlugin/utils.ts @@ -0,0 +1,223 @@ +function getHashtagRegexStringChars(): Readonly<{ + alpha: string; + alphanumeric: string; + leftChars: string; + rightChars: string; +}> { + // Latin accented characters + // Excludes 0xd7 from the range + // (the multiplication sign, confusable with "x"). + // Also excludes 0xf7, the division sign + const latinAccents = + '\xC0-\xD6' + + '\xD8-\xF6' + + '\xF8-\xFF' + + '\u0100-\u024F' + + '\u0253-\u0254' + + '\u0256-\u0257' + + '\u0259' + + '\u025B' + + '\u0263' + + '\u0268' + + '\u026F' + + '\u0272' + + '\u0289' + + '\u028B' + + '\u02BB' + + '\u0300-\u036F' + + '\u1E00-\u1EFF'; + + // Cyrillic (Russian, Ukrainian, etc.) + const nonLatinChars = + '\u0400-\u04FF' + // Cyrillic + '\u0500-\u0527' + // Cyrillic Supplement + '\u2DE0-\u2DFF' + // Cyrillic Extended A + '\uA640-\uA69F' + // Cyrillic Extended B + '\u0591-\u05BF' + // Hebrew + '\u05C1-\u05C2' + + '\u05C4-\u05C5' + + '\u05C7' + + '\u05D0-\u05EA' + + '\u05F0-\u05F4' + + '\uFB12-\uFB28' + // Hebrew Presentation Forms + '\uFB2A-\uFB36' + + '\uFB38-\uFB3C' + + '\uFB3E' + + '\uFB40-\uFB41' + + '\uFB43-\uFB44' + + '\uFB46-\uFB4F' + + '\u0610-\u061A' + // Arabic + '\u0620-\u065F' + + '\u066E-\u06D3' + + '\u06D5-\u06DC' + + '\u06DE-\u06E8' + + '\u06EA-\u06EF' + + '\u06FA-\u06FC' + + '\u06FF' + + '\u0750-\u077F' + // Arabic Supplement + '\u08A0' + // Arabic Extended A + '\u08A2-\u08AC' + + '\u08E4-\u08FE' + + '\uFB50-\uFBB1' + // Arabic Pres. Forms A + '\uFBD3-\uFD3D' + + '\uFD50-\uFD8F' + + '\uFD92-\uFDC7' + + '\uFDF0-\uFDFB' + + '\uFE70-\uFE74' + // Arabic Pres. Forms B + '\uFE76-\uFEFC' + + '\u200C-\u200C' + // Zero-Width Non-Joiner + '\u0E01-\u0E3A' + // Thai + '\u0E40-\u0E4E' + // Hangul (Korean) + '\u1100-\u11FF' + // Hangul Jamo + '\u3130-\u3185' + // Hangul Compatibility Jamo + '\uA960-\uA97F' + // Hangul Jamo Extended-A + '\uAC00-\uD7AF' + // Hangul Syllables + '\uD7B0-\uD7FF' + // Hangul Jamo Extended-B + '\uFFA1-\uFFDC'; // Half-width Hangul + + const charCode = String.fromCharCode; + + const cjkChars = + '\u30A1-\u30FA\u30FC-\u30FE' + // Katakana (full-width) + '\uFF66-\uFF9F' + // Katakana (half-width) + '\uFF10-\uFF19\uFF21-\uFF3A' + + '\uFF41-\uFF5A' + // Latin (full-width) + '\u3041-\u3096\u3099-\u309E' + // Hiragana + '\u3400-\u4DBF' + // Kanji (CJK Extension A) + `\u4E00-\u9FFF${ + // Kanji (Unified) + // Disabled as it breaks the Regex. + // charCode(0x20000) + '-' + charCode(0x2A6DF) + // Kanji (CJK Extension B) + charCode(0x2a700) + }-${ + charCode(0x2b73f) // Kanji (CJK Extension C) + }${charCode(0x2b740)}-${ + charCode(0x2b81f) // Kanji (CJK Extension D) + }${charCode(0x2f800)}-${charCode(0x2fa1f)}\u3003\u3005\u303B`; // Kanji (CJK supplement) + + const otherChars = latinAccents + nonLatinChars + cjkChars; + // equivalent of \p{L} + + const unicodeLetters = + '\u0041-\u005A\u0061-\u007A\u00AA\u00B5\u00BA\u00C0-\u00D6\u00D8-\u00F6' + + '\u00F8-\u0241\u0250-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EE\u037A\u0386' + + '\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03CE\u03D0-\u03F5\u03F7-\u0481' + + '\u048A-\u04CE\u04D0-\u04F9\u0500-\u050F\u0531-\u0556\u0559\u0561-\u0587' + + '\u05D0-\u05EA\u05F0-\u05F2\u0621-\u063A\u0640-\u064A\u066E-\u066F' + + '\u0671-\u06D3\u06D5\u06E5-\u06E6\u06EE-\u06EF\u06FA-\u06FC\u06FF\u0710' + + '\u0712-\u072F\u074D-\u076D\u0780-\u07A5\u07B1\u0904-\u0939\u093D\u0950' + + '\u0958-\u0961\u097D\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0' + + '\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC-\u09DD\u09DF-\u09E1\u09F0-\u09F1' + + '\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33' + + '\u0A35-\u0A36\u0A38-\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D' + + '\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABD' + + '\u0AD0\u0AE0-\u0AE1\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30' + + '\u0B32-\u0B33\u0B35-\u0B39\u0B3D\u0B5C-\u0B5D\u0B5F-\u0B61\u0B71\u0B83' + + '\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F' + + '\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0C05-\u0C0C\u0C0E-\u0C10' + + '\u0C12-\u0C28\u0C2A-\u0C33\u0C35-\u0C39\u0C60-\u0C61\u0C85-\u0C8C' + + '\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE' + + '\u0CE0-\u0CE1\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D28\u0D2A-\u0D39' + + '\u0D60-\u0D61\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6' + + '\u0E01-\u0E30\u0E32-\u0E33\u0E40-\u0E46\u0E81-\u0E82\u0E84\u0E87-\u0E88' + + '\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7' + + '\u0EAA-\u0EAB\u0EAD-\u0EB0\u0EB2-\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6' + + '\u0EDC-\u0EDD\u0F00\u0F40-\u0F47\u0F49-\u0F6A\u0F88-\u0F8B\u1000-\u1021' + + '\u1023-\u1027\u1029-\u102A\u1050-\u1055\u10A0-\u10C5\u10D0-\u10FA\u10FC' + + '\u1100-\u1159\u115F-\u11A2\u11A8-\u11F9\u1200-\u1248\u124A-\u124D' + + '\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0' + + '\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310' + + '\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F4\u1401-\u166C' + + '\u166F-\u1676\u1681-\u169A\u16A0-\u16EA\u1700-\u170C\u170E-\u1711' + + '\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7' + + '\u17DC\u1820-\u1877\u1880-\u18A8\u1900-\u191C\u1950-\u196D\u1970-\u1974' + + '\u1980-\u19A9\u19C1-\u19C7\u1A00-\u1A16\u1D00-\u1DBF\u1E00-\u1E9B' + + '\u1EA0-\u1EF9\u1F00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D' + + '\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC' + + '\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC' + + '\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u2094\u2102\u2107' + + '\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D' + + '\u212F-\u2131\u2133-\u2139\u213C-\u213F\u2145-\u2149\u2C00-\u2C2E' + + '\u2C30-\u2C5E\u2C80-\u2CE4\u2D00-\u2D25\u2D30-\u2D65\u2D6F\u2D80-\u2D96' + + '\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6' + + '\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u3005-\u3006\u3031-\u3035' + + '\u303B-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF' + + '\u3105-\u312C\u3131-\u318E\u31A0-\u31B7\u31F0-\u31FF\u3400-\u4DB5' + + '\u4E00-\u9FBB\uA000-\uA48C\uA800-\uA801\uA803-\uA805\uA807-\uA80A' + + '\uA80C-\uA822\uAC00-\uD7A3\uF900-\uFA2D\uFA30-\uFA6A\uFA70-\uFAD9' + + '\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C' + + '\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F' + + '\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A' + + '\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7' + + '\uFFDA-\uFFDC'; + + // equivalent of \p{Mn}\p{Mc} + const unicodeAccents = + '\u0300-\u036F\u0483-\u0486\u0591-\u05B9\u05BB-\u05BD\u05BF' + + '\u05C1-\u05C2\u05C4-\u05C5\u05C7\u0610-\u0615\u064B-\u065E\u0670' + + '\u06D6-\u06DC\u06DF-\u06E4\u06E7-\u06E8\u06EA-\u06ED\u0711\u0730-\u074A' + + '\u07A6-\u07B0\u0901-\u0903\u093C\u093E-\u094D\u0951-\u0954\u0962-\u0963' + + '\u0981-\u0983\u09BC\u09BE-\u09C4\u09C7-\u09C8\u09CB-\u09CD\u09D7' + + '\u09E2-\u09E3\u0A01-\u0A03\u0A3C\u0A3E-\u0A42\u0A47-\u0A48\u0A4B-\u0A4D' + + '\u0A70-\u0A71\u0A81-\u0A83\u0ABC\u0ABE-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD' + + '\u0AE2-\u0AE3\u0B01-\u0B03\u0B3C\u0B3E-\u0B43\u0B47-\u0B48\u0B4B-\u0B4D' + + '\u0B56-\u0B57\u0B82\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD7' + + '\u0C01-\u0C03\u0C3E-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55-\u0C56' + + '\u0C82-\u0C83\u0CBC\u0CBE-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5-\u0CD6' + + '\u0D02-\u0D03\u0D3E-\u0D43\u0D46-\u0D48\u0D4A-\u0D4D\u0D57\u0D82-\u0D83' + + '\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2-\u0DF3\u0E31\u0E34-\u0E3A' + + '\u0E47-\u0E4E\u0EB1\u0EB4-\u0EB9\u0EBB-\u0EBC\u0EC8-\u0ECD\u0F18-\u0F19' + + '\u0F35\u0F37\u0F39\u0F3E-\u0F3F\u0F71-\u0F84\u0F86-\u0F87\u0F90-\u0F97' + + '\u0F99-\u0FBC\u0FC6\u102C-\u1032\u1036-\u1039\u1056-\u1059\u135F' + + '\u1712-\u1714\u1732-\u1734\u1752-\u1753\u1772-\u1773\u17B6-\u17D3\u17DD' + + '\u180B-\u180D\u18A9\u1920-\u192B\u1930-\u193B\u19B0-\u19C0\u19C8-\u19C9' + + '\u1A17-\u1A1B\u1DC0-\u1DC3\u20D0-\u20DC\u20E1\u20E5-\u20EB\u302A-\u302F' + + '\u3099-\u309A\uA802\uA806\uA80B\uA823-\uA827\uFB1E\uFE00-\uFE0F' + + '\uFE20-\uFE23'; + + // equivalent of \p{Dn} + const unicodeDigits = + '\u0030-\u0039\u0660-\u0669\u06F0-\u06F9\u0966-\u096F\u09E6-\u09EF' + + '\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0BE6-\u0BEF\u0C66-\u0C6F' + + '\u0CE6-\u0CEF\u0D66-\u0D6F\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F29' + + '\u1040-\u1049\u17E0-\u17E9\u1810-\u1819\u1946-\u194F\u19D0-\u19D9' + + '\uFF10-\uFF19'; + + // An alpha char is a unicode chars including unicode marks or + // letter or char in otherChars range + const alpha = unicodeLetters; + + // A numeric character is any with the number digit property, or + // underscore. These characters can be included in hashtags, but a hashtag + // cannot have only these characters. + const numeric = `${unicodeDigits}_`; + + // Alphanumeric char is any alpha char or a unicode char with decimal + // number property \p{Nd} + const alphanumeric = alpha + numeric; + const leftChars = '{'; + const rightChars = '}'; + + return { + alpha, + alphanumeric, + leftChars, + rightChars + }; +} + +export function getHashtagRegexString(): string { + const { leftChars, rightChars } = getHashtagRegexStringChars(); + + const hashLeftCharList = `[${leftChars}]`; + const hashRightCharList = `[${rightChars}]`; + + // A hashtag contains characters, numbers and underscores, + // but not all numbers. + const hashtag = + `(${hashLeftCharList})` + + `(${hashLeftCharList})([a-zA-Z0-9_]{0,29}` + + `)(${hashRightCharList})(${hashRightCharList})`; + + return hashtag; +} diff --git a/packages/web/components/common/Textarea/PromptEditor/type.d.ts b/packages/web/components/common/Textarea/PromptEditor/type.d.ts new file mode 100644 index 000000000..b96de6e54 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/type.d.ts @@ -0,0 +1,5 @@ +export type PickerMenuItemType = { + key: string; + label: string; + icon?: string; +}; diff --git a/packages/web/components/common/Textarea/PromptEditor/utils.ts b/packages/web/components/common/Textarea/PromptEditor/utils.ts new file mode 100644 index 000000000..a401ff039 --- /dev/null +++ b/packages/web/components/common/Textarea/PromptEditor/utils.ts @@ -0,0 +1,275 @@ +import type { Klass, LexicalEditor, LexicalNode } from 'lexical'; +import type { EntityMatch } from '@lexical/text'; +import { $createTextNode, $getRoot, $isTextNode, TextNode } from 'lexical'; +import { useCallback } from 'react'; + +export function registerLexicalTextEntity( + editor: LexicalEditor, + getMatch: (text: string) => null | EntityMatch, + targetNode: Klass, + createNode: (textNode: TextNode) => T +): Array<() => void> { + const isTargetNode = (node: LexicalNode | null | undefined): node is T => { + return node instanceof targetNode; + }; + + const replaceWithSimpleText = (node: TextNode): void => { + const textNode = $createTextNode(node.getTextContent()); + textNode.setFormat(node.getFormat()); + node.replace(textNode); + }; + + const getMode = (node: TextNode): number => { + return node.getLatest().__mode; + }; + + const textNodeTransform = (node: TextNode) => { + if (!node.isSimpleText()) { + return; + } + + const prevSibling = node.getPreviousSibling(); + let text = node.getTextContent(); + let currentNode = node; + let match; + + if ($isTextNode(prevSibling)) { + const previousText = prevSibling.getTextContent(); + const combinedText = previousText + text; + const prevMatch = getMatch(combinedText); + + if (isTargetNode(prevSibling)) { + if (prevMatch === null || getMode(prevSibling) !== 0) { + replaceWithSimpleText(prevSibling); + + return; + } else { + const diff = prevMatch.end - previousText.length; + + if (diff > 0) { + const concatText = text.slice(0, diff); + const newTextContent = previousText + concatText; + prevSibling.select(); + prevSibling.setTextContent(newTextContent); + + if (diff === text.length) { + node.remove(); + } else { + const remainingText = text.slice(diff); + node.setTextContent(remainingText); + } + + return; + } + } + } else if (prevMatch === null || prevMatch.start < previousText.length) { + return; + } + } + + // eslint-disable-next-line no-constant-condition + while (true) { + match = getMatch(text); + let nextText = match === null ? '' : text.slice(match.end); + text = nextText; + + if (nextText === '') { + const nextSibling = currentNode.getNextSibling(); + + if ($isTextNode(nextSibling)) { + nextText = currentNode.getTextContent() + nextSibling.getTextContent(); + const nextMatch = getMatch(nextText); + + if (nextMatch === null) { + if (isTargetNode(nextSibling)) { + replaceWithSimpleText(nextSibling); + } else { + nextSibling.markDirty(); + } + + return; + } else if (nextMatch.start !== 0) { + return; + } + } + } else { + const nextMatch = getMatch(nextText); + + if (nextMatch !== null && nextMatch.start === 0) { + return; + } + } + + if (match === null) { + return; + } + + if (match.start === 0 && $isTextNode(prevSibling) && prevSibling.isTextEntity()) { + continue; + } + + let nodeToReplace; + + if (match.start === 0) { + [nodeToReplace, currentNode] = currentNode.splitText(match.end); + } else { + [, nodeToReplace, currentNode] = currentNode.splitText(match.start, match.end); + } + + const replacementNode = createNode(nodeToReplace); + replacementNode.setFormat(nodeToReplace.getFormat()); + nodeToReplace.replace(replacementNode); + + if (currentNode == null) { + return; + } + } + }; + + const reverseNodeTransform = (node: T) => { + const text = node.getTextContent(); + const match = getMatch(text); + + if (match === null || match.start !== 0) { + replaceWithSimpleText(node); + + return; + } + + if (text.length > match.end) { + // This will split out the rest of the text as simple text + node.splitText(match.end); + + return; + } + + const prevSibling = node.getPreviousSibling(); + + if ($isTextNode(prevSibling) && prevSibling.isTextEntity()) { + replaceWithSimpleText(prevSibling); + replaceWithSimpleText(node); + } + + const nextSibling = node.getNextSibling(); + + if ($isTextNode(nextSibling) && nextSibling.isTextEntity()) { + replaceWithSimpleText(nextSibling); + + // This may have already been converted in the previous block + if (isTargetNode(node)) { + replaceWithSimpleText(node); + } + } + }; + + const removePlainTextTransform = editor.registerNodeTransform(TextNode, textNodeTransform); + const removeReverseNodeTransform = editor.registerNodeTransform( + targetNode, + reverseNodeTransform + ); + + return [removePlainTextTransform, removeReverseNodeTransform]; +} + +export function textToEditorState(text: string = '') { + const paragraph = text?.split('\n'); + + return JSON.stringify({ + root: { + children: paragraph.map((p) => { + return { + children: [ + { + detail: 0, + format: 0, + mode: 'normal', + style: '', + text: p, + type: 'text', + version: 1 + } + ], + direction: 'ltr', + format: '', + indent: 0, + type: 'paragraph', + version: 1 + }; + }), + direction: 'ltr', + format: '', + indent: 0, + type: 'root', + version: 1 + } + }); +} + +export function editorStateToText(editor: LexicalEditor) { + const stringifiedEditorState = JSON.stringify(editor.getEditorState().toJSON()); + const parsedEditorState = editor.parseEditorState(stringifiedEditorState); + const editorStateTextString = parsedEditorState.read(() => $getRoot().getTextContent()); + const compressedText = editorStateTextString.replace(/\n+/g, '\n\n'); + + return compressedText; +} + +const varRegex = /\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}/g; +export const getVars = (value: string) => { + if (!value) return []; + // .filter((item) => { + // return ![CONTEXT_PLACEHOLDER_TEXT, HISTORY_PLACEHOLDER_TEXT, QUERY_PLACEHOLDER_TEXT, PRE_PROMPT_PLACEHOLDER_TEXT].includes(item) + // }) + const keys = + value + .match(varRegex) + ?.map((item) => { + return item.replace('{{', '').replace('}}', ''); + }) + .filter((key) => key.length <= 10) || []; + const keyObj: Record = {}; + // remove duplicate keys + const res: string[] = []; + keys.forEach((key) => { + if (keyObj[key]) return; + + keyObj[key] = true; + res.push(key); + }); + return res; +}; + +export type MenuTextMatch = { + leadOffset: number; + matchingString: string; + replaceableString: string; +}; +export type TriggerFn = (text: string, editor: LexicalEditor) => MenuTextMatch | null; +export const PUNCTUATION = '\\.,\\+\\*\\?\\$\\@\\|#{}\\(\\)\\^\\-\\[\\]\\\\/!%\'"~=<>_:;'; +export function useBasicTypeaheadTriggerMatch( + trigger: string, + { minLength = 1, maxLength = 75 }: { minLength?: number; maxLength?: number } +): TriggerFn { + return useCallback( + (text: string) => { + const validChars = `[^${trigger}${PUNCTUATION}\\s]`; + const TypeaheadTriggerRegex = new RegExp( + `([^${trigger}]|^)(` + `[${trigger}]` + `((?:${validChars}){0,${maxLength}})` + ')$' + ); + const match = TypeaheadTriggerRegex.exec(text); + if (match !== null) { + const maybeLeadingWhitespace = match[1]; + const matchingString = match[3]; + if (matchingString.length >= minLength) { + return { + leadOffset: match.index + maybeLeadingWhitespace.length, + matchingString, + replaceableString: match[2] + }; + } + } + return null; + }, + [maxLength, minLength, trigger] + ); +} diff --git a/packages/web/hooks/useStep.tsx b/packages/web/hooks/useStep.tsx new file mode 100644 index 000000000..2d33c8c24 --- /dev/null +++ b/packages/web/hooks/useStep.tsx @@ -0,0 +1,98 @@ +import { + Box, + Flex, + IconButton, + Step, + StepDescription, + StepIcon, + StepIndicator, + StepNumber, + StepSeparator, + StepStatus, + StepTitle, + Stepper, + css, + useSteps +} from '@chakra-ui/react'; +import React, { useCallback, useState } from 'react'; + +export const useMyStep = ({ + defaultStep = 0, + steps = [] +}: { + defaultStep?: number; + steps: { title?: string; description?: string }[]; +}) => { + const { activeStep, goToNext, goToPrevious } = useSteps({ + index: defaultStep, + count: steps.length + }); + + const MyStep = useCallback( + () => ( + + {steps.map((step, index) => ( + + + } + incomplete={ + + {index + 1} + + } + active={ + + {index + 1} + + } + /> + + + + {step.title} + + + + + ))} + + ), + [steps, activeStep] + ); + + return { + activeStep, + goToNext, + goToPrevious, + MyStep + }; +}; diff --git a/packages/web/package.json b/packages/web/package.json index 119b245f6..3972fdef1 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -21,7 +21,11 @@ "react": "18.2.0", "react-dom": "18.2.0", "react-i18next": "^12.3.1", - "turndown": "^7.1.2" + "turndown": "^7.1.2", + "lexical":"0.12.6", + "@lexical/react": "0.12.6", + "@lexical/utils": "0.12.6", + "@lexical/text": "0.12.6" }, "devDependencies": { "@types/react": "18.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eaa3df1c3..c23056652 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,9 +96,6 @@ importers: jsonwebtoken: specifier: ^9.0.2 version: registry.npmmirror.com/jsonwebtoken@9.0.2 - mammoth: - specifier: ^1.6.0 - version: registry.npmmirror.com/mammoth@1.6.0 mongoose: specifier: ^7.0.2 version: registry.npmmirror.com/mongoose@7.0.2 @@ -114,9 +111,6 @@ importers: node-cron: specifier: ^3.0.3 version: registry.npmmirror.com/node-cron@3.0.3 - pdfjs-dist: - specifier: ^4.0.269 - version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) pg: specifier: ^8.10.0 version: registry.npmmirror.com/pg@8.10.0 @@ -175,6 +169,15 @@ importers: '@fingerprintjs/fingerprintjs': specifier: ^4.2.1 version: registry.npmmirror.com/@fingerprintjs/fingerprintjs@4.2.1 + '@lexical/react': + specifier: 0.12.6 + version: registry.npmmirror.com/@lexical/react@0.12.6(lexical@0.12.6)(react-dom@18.2.0)(react@18.2.0)(yjs@13.6.10) + '@lexical/text': + specifier: 0.12.6 + version: registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6) + '@lexical/utils': + specifier: 0.12.6 + version: registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) '@monaco-editor/react': specifier: ^4.6.0 version: registry.npmmirror.com/@monaco-editor/react@4.6.0(monaco-editor@0.45.0)(react-dom@18.2.0)(react@18.2.0) @@ -184,6 +187,9 @@ importers: joplin-turndown-plugin-gfm: specifier: ^1.0.12 version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12 + lexical: + specifier: 0.12.6 + version: registry.npmmirror.com/lexical@0.12.6 mammoth: specifier: ^1.6.0 version: registry.npmmirror.com/mammoth@1.6.0 @@ -192,7 +198,7 @@ importers: version: registry.npmmirror.com/next-i18next@13.3.0(i18next@22.5.1)(next@13.5.2)(react-i18next@12.3.1)(react@18.2.0) pdfjs-dist: specifier: ^4.0.269 - version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) + version: registry.npmmirror.com/pdfjs-dist@4.0.269 react: specifier: 18.2.0 version: registry.npmmirror.com/react@18.2.0 @@ -3464,6 +3470,7 @@ packages: resolution: {integrity: sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@emotion/memoize/-/memoize-0.7.4.tgz} name: '@emotion/memoize' version: 0.7.4 + requiresBuild: true dev: false optional: true @@ -3920,12 +3927,297 @@ packages: '@jridgewell/resolve-uri': registry.npmmirror.com/@jridgewell/resolve-uri@3.1.1 '@jridgewell/sourcemap-codec': registry.npmmirror.com/@jridgewell/sourcemap-codec@1.4.15 - registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11(encoding@0.1.13): + registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-rJFp7tXzawCrMWWRsjCR80dZoIkLJ/EPgPmTk3xqpc+9ntlwbkm3LUOdFmgN+pshnhiZTQBwbFqg/QbsA1Pw9g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/clipboard/-/clipboard-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/clipboard/0.12.6 + name: '@lexical/clipboard' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/html': registry.npmmirror.com/@lexical/html@0.12.6(lexical@0.12.6) + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/code@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-D0IBKLzDFfVqk+3KPlJd2gWIq+h5QOsVn5Atz/Eh2eLRpOakSsZiRjmddsijoLsZbvgo1HObRPQAoeATRPWIzg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/code/-/code-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/code/0.12.6 + name: '@lexical/code' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + prismjs: registry.npmmirror.com/prismjs@1.29.0 + dev: false + + registry.npmmirror.com/@lexical/dragon@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-VKbXzdtF6qizwESx7Zag/VGiYKMAc+xpJF7tcwv5SH8I4bnseoozafzxRG6AE7J9nzGwO74ypKqPmmpP9e20BA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/dragon/-/dragon-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/dragon/0.12.6 + name: '@lexical/dragon' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/hashtag@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-SiEId/IBIqUKJJKGg8HSumalfKGxtZQJRkF7Q50FqFSU906V1lcM1jkU7aVw0hiuEHg3H+vFBmNTRdXKyoibsw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/hashtag/-/hashtag-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/hashtag/0.12.6 + name: '@lexical/hashtag' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/history@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-3vvbUF6XHuk/9985IQIXP15g+nr7SlwsPrd2IteOg6aNF+HeE2ttJS5dOiSJLnVZm+AX0OMgejMC1uU2uiZOtA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/history/-/history-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/history/0.12.6 + name: '@lexical/history' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/html@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-HVlJLCkazLbLpxdw0mwMkteQuv6OMkJTlAi6qGJimtuqJLm45BpaQ16PTpUmFWpWeIHL2XpvcDX/lj5fm68XPA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/html/-/html-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/html/0.12.6 + name: '@lexical/html' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/link@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-mrFFWR0EZ9liRUzHZqb2ijUDZqkCM+bNsyYqLh4I1CrJpzQtakyIEJt/GzYz4IHmmsRqwcc2zXUP/4kENjjPlQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/link/-/link-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/link/0.12.6 + name: '@lexical/link' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-9DFe8vpSxZ8NQZ/67ZFNiRptB3XPa7mUl0Rmd5WpbJHJHmiORyngYkYgKOW56T/TCtYcLfkTbctMhsIt8OeIqQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/list/-/list-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/list/0.12.6 + name: '@lexical/list' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/mark@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-utk6kgTSTuzmM0+B4imGTGwC4gQRCQ4hHEZTVbkIDbONvjbo9g6xfbTO9g6Qxs2h7Zt0Q2eDA7RG4nwC3vN1KQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/mark/-/mark-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/mark/0.12.6 + name: '@lexical/mark' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/markdown@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(lexical@0.12.6): + resolution: {integrity: sha512-q1cQ4w6KYxUF1N6nGwJTZwn8szLo0kbr8DzI62samZTxeztA0ByMSZLzvO5LSGhgeDremuWx5oa97s2qJMQZFw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/markdown/-/markdown-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/markdown/0.12.6 + name: '@lexical/markdown' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/code': registry.npmmirror.com/@lexical/code@0.12.6(lexical@0.12.6) + '@lexical/link': registry.npmmirror.com/@lexical/link@0.12.6(lexical@0.12.6) + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/rich-text': registry.npmmirror.com/@lexical/rich-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6) + '@lexical/text': registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + transitivePeerDependencies: + - '@lexical/clipboard' + - '@lexical/selection' + dev: false + + registry.npmmirror.com/@lexical/offset@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-5NgIaWCvMuOQNf3SZSNn459QfsN7SmLl+Tu4ISqxyZRoMV5Sfojzion9MjCVmt1YSsIS4B29NYQvGQ/li1saOw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/offset/-/offset-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/offset/0.12.6 + name: '@lexical/offset' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/overflow@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-4TZJhTGkn7xvR+rumSYW9U/OIsbith0kVGOvZZf+DM/t9fb0IVQWWSWmMlXJ5XNt/qXLFof3HFyJhK84dsN3NA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/overflow/-/overflow-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/overflow/0.12.6 + name: '@lexical/overflow' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/plain-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6): + resolution: {integrity: sha512-YF+EaWGQIxR1SHgeSuPrrqqSK8RYDxGv9RYyuIPvWXpt3M9NWw7hyAn7zxmXGgv2BhIicyHGPy5CyQgt3Mkb/w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/plain-text/-/plain-text-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/plain-text/0.12.6 + name: '@lexical/plain-text' + version: 0.12.6 + peerDependencies: + '@lexical/clipboard': 0.12.6 + '@lexical/selection': 0.12.6 + '@lexical/utils': 0.12.6 + lexical: 0.12.6 + dependencies: + '@lexical/clipboard': registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/react@0.12.6(lexical@0.12.6)(react-dom@18.2.0)(react@18.2.0)(yjs@13.6.10): + resolution: {integrity: sha512-Pto4wsVwrnY94tzcCXP2kWukQejSRPDfwOPd+EFh8dUzj+L7fa9Pze2wVgCRZpEohwfbcgAdEsvmSbhz+yGkog==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/react/-/react-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/react/0.12.6 + name: '@lexical/react' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + react: '>=17.x' + react-dom: '>=17.x' + dependencies: + '@lexical/clipboard': registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6) + '@lexical/code': registry.npmmirror.com/@lexical/code@0.12.6(lexical@0.12.6) + '@lexical/dragon': registry.npmmirror.com/@lexical/dragon@0.12.6(lexical@0.12.6) + '@lexical/hashtag': registry.npmmirror.com/@lexical/hashtag@0.12.6(lexical@0.12.6) + '@lexical/history': registry.npmmirror.com/@lexical/history@0.12.6(lexical@0.12.6) + '@lexical/link': registry.npmmirror.com/@lexical/link@0.12.6(lexical@0.12.6) + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/mark': registry.npmmirror.com/@lexical/mark@0.12.6(lexical@0.12.6) + '@lexical/markdown': registry.npmmirror.com/@lexical/markdown@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(lexical@0.12.6) + '@lexical/overflow': registry.npmmirror.com/@lexical/overflow@0.12.6(lexical@0.12.6) + '@lexical/plain-text': registry.npmmirror.com/@lexical/plain-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6) + '@lexical/rich-text': registry.npmmirror.com/@lexical/rich-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/table': registry.npmmirror.com/@lexical/table@0.12.6(lexical@0.12.6) + '@lexical/text': registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + '@lexical/yjs': registry.npmmirror.com/@lexical/yjs@0.12.6(lexical@0.12.6)(yjs@13.6.10) + lexical: registry.npmmirror.com/lexical@0.12.6 + react: registry.npmmirror.com/react@18.2.0 + react-dom: registry.npmmirror.com/react-dom@18.2.0(react@18.2.0) + react-error-boundary: registry.npmmirror.com/react-error-boundary@3.1.4(react@18.2.0) + transitivePeerDependencies: + - yjs + dev: false + + registry.npmmirror.com/@lexical/rich-text@0.12.6(@lexical/clipboard@0.12.6)(@lexical/selection@0.12.6)(@lexical/utils@0.12.6)(lexical@0.12.6): + resolution: {integrity: sha512-fRZHy2ug6Pq+pJK7trr9phTGaD2ba3If8o36dphOsl27MtUllpz68lcXL6mUonzJhAu4um1e9u7GFR3dLp+cVA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/rich-text/-/rich-text-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/rich-text/0.12.6 + name: '@lexical/rich-text' + version: 0.12.6 + peerDependencies: + '@lexical/clipboard': 0.12.6 + '@lexical/selection': 0.12.6 + '@lexical/utils': 0.12.6 + lexical: 0.12.6 + dependencies: + '@lexical/clipboard': registry.npmmirror.com/@lexical/clipboard@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-HJBEazVwOe6duyHV6+vB/MS4kUBlCV05Cfcigdx8HlLLFQRWPqHrTpaxKz4jfb9ar0SlI2W1BUNbySAxMkC/HQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/selection/-/selection-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/selection/0.12.6 + name: '@lexical/selection' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/table@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-rUh9/fN831T6UpNiPuzx0x6HNi/eQ7W5AQrVBwwzEwkbwAqnE0n28DP924AUbX72UsQNHtywgmDApMoEV7W2iQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/table/-/table-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/table/0.12.6 + name: '@lexical/table' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/utils': registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/text@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-WfqfH9gvPAx9Hi9wrJDWECdvt6turPQXImCRI657LVfsP2hHh4eHpcSnd3YYH313pv98HPWmeMstBbEieYwTpQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/text/-/text-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/text/0.12.6 + name: '@lexical/text' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/utils@0.12.6(lexical@0.12.6): + resolution: {integrity: sha512-hK5r/TD2nH5TfWSiCxy7/lh0s11qJcI1wo++PBQOR9o937pQ+/Zr/1tMOc8MnrTpl89mtmYtPfWW3f++HH1Yog==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/utils/-/utils-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/utils/0.12.6 + name: '@lexical/utils' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + dependencies: + '@lexical/list': registry.npmmirror.com/@lexical/list@0.12.6(lexical@0.12.6) + '@lexical/selection': registry.npmmirror.com/@lexical/selection@0.12.6(lexical@0.12.6) + '@lexical/table': registry.npmmirror.com/@lexical/table@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + dev: false + + registry.npmmirror.com/@lexical/yjs@0.12.6(lexical@0.12.6)(yjs@13.6.10): + resolution: {integrity: sha512-I/Yf/Qm8/ydU983kWpFBlDFNFQXLYur5uaAimTSBcJuqHmy3cv1xM7Xrq4BtM+0orKgWJt8vR8cLVIU9sAmzfw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@lexical/yjs/-/yjs-0.12.6.tgz} + id: registry.npmmirror.com/@lexical/yjs/0.12.6 + name: '@lexical/yjs' + version: 0.12.6 + peerDependencies: + lexical: 0.12.6 + yjs: '>=13.5.22' + dependencies: + '@lexical/offset': registry.npmmirror.com/@lexical/offset@0.12.6(lexical@0.12.6) + lexical: registry.npmmirror.com/lexical@0.12.6 + yjs: registry.npmmirror.com/yjs@13.6.10 + dev: false + + registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11: resolution: {integrity: sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz} - id: registry.npmmirror.com/@mapbox/node-pre-gyp/1.0.11 name: '@mapbox/node-pre-gyp' version: 1.0.11 hasBin: true + requiresBuild: true dependencies: detect-libc: registry.npmmirror.com/detect-libc@2.0.2 https-proxy-agent: registry.npmmirror.com/https-proxy-agent@5.0.1 @@ -5391,6 +5683,7 @@ packages: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abbrev/-/abbrev-1.1.1.tgz} name: abbrev version: 1.1.1 + requiresBuild: true dev: false optional: true @@ -5437,6 +5730,7 @@ packages: name: agent-base version: 6.0.2 engines: {node: '>= 6.0.0'} + requiresBuild: true dependencies: debug: registry.npmmirror.com/debug@4.3.4 transitivePeerDependencies: @@ -5539,6 +5833,7 @@ packages: resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/aproba/-/aproba-2.0.0.tgz} name: aproba version: 2.0.0 + requiresBuild: true dev: false optional: true @@ -5547,6 +5842,7 @@ packages: name: are-we-there-yet version: 2.0.0 engines: {node: '>=10'} + requiresBuild: true dependencies: delegates: registry.npmmirror.com/delegates@1.0.0 readable-stream: registry.npmmirror.com/readable-stream@3.6.2 @@ -6113,15 +6409,14 @@ packages: name: caniuse-lite version: 1.0.30001574 - registry.npmmirror.com/canvas@2.11.2(encoding@0.1.13): + registry.npmmirror.com/canvas@2.11.2: resolution: {integrity: sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/canvas/-/canvas-2.11.2.tgz} - id: registry.npmmirror.com/canvas/2.11.2 name: canvas version: 2.11.2 engines: {node: '>=6'} requiresBuild: true dependencies: - '@mapbox/node-pre-gyp': registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11(encoding@0.1.13) + '@mapbox/node-pre-gyp': registry.npmmirror.com/@mapbox/node-pre-gyp@1.0.11 nan: registry.npmmirror.com/nan@2.18.0 simple-get: registry.npmmirror.com/simple-get@3.1.1 transitivePeerDependencies: @@ -6272,6 +6567,7 @@ packages: name: chownr version: 2.0.0 engines: {node: '>=10'} + requiresBuild: true dev: false optional: true @@ -6417,6 +6713,7 @@ packages: name: color-support version: 1.1.3 hasBin: true + requiresBuild: true dev: false optional: true @@ -6513,6 +6810,7 @@ packages: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/console-control-strings/-/console-control-strings-1.1.0.tgz} name: console-control-strings version: 1.1.0 + requiresBuild: true dev: false optional: true @@ -7192,6 +7490,7 @@ packages: name: decompress-response version: 4.2.1 engines: {node: '>=8'} + requiresBuild: true dependencies: mimic-response: registry.npmmirror.com/mimic-response@2.1.0 dev: false @@ -7259,6 +7558,7 @@ packages: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/delegates/-/delegates-1.0.0.tgz} name: delegates version: 1.0.0 + requiresBuild: true dev: false optional: true @@ -7296,6 +7596,7 @@ packages: name: detect-libc version: 2.0.2 engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -7534,6 +7835,7 @@ packages: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz} name: emoji-regex version: 8.0.0 + requiresBuild: true registry.npmmirror.com/emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/emoji-regex/-/emoji-regex-9.2.2.tgz} @@ -8657,6 +8959,7 @@ packages: name: fs-minipass version: 2.1.0 engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: registry.npmmirror.com/minipass@3.3.6 dev: false @@ -8704,6 +9007,7 @@ packages: name: gauge version: 3.0.2 engines: {node: '>=10'} + requiresBuild: true dependencies: aproba: registry.npmmirror.com/aproba@2.0.0 color-support: registry.npmmirror.com/color-support@1.1.3 @@ -8925,6 +9229,7 @@ packages: resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/has-unicode/-/has-unicode-2.0.1.tgz} name: has-unicode version: 2.0.1 + requiresBuild: true dev: false optional: true @@ -9118,6 +9423,7 @@ packages: name: https-proxy-agent version: 5.0.1 engines: {node: '>= 6'} + requiresBuild: true dependencies: agent-base: registry.npmmirror.com/agent-base@6.0.2 debug: registry.npmmirror.com/debug@4.3.4 @@ -9635,6 +9941,12 @@ packages: engines: {node: '>=10'} dev: true + registry.npmmirror.com/isomorphic.js@0.2.5: + resolution: {integrity: sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/isomorphic.js/-/isomorphic.js-0.2.5.tgz} + name: isomorphic.js + version: 0.2.5 + dev: false + registry.npmmirror.com/iterator.prototype@1.1.2: resolution: {integrity: sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/iterator.prototype/-/iterator.prototype-1.1.2.tgz} name: iterator.prototype @@ -9884,6 +10196,22 @@ packages: type-check: registry.npmmirror.com/type-check@0.4.0 dev: true + registry.npmmirror.com/lexical@0.12.6: + resolution: {integrity: sha512-Nlfjc+k9cIWpOMv7XufF0Mv09TAXSemNAuAqFLaOwTcN+RvhvYTDtVLSp9D9r+5I097fYs1Vf/UYwH2xEpkFfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lexical/-/lexical-0.12.6.tgz} + name: lexical + version: 0.12.6 + dev: false + + registry.npmmirror.com/lib0@0.2.88: + resolution: {integrity: sha512-KyroiEvCeZcZEMx5Ys+b4u4eEBbA1ch7XUaBhYpwa/nPMrzTjUhI4RfcytmQfYoTBPcdyx+FX6WFNIoNuJzJfQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lib0/-/lib0-0.2.88.tgz} + name: lib0 + version: 0.2.88 + engines: {node: '>=16'} + hasBin: true + dependencies: + isomorphic.js: registry.npmmirror.com/isomorphic.js@0.2.5 + dev: false + registry.npmmirror.com/lie@3.3.0: resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/lie/-/lie-3.3.0.tgz} name: lie @@ -10120,6 +10448,7 @@ packages: name: make-dir version: 3.1.0 engines: {node: '>=8'} + requiresBuild: true dependencies: semver: registry.npmmirror.com/semver@6.3.1 dev: false @@ -10369,6 +10698,7 @@ packages: resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/memory-pager/-/memory-pager-1.5.0.tgz} name: memory-pager version: 1.5.0 + requiresBuild: true dev: false optional: true @@ -10804,6 +11134,7 @@ packages: name: mimic-response version: 2.1.0 engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -10836,6 +11167,7 @@ packages: name: minipass version: 3.3.6 engines: {node: '>=8'} + requiresBuild: true dependencies: yallist: registry.npmmirror.com/yallist@4.0.0 dev: false @@ -10846,6 +11178,7 @@ packages: name: minipass version: 5.0.0 engines: {node: '>=8'} + requiresBuild: true dev: false optional: true @@ -10854,6 +11187,7 @@ packages: name: minizlib version: 2.1.2 engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: registry.npmmirror.com/minipass@3.3.6 yallist: registry.npmmirror.com/yallist@4.0.0 @@ -10875,6 +11209,7 @@ packages: version: 1.0.4 engines: {node: '>=10'} hasBin: true + requiresBuild: true dev: false optional: true @@ -10997,6 +11332,7 @@ packages: resolution: {integrity: sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/nan/-/nan-2.18.0.tgz} name: nan version: 2.18.0 + requiresBuild: true dev: false optional: true @@ -11189,6 +11525,7 @@ packages: version: 5.0.0 engines: {node: '>=6'} hasBin: true + requiresBuild: true dependencies: abbrev: registry.npmmirror.com/abbrev@1.1.1 dev: false @@ -11213,6 +11550,7 @@ packages: resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/npmlog/-/npmlog-5.0.1.tgz} name: npmlog version: 5.0.1 + requiresBuild: true dependencies: are-we-there-yet: registry.npmmirror.com/are-we-there-yet@2.0.0 console-control-strings: registry.npmmirror.com/console-control-strings@1.1.0 @@ -11616,14 +11954,13 @@ packages: sha.js: registry.npmmirror.com/sha.js@2.4.11 dev: true - registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13): + registry.npmmirror.com/pdfjs-dist@4.0.269: resolution: {integrity: sha512-jjWO56tcOjnmPqDf8PmXDeZ781AGvpHMYI3HhNtaFKTRXXPaD1ArSrhVe38/XsrIQJ0onISCND/vuXaWJkiDWw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pdfjs-dist/-/pdfjs-dist-4.0.269.tgz} - id: registry.npmmirror.com/pdfjs-dist/4.0.269 name: pdfjs-dist version: 4.0.269 engines: {node: '>=18'} optionalDependencies: - canvas: registry.npmmirror.com/canvas@2.11.2(encoding@0.1.13) + canvas: registry.npmmirror.com/canvas@2.11.2 path2d-polyfill: registry.npmmirror.com/path2d-polyfill@2.0.1 transitivePeerDependencies: - encoding @@ -11990,6 +12327,19 @@ packages: react: registry.npmmirror.com/react@18.2.0 scheduler: registry.npmmirror.com/scheduler@0.23.0 + registry.npmmirror.com/react-error-boundary@3.1.4(react@18.2.0): + resolution: {integrity: sha512-uM9uPzZJTF6wRQORmSrvOIgt4lJ9MC1sNgEOj2XGsDTRE4kmpWxg7ENK9EWNKJRMAOY9z0MuF4yIfl6gp4sotA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/react-error-boundary/-/react-error-boundary-3.1.4.tgz} + id: registry.npmmirror.com/react-error-boundary/3.1.4 + name: react-error-boundary + version: 3.1.4 + engines: {node: '>=10', npm: '>=6'} + peerDependencies: + react: '>=16.13.1' + dependencies: + '@babel/runtime': registry.npmmirror.com/@babel/runtime@7.23.7 + react: registry.npmmirror.com/react@18.2.0 + dev: false + registry.npmmirror.com/react-fast-compare@3.2.2: resolution: {integrity: sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/react-fast-compare/-/react-fast-compare-3.2.2.tgz} name: react-fast-compare @@ -12694,6 +13044,7 @@ packages: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/set-blocking/-/set-blocking-2.0.0.tgz} name: set-blocking version: 2.0.0 + requiresBuild: true dev: false optional: true @@ -12780,6 +13131,7 @@ packages: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/simple-concat/-/simple-concat-1.0.1.tgz} name: simple-concat version: 1.0.1 + requiresBuild: true dev: false optional: true @@ -12787,6 +13139,7 @@ packages: resolution: {integrity: sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/simple-get/-/simple-get-3.1.1.tgz} name: simple-get version: 3.1.1 + requiresBuild: true dependencies: decompress-response: registry.npmmirror.com/decompress-response@4.2.1 once: registry.npmmirror.com/once@1.4.0 @@ -12886,6 +13239,7 @@ packages: resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz} name: sparse-bitfield version: 3.0.3 + requiresBuild: true dependencies: memory-pager: registry.npmmirror.com/memory-pager@1.5.0 dev: false @@ -13181,6 +13535,7 @@ packages: name: tar version: 6.2.0 engines: {node: '>=10'} + requiresBuild: true dependencies: chownr: registry.npmmirror.com/chownr@2.0.0 fs-minipass: registry.npmmirror.com/fs-minipass@2.1.0 @@ -14115,6 +14470,7 @@ packages: resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/wide-align/-/wide-align-1.1.5.tgz} name: wide-align version: 1.1.5 + requiresBuild: true dependencies: string-width: registry.npmmirror.com/string-width@4.2.3 dev: false @@ -14183,6 +14539,15 @@ packages: engines: {node: '>= 14'} dev: true + registry.npmmirror.com/yjs@13.6.10: + resolution: {integrity: sha512-1JcyQek1vaMyrDm7Fqfa+pvHg/DURSbVo4VmeN7wjnTKB/lZrfIPhdCj7d8sboK6zLfRBJXegTjc9JlaDd8/Zw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/yjs/-/yjs-13.6.10.tgz} + name: yjs + version: 13.6.10 + engines: {node: '>=16.0.0', npm: '>=8.0.0'} + dependencies: + lib0: registry.npmmirror.com/lib0@0.2.88 + dev: false + registry.npmmirror.com/yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/yocto-queue/-/yocto-queue-0.1.0.tgz} name: yocto-queue diff --git a/projects/app/data/simpleTemplates/fastgpt-simple.json b/projects/app/data/simpleTemplates/fastgpt-simple.json index 819691ff6..baf397405 100644 --- a/projects/app/data/simpleTemplates/fastgpt-simple.json +++ b/projects/app/data/simpleTemplates/fastgpt-simple.json @@ -1,6 +1,6 @@ { - "name": "极简模板", - "desc": "极简模板\n已内置参数细节", + "name": "core.app.template.Simple template", + "desc": "core.app.template.Simple template tip", "systemForm": { "aiSettings": { "model": true, diff --git a/projects/app/next.config.js b/projects/app/next.config.js index ef9310e53..aa2da3e4b 100644 --- a/projects/app/next.config.js +++ b/projects/app/next.config.js @@ -45,7 +45,7 @@ const nextConfig = { }, transpilePackages: ['@fastgpt/*'], experimental: { - serverComponentsExternalPackages: ['mongoose', 'pg'], + serverComponentsExternalPackages: ['mongoose', 'pg', '@chakra-ui/react', '@lexical/react'], outputFileTracingRoot: path.join(__dirname, '../../') } }; diff --git a/projects/app/package.json b/projects/app/package.json index baaaa052c..a90754dbc 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -1,6 +1,6 @@ { "name": "app", - "version": "4.6.6", + "version": "4.6.7", "private": false, "scripts": { "dev": "next dev", diff --git a/projects/app/public/docs/versionIntro.md b/projects/app/public/docs/versionIntro.md index 95e1767fa..15150a81f 100644 --- a/projects/app/public/docs/versionIntro.md +++ b/projects/app/public/docs/versionIntro.md @@ -1,12 +1,11 @@ -### Fast GPT V4.6.6 +### Fast GPT V4.6.7 -1. 新增 - Http 模块请求头支持 Json 编辑器。 -2. 新增 - 搜索方式：分离向量语义检索，全文检索和重排，通过 RRF 进行排序合并。 -3. 新增 - [问题补全模块](https://doc.fastgpt.in/docs/workflow/modules/coreferenceresolution/) -5. 新增 - [文本编辑模块](https://doc.fastgpt.in/docs/workflow/modules/text_editor/) -6. 新增 - [判断器模块](https://doc.fastgpt.in/docs/workflow/modules/tfswitch/) -7. 新增 - [自定义反馈模块](https://doc.fastgpt.in/docs/workflow/modules/custom_feedback/) -8. 新增 - 【内容提取】模块支持选择模型，以及字段枚举 -9. [使用文档](https://doc.fastgpt.in/docs/intro/) -10. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow) -11. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/) +1. 修改了知识库UI及新的导入交互方式。 +2. 优化知识库和对话的数据索引，加快数据操作。 +3. 知识库 openAPI，支持通过 API 操作知识库。 +4. 新增 - 输入框变量提示。输入 { 号后将会获得可用变量提示。根据社区针对高级编排的反馈，我们计划于 2 月份的版本中，优化变量内容，支持模块的局部变量以及更多全局变量写入。 +5. 修复 - API 对话时，chatId 冲突问题。 +6. 修复 - Iframe 嵌入网页可能导致的 window.onLoad 冲突。 +7. [使用文档](https://doc.fastgpt.in/docs/intro/) +8. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow) +9. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/) diff --git a/projects/app/public/imgs/files/collection.svg b/projects/app/public/imgs/files/collection.svg deleted file mode 100644 index dbdb227ad..000000000 --- a/projects/app/public/imgs/files/collection.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/folder.svg b/projects/app/public/imgs/files/folder.svg index 602393396..7e8e49578 100644 --- a/projects/app/public/imgs/files/folder.svg +++ b/projects/app/public/imgs/files/folder.svg @@ -1 +1,5 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/projects/app/public/imgs/files/link.svg b/projects/app/public/imgs/files/link.svg deleted file mode 100644 index 68534ba48..000000000 --- a/projects/app/public/imgs/files/link.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/manual.svg b/projects/app/public/imgs/files/manual.svg deleted file mode 100644 index fa8b3106a..000000000 --- a/projects/app/public/imgs/files/manual.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/mark.svg b/projects/app/public/imgs/files/mark.svg deleted file mode 100644 index 8c86e0099..000000000 --- a/projects/app/public/imgs/files/mark.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/files/pdf.svg b/projects/app/public/imgs/files/pdf.svg deleted file mode 100644 index 90940bb31..000000000 --- a/projects/app/public/imgs/files/pdf.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/modal/folder.svg b/projects/app/public/imgs/modal/folder.svg deleted file mode 100644 index 78d213c83..000000000 --- a/projects/app/public/imgs/modal/folder.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/projects/app/public/imgs/module/ai.svg b/projects/app/public/imgs/module/ai.svg index 2fd7b1177..252540059 100644 --- a/projects/app/public/imgs/module/ai.svg +++ b/projects/app/public/imgs/module/ai.svg @@ -1,11 +1,5 @@ - - + - + d="M15.128 4.05015H16.25C18.3211 4.05015 20 5.72908 20 7.80011V12.8001C20 14.8711 18.3211 16.55 16.25 16.55H3.75C1.67895 16.55 0 14.8711 0 12.8001V7.80011C0 5.72908 1.67895 4.05015 3.75 4.05015H4.872L4.4153 1.31012C4.31315 0.697277 4.7272 0.117632 5.34005 0.015483C5.9529 -0.0866661 6.53255 0.32738 6.6347 0.940225L7.1347 3.9402C7.14085 3.977 7.1451 4.0137 7.1476 4.05015H12.8524C12.8549 4.01365 12.8591 3.977 12.8653 3.9402L13.3653 0.940225C13.4675 0.32738 14.0471 -0.0866661 14.66 0.015483C15.2728 0.117632 15.6869 0.697277 15.5847 1.31012L15.128 4.05015ZM4.9 9.3001V10.3001C4.9 10.9214 5.4037 11.4251 6.025 11.4251C6.6463 11.4251 7.15 10.9214 7.15 10.3001V9.3001C7.15 8.6788 6.6463 8.17511 6.025 8.17511C5.4037 8.17511 4.9 8.6788 4.9 9.3001ZM12.85 9.3001V10.3001C12.85 10.9214 13.3537 11.4251 13.975 11.4251C14.5963 11.4251 15.1 10.9214 15.1 10.3001V9.3001C15.1 8.6788 14.5963 8.17511 13.975 8.17511C13.3537 8.17511 12.85 8.6788 12.85 9.3001ZM5 20C4.3787 20 3.875 19.4963 3.875 18.875C3.875 18.2537 4.3787 17.75 5 17.75H15C15.6213 17.75 16.125 18.2537 16.125 18.875C16.125 19.4963 15.6213 20 15 20H5Z" + fill="#3370ff" /> \ No newline at end of file diff --git a/projects/app/public/imgs/module/templates.png b/projects/app/public/imgs/module/templates.png deleted file mode 100644 index b6f7e49b840063825ebb63296d0f6c334df5b678..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2444 zcmbVNc{CJU8^1GXVx~9QVh~9o$}ln6#*ncaTh?UBT2zcBnq+v(mO?3ov4@bYBxRpM zV~ZHFH}){fF5dcP&i9@3{{Q`P?{n_&_bk8Xp8K2|XF)JHz=Pxg030wf)U#q)GJEac z$J(h_w+@zp1X>yBg0gP01pv79jr4TRhdM7)A}p-!gxXf_<_q)!^^4s7S$-wfxn+`q zRG|_NeC~OjB*(|b6-9@H6-wJ53Z1uqT{LAbz`JBRW-B0AbdPb#aSWPsUdIw%{qC)* z$?#WNvE<4Ann3jA^x>b)oUL`FRhRbKEy~(Iw~5U9XLaP{W~%qqLB?i*IkrVJa^(6} zxkku%NsUrMri z=9isD^~ECIs-;Iz&i#=->9khow|c=%4bKpC=#LO2)z`PwuWq7ZyxttGTQMMqlNA0n z9D|7=tW&mwk<32m z8ZC2g_wTm9k78zS$*TyMpga#VG|adTsAU7C#%ixh%i~cPWm|^PZ_$IA^ptYtTfrYX z;bOAl#`BhB;5x)5*80-4?a>})Mw6ZqM}HjuI%00&W*gq3k0yxXB2zj>oj?=qzGpfX zG}+2MXoUbVl_ysn0J!k{ED{F%`|aO&LV(|6hL0F%vULCYgbP&I#ODP7U~u2_g3$jH zWLpkQDL4GxNrBC6@A#WBQA4yVS^))sH}=hkOFf^fo?o3Plu%Xpc-vGOsjS*}^W+^< zlFg*s+R^@i-K5^dW8+xG=5b={%(t1RLe`kONd#Pjh@X35eH-G4ZBf%ymbqxSNJ@&% zEpcVfato4s>>y*!?ue5~rpMh49oj$VU7s3wQ9tIPWo}F$>4M|qDPhxmDuPggYOJOa#LFA5~(bI175H+YD4Ktl*!A2;4)(ev2nhQ zVW_;R?r%BaSxEi5b{?PW8zg++>|}%T^D7f3?@7b~j_kR_OgC@4$!=K0-two2wj1M( zzrWve5}x*QhKT^`354&8F40zb8mkq6kblqd&R31>lSqkD;M^$ zE8Klu)9L!UC5np5>v1V6Yu8@i@l~l#-G|Rgx@)!W9O~aa{pm=_F0r{w4MVscer2#l zqh_B>Ps8-4R2v_!W*sznNc*JJFY!H*Pa6)f?Vt3<;(Q$wZ$t=0i6|2P*x+Teu!L*i z@~D-UiJP+@-_subQ(u%W#UZ+q^uk{9w4}ZUCm^YwgYzHF$lynp$?I`&%s_6wh{F{I^c;ITY3n3nnn zBQy5xL&~WIxp}ke9lsstGna0?#*A!>XA~Q@Uwr40+cS7i+|3l1Z+ZC;nC|e zmC`(Wm>dAfewPa_*4r0soeZPu9T$wN1_~D{Hy3KEgKx@2F{{IHdK7Wqc!Wmet%2P+ z#+0p5=S39Cum0EU#&xG)cRgpTGZ+O=-0CzfpEIiNWsd$#XHvVB5*77^U#~%b?wn`X zp(>BX!TOt3QXCXnx{Q^k`|!c6Bcoc)8S}RTJzgS%^PCH%X8HF|x;aCjkCllG< z3m`zwpn*8qYl{_MyURvQzzl+IGoU_RKpk{3*$KD)D~cwGhVd4G_FRB&2!vzaD2ve&+Bj?YV2Kd9Ud z2^PZYErVT;Yz?{)I9T@A@F7ksCYJ5Xhf`zM1BD<>gmpi!b}P8^|81xf(v&XFJN&}7 zX5J`YK~eYGwH8S7DTm}pfel-~WAi2Y$bab_a@3R-|4)U_7G{w4JOWw?nu>R4~!EB@W zQTbx)S#@|X%n0waN@G-y*4o7I1iB~8hRa9=J{Kat4UTu|$*}ngld#`G7T*ldZ|@rT zC}lLzPtRdI{8Ch~`LOqvQJj)Sp47wzZ>Da*WzV*5F%wL#cYmn4o_b`{SLHCr{Gr1h z4}A}JSlHkrKl$;YP>Q)?Hli%5-ZE{;@Cp{?f12~Ww-met-~ygT3$UhCWV ztx-p!RI@gSvVcz2-c}&p$H4&=HBPEpYaC0znwAoIx~_D6t$5;n;)Z7+WqMiVR2!V$ ze1US`zWzg!C;H|&%U0N0r7j$Y;T&Oge*zJ5xZTuT=P3O7rGx;L^t9Cmj~03P)x>o= zeW+81)R`;jfSEJpH_uRk-_YX77L6+P!0Ssnl)#XjeVdrVvOgc~_PnH9?f)455Q`z2 zD^*?Zo}DS`4yWafbvE>yZyGSZRogfQ~+9}1jefSh43VUaK-Q4H`0dV)QaPo z#vw<(=$$_8;QeATFRZQkLXO=bqFT*=S?QVkIdh{|3$x0kRR^-*yF1z#r-822oqwLi xEV^@cpH5ixNmOkZOB#XXh^iX&Pz{28e`}}7 ); diff --git a/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx b/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx index 75b3125fd..57abc080e 100644 --- a/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx +++ b/projects/app/src/components/core/module/Flow/components/nodes/NodeUserGuide.tsx @@ -1,23 +1,10 @@ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useTransition } from 'react'; import { NodeProps } from 'reactflow'; -import { - Box, - Flex, - Textarea, - useTheme, - Table, - Thead, - Tbody, - Tr, - Th, - Td, - TableContainer, - Switch -} from '@chakra-ui/react'; +import { Box, Flex, Textarea, useTheme } from '@chakra-ui/react'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { FlowModuleItemType, ModuleItemType } from '@fastgpt/global/core/module/type.d'; import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; -import { welcomeTextTip, variableTip } from '@fastgpt/global/core/module/template/tip'; +import { welcomeTextTip } from '@fastgpt/global/core/module/template/tip'; import { onChangeNode } from '../../FlowProvider'; import VariableEdit from '../modules/VariableEdit'; @@ -29,6 +16,7 @@ import type { VariableItemType } from '@fastgpt/global/core/module/type.d'; import QGSwitch from '@/components/core/module/Flow/components/modules/QGSwitch'; import TTSSelect from '@/components/core/module/Flow/components/modules/TTSSelect'; import { splitGuideModule } from '@fastgpt/global/core/module/utils'; +import { useTranslation } from 'next-i18next'; const NodeUserGuide = React.memo(function NodeUserGuide({ data }: { data: FlowModuleItemType }) { const theme = useTheme(); @@ -56,19 +44,18 @@ export default function Node({ data }: NodeProps) { return ; } export function WelcomeText({ data }: { data: FlowModuleItemType }) { + const { t } = useTranslation(); const { inputs, moduleId } = data; + const [, startTst] = useTransition(); - const welcomeText = useMemo( - () => inputs.find((item) => item.key === ModuleInputKeyEnum.welcomeText), - [inputs] - ); + const welcomeText = inputs.find((item) => item.key === ModuleInputKeyEnum.welcomeText); return ( <> - 开场白 - + {t('core.app.Welcome Text')} + @@ -79,16 +66,18 @@ export function WelcomeText({ data }: { data: FlowModuleItemType }) { resize={'both'} defaultValue={welcomeText.value} bg={'myWhite.500'} - placeholder={welcomeTextTip} + placeholder={t(welcomeTextTip)} onChange={(e) => { - onChangeNode({ - moduleId, - key: ModuleInputKeyEnum.welcomeText, - type: 'updateInput', - value: { - ...welcomeText, - value: e.target.value - } + startTst(() => { + onChangeNode({ + moduleId, + key: ModuleInputKeyEnum.welcomeText, + type: 'updateInput', + value: { + ...welcomeText, + value: e.target.value + } + }); }); }} /> diff --git a/projects/app/src/components/core/module/Flow/components/nodes/abandon/NodeVariable.tsx b/projects/app/src/components/core/module/Flow/components/nodes/abandon/NodeVariable.tsx deleted file mode 100644 index d16d1fff8..000000000 --- a/projects/app/src/components/core/module/Flow/components/nodes/abandon/NodeVariable.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* Abandon */ -import React, { useCallback, useMemo, useState } from 'react'; -import { NodeProps } from 'reactflow'; -import { Box, Button, Table, Thead, Tbody, Tr, Th, Td, TableContainer } from '@chakra-ui/react'; -import { AddIcon } from '@chakra-ui/icons'; -import NodeCard from '../../render/NodeCard'; -import { FlowModuleItemType } from '@fastgpt/global/core/module/type.d'; -import Container from '../../modules/Container'; -import { VariableInputEnum, ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; -import type { VariableItemType } from '@fastgpt/global/core/module/type.d'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import { customAlphabet } from 'nanoid'; -const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6); -import VariableEditModal, { addVariable } from '../../modules/VariableEdit'; -import { onChangeNode } from '../../../FlowProvider'; - -export const defaultVariable: VariableItemType = { - id: nanoid(), - key: 'key', - label: 'label', - type: VariableInputEnum.input, - required: true, - maxLen: 50, - enums: [{ value: '' }] -}; - -const NodeUserGuide = ({ data }: NodeProps) => { - const { inputs, moduleId } = data; - - const variables = useMemo( - () => - (inputs.find((item) => item.key === ModuleInputKeyEnum.variables) - ?.value as VariableItemType[]) || [], - [inputs] - ); - - return ( - <> - - - - onChangeNode({ - moduleId, - key: ModuleInputKeyEnum.variables, - type: 'updateInput', - value: { - ...inputs.find((item) => item.key === ModuleInputKeyEnum.variables), - value: e - } - }) - } - /> - - - - ); -}; -export default React.memo(NodeUserGuide); diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx index b843b5e73..fb9728102 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useMemo, useState } from 'react'; +import React, { useMemo } from 'react'; import type { FlowNodeInputItemType } from '@fastgpt/global/core/module/node/type'; import { Box } from '@chakra-ui/react'; import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/module/node/constant'; @@ -6,7 +6,7 @@ import dynamic from 'next/dynamic'; import InputLabel from './Label'; import type { RenderInputProps } from './type.d'; -import { getFlowStore, type useFlowProviderStoreType } from '../../../FlowProvider'; +import { useFlowProviderStore } from '../../../FlowProvider'; import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; const RenderList: { @@ -77,8 +77,8 @@ type Props = { moduleId: string; CustomComponent?: Record React.ReactNode>; }; -const RenderInput = ({ flowInputList, moduleId, CustomComponent = {} }: Props) => { - const [mode, setMode] = useState('app'); +const RenderInput = ({ flowInputList, moduleId, CustomComponent }: Props) => { + const { mode } = useFlowProviderStore(); const sortInputs = useMemo( () => @@ -109,48 +109,43 @@ const RenderInput = ({ flowInputList, moduleId, CustomComponent = {} }: Props) = [mode, sortInputs] ); - useEffect(() => { - async () => { - const { mode } = await getFlowStore(); - setMode(mode); - }; - }, []); + const memoCustomComponent = useMemo(() => CustomComponent || {}, [CustomComponent]); - return ( - <> - {filterInputs.map((input) => { - const RenderComponent = (() => { - if (input.type === FlowNodeInputTypeEnum.custom && CustomComponent[input.key]) { - return <>{CustomComponent[input.key]({ ...input })}; - } - const Component = RenderList.find((item) => item.types.includes(input.type))?.Component; + const Render = useMemo(() => { + return filterInputs.map((input) => { + const RenderComponent = (() => { + if (input.type === FlowNodeInputTypeEnum.custom && memoCustomComponent[input.key]) { + return <>{memoCustomComponent[input.key]({ ...input })}; + } + const Component = RenderList.find((item) => item.types.includes(input.type))?.Component; - if (!Component) return null; - return ; - })(); + if (!Component) return null; + return ; + })(); - return ( - - {input.key === ModuleInputKeyEnum.userChatInput && ( - - )} - {input.type !== FlowNodeInputTypeEnum.hidden && ( - <> - {!!input.label && ( - - )} - {!!RenderComponent && ( - - {RenderComponent} - - )} - - )} - - ); - })} - - ); + return ( + + {input.key === ModuleInputKeyEnum.userChatInput && ( + + )} + {input.type !== FlowNodeInputTypeEnum.hidden && ( + <> + {!!input.label && ( + + )} + {!!RenderComponent && ( + + {RenderComponent} + + )} + + )} + + ); + }); + }, [memoCustomComponent, filterInputs, mode, moduleId]); + + return <>{Render}; }; export default React.memo(RenderInput); diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx index fff3d7ef6..56ec626e0 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDataset.tsx @@ -1,20 +1,39 @@ -import React, { useMemo } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import type { RenderInputProps } from '../type'; -import { onChangeNode } from '../../../../FlowProvider'; +import { getFlowStore, onChangeNode, useFlowProviderStoreType } from '../../../../FlowProvider'; import { Box, Button, Flex, Grid, useDisclosure, useTheme } from '@chakra-ui/react'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { SelectedDatasetType } from '@fastgpt/global/core/module/api'; import Avatar from '@/components/Avatar'; -import DatasetSelectModal from '@/components/core/module/DatasetSelectModal'; import { useQuery } from '@tanstack/react-query'; +import { useTranslation } from 'next-i18next'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; +import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; +import { chatModelList } from '@/web/common/system/staticData'; -const SelectDatasetRender = ({ item, moduleId }: RenderInputProps) => { +import dynamic from 'next/dynamic'; +import MyIcon from '@fastgpt/web/components/common/Icon'; + +const DatasetSelectModal = dynamic(() => import('@/components/core/module/DatasetSelectModal')); +const DatasetParamsModal = dynamic(() => import('@/components/core/module/DatasetParamsModal')); + +const SelectDatasetRender = ({ inputs = [], item, moduleId }: RenderInputProps) => { + const { t } = useTranslation(); const theme = useTheme(); + const [nodes, setNodes] = useState([]); + const [data, setData] = useState({ + searchMode: DatasetSearchModeEnum.embedding, + limit: 5, + similarity: 0.5, + usingReRank: false + }); + const { allDatasets, loadAllDatasets } = useDatasetStore(); const { - isOpen: isOpenKbSelect, - onOpen: onOpenKbSelect, - onClose: onCloseKbSelect + isOpen: isOpenDatasetSelect, + onOpen: onOpenDatasetSelect, + onClose: onCloseDatasetSelect } = useDisclosure(); const selectedDatasets = useMemo(() => { @@ -22,14 +41,68 @@ const SelectDatasetRender = ({ item, moduleId }: RenderInputProps) => { return allDatasets.filter((dataset) => value?.find((item) => item.datasetId === dataset._id)); }, [allDatasets, item.value]); + const tokenLimit = useMemo(() => { + let maxTokens = 3000; + + nodes.forEach((item) => { + if (item.type === FlowNodeTypeEnum.chatNode) { + const model = + item.data.inputs.find((item) => item.key === ModuleInputKeyEnum.aiModel)?.value || ''; + const quoteMaxToken = + chatModelList.find((item) => item.model === model)?.quoteMaxToken || 3000; + + maxTokens = Math.max(maxTokens, quoteMaxToken); + } + }); + + return maxTokens; + }, [nodes]); + + const { + isOpen: isOpenDatasetPrams, + onOpen: onOpenDatasetParams, + onClose: onCloseDatasetParams + } = useDisclosure(); + useQuery(['loadAllDatasets'], loadAllDatasets); + useEffect(() => { + inputs.forEach((input) => { + // @ts-ignore + if (data[input.key] !== undefined) { + setData((state) => ({ + ...state, + [input.key]: input.value + })); + } + }); + }, [inputs]); + + useEffect(() => { + async () => { + const { nodes } = await getFlowStore(); + setNodes(nodes); + }; + }, []); + return ( <> - + {/* } + onClick={onOpenDatasetParams} + > + {t('core.dataset.search.Params Setting')} + */} {selectedDatasets.map((item) => ( { ))} - {isOpenKbSelect && ( + {isOpenDatasetSelect && ( { onChangeNode({ @@ -68,9 +141,32 @@ const SelectDatasetRender = ({ item, moduleId }: RenderInputProps) => { } }); }} - onClose={onCloseKbSelect} + onClose={onCloseDatasetSelect} /> )} + {/* {isOpenDatasetPrams && ( + { + for (let key in e) { + const item = inputs.find((input) => input.key === key); + if (!item) continue; + onChangeNode({ + moduleId, + type: 'updateInput', + key, + value: { + ...item, + //@ts-ignore + value: e[key] + } + }); + } + }} + /> + )} */} ); }; diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx index 38fc9c138..e31f2ed1a 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/SelectDatasetParams.tsx @@ -1,9 +1,9 @@ import React, { useEffect, useMemo, useState } from 'react'; import type { RenderInputProps } from '../type'; -import { getFlowStore, onChangeNode, useFlowProviderStoreType } from '../../../../FlowProvider'; +import { onChangeNode, useFlowProviderStore } from '../../../../FlowProvider'; import { Button, useDisclosure } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; -import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; import { chatModelList } from '@/web/common/system/staticData'; @@ -11,7 +11,7 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import DatasetParamsModal from '@/components/core/module/DatasetParamsModal'; const SelectDatasetParam = ({ inputs = [], moduleId }: RenderInputProps) => { - const [nodes, setNodes] = useState([]); + const { nodes } = useFlowProviderStore(); const { t } = useTranslation(); const [data, setData] = useState({ @@ -52,47 +52,44 @@ const SelectDatasetParam = ({ inputs = [], moduleId }: RenderInputProps) => { }); }, [inputs]); - useEffect(() => { - async () => { - const { nodes } = await getFlowStore(); - setNodes(nodes); - }; - }, []); + const Render = useMemo(() => { + return ( + <> + } + onClick={onOpen} + > + {t('core.dataset.search.Params Setting')} + + {isOpen && ( + { + for (let key in e) { + const item = inputs.find((input) => input.key === key); + if (!item) continue; + onChangeNode({ + moduleId, + type: 'updateInput', + key, + value: { + ...item, + //@ts-ignore + value: e[key] + } + }); + } + }} + /> + )} + + ); + }, [data, inputs, isOpen, moduleId, onClose, onOpen, t, tokenLimit]); - return ( - <> - } - onClick={onOpen} - > - {t('core.dataset.search.Params Setting')} - - {isOpen && ( - { - for (let key in e) { - const item = inputs.find((input) => input.key === key); - if (!item) continue; - onChangeNode({ - moduleId, - type: 'updateInput', - key, - value: { - ...item, - //@ts-ignore - value: e[key] - } - }); - } - }} - /> - )} - - ); + return Render; }; export default React.memo(SelectDatasetParam); diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx index 9bffa9ece..af5a0e040 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/Textarea.tsx @@ -1,38 +1,53 @@ -import React, { useCallback } from 'react'; +import React, { useCallback, useMemo, useTransition } from 'react'; import type { RenderInputProps } from '../type'; -import { onChangeNode } from '../../../../FlowProvider'; +import { useFlowProviderStore, onChangeNode } from '../../../../FlowProvider'; import { useTranslation } from 'next-i18next'; -import PromptTextarea from '@/components/common/Textarea/PromptTextarea'; +import PromptEditor from '@fastgpt/web/components/common/Textarea/PromptEditor'; +import { + formatVariablesIcon, + getGuideModule, + splitGuideModule +} from '@fastgpt/global/core/module/utils'; const TextareaRender = ({ item, moduleId }: RenderInputProps) => { const { t } = useTranslation(); + const [, startTst] = useTransition(); + const { nodes } = useFlowProviderStore(); - const update = useCallback( - (value: string) => { - onChangeNode({ - moduleId, - type: 'updateInput', - key: item.key, - value: { - ...item, - value - } + // get variable + const variables = useMemo( + () => + formatVariablesIcon( + splitGuideModule(getGuideModule(nodes.map((node) => node.data)))?.variableModules || [] + ), + [nodes] + ); + + const onChange = useCallback( + (e: string) => { + startTst(() => { + onChangeNode({ + moduleId, + type: 'updateInput', + key: item.key, + value: { + ...item, + value: e + } + }); }); }, [item, moduleId] ); return ( - { - update(e.target.value); - }} + onChange={onChange} /> ); }; diff --git a/projects/app/src/components/core/module/Flow/index.tsx b/projects/app/src/components/core/module/Flow/index.tsx index fb66b0ebe..a8a04f4a7 100644 --- a/projects/app/src/components/core/module/Flow/index.tsx +++ b/projects/app/src/components/core/module/Flow/index.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from 'react'; +import React, { useMemo } from 'react'; import ReactFlow, { Background, Controls, ReactFlowProvider } from 'reactflow'; import { Box, Flex, IconButton, useDisclosure } from '@chakra-ui/react'; import { SmallCloseIcon } from '@chakra-ui/icons'; @@ -15,7 +15,6 @@ import 'reactflow/dist/style.css'; const NodeSimple = dynamic(() => import('./components/nodes/NodeSimple')); const nodeTypes: Record<`${FlowNodeTypeEnum}`, any> = { [FlowNodeTypeEnum.userGuide]: dynamic(() => import('./components/nodes/NodeUserGuide')), - [FlowNodeTypeEnum.variable]: dynamic(() => import('./components/nodes/abandon/NodeVariable')), [FlowNodeTypeEnum.questionInput]: dynamic(() => import('./components/nodes/NodeQuestionInput')), [FlowNodeTypeEnum.historyNode]: NodeSimple, [FlowNodeTypeEnum.chatNode]: NodeSimple, @@ -38,6 +37,16 @@ const Container = React.memo(function Container() { const { reactFlowWrapper, nodes, onNodesChange, edges, onEdgesChange, onConnect } = useFlowProviderStore(); + const memoRenderTools = useMemo( + () => ( + <> + + + + ), + [] + ); + return ( - - + {memoRenderTools} ); }); @@ -81,48 +89,54 @@ const Flow = ({ onClose: onCloseTemplate } = useDisclosure(); + const memoRenderContainer = useMemo(() => { + return ( + { + e.preventDefault(); + return false; + }} + > + {/* open module template */} + } + transform={isOpenTemplate ? '' : 'rotate(135deg)'} + transition={'0.2s ease'} + aria-label={''} + zIndex={1} + boxShadow={'2px 2px 6px #85b1ff'} + onClick={() => { + isOpenTemplate ? onCloseTemplate() : onOpenTemplate(); + }} + /> + + + + + + ); + }, [data, isOpenTemplate, onCloseTemplate, onOpenTemplate, templates]); + return ( {Header} - { - e.preventDefault(); - return false; - }} - > - {/* open module template */} - } - transform={isOpenTemplate ? '' : 'rotate(135deg)'} - transition={'0.2s ease'} - aria-label={''} - zIndex={1} - boxShadow={'2px 2px 6px #85b1ff'} - onClick={() => { - isOpenTemplate ? onCloseTemplate() : onOpenTemplate(); - }} - /> - - - - - + {memoRenderContainer} diff --git a/projects/app/src/components/support/apikey/Table.tsx b/projects/app/src/components/support/apikey/Table.tsx index 6f751e176..16cb5edaa 100644 --- a/projects/app/src/components/support/apikey/Table.tsx +++ b/projects/app/src/components/support/apikey/Table.tsx @@ -81,7 +81,7 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { - API 秘钥管理 + {t('support.openapi.Api manager')} {feConfigs?.docUrl && ( { ml={1} color={'primary.500'} > - 查看文档 + {t('common.Read document')} )} @@ -106,10 +106,10 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { borderRadius={'md'} cursor={'pointer'} userSelect={'none'} - onClick={() => copyData(baseUrl, '已复制 API 地址')} + onClick={() => copyData(baseUrl, t('support.openapi.Copy success'))} > - API根地址 + {t('support.openapi.Api baseurl')} {baseUrl} @@ -127,7 +127,7 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { }) } > - 新建 + {t('common.New Create')} @@ -137,16 +137,16 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => {
{t('Name')} Api Key - 已用额度(￥) + {t('support.openapi.Usage')} {feConfigs?.isPlus && ( <> - 最大额度(￥) - 过期时间 + {t('support.openapi.Max usage')} + {t('common.Expired Time')} )} - 创建时间 - 最后一次使用时间 + {t('common.Create Time')} + {t('common.Last use time')}
@@ -158,7 +158,11 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { {usage} {feConfigs?.isPlus && ( <> - {limit?.credit && limit?.credit > -1 ? `${limit?.credit}` : '无限制'} + + {limit?.credit && limit?.credit > -1 + ? `${limit?.credit}` + : t('common.Unlimited')} + {limit?.expiredTime ? dayjs(limit?.expiredTime).format('YYYY/MM/DD\nHH:mm') @@ -168,7 +172,9 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { )} {dayjs(createTime).format('YYYY/MM/DD\nHH:mm:ss')} - {lastUsedTime ? dayjs(lastUsedTime).format('YYYY/MM/DD\nHH:mm:ss') : '没有使用过'} + {lastUsedTime + ? dayjs(lastUsedTime).format('YYYY/MM/DD\nHH:mm:ss') + : t('common.Un used')} @@ -229,10 +235,10 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => { title={ - 新的 API 秘钥 + {t('support.openapi.New api key')} - 请保管好你的秘钥，秘钥不会再次展示~ + {t('support.openapi.New api key tip')} } @@ -359,14 +365,14 @@ function EditKeyModal({ diff --git a/projects/app/src/components/support/wallet/Price.tsx b/projects/app/src/components/support/wallet/Price.tsx index 6182da4b1..7a3104ed3 100644 --- a/projects/app/src/components/support/wallet/Price.tsx +++ b/projects/app/src/components/support/wallet/Price.tsx @@ -33,20 +33,16 @@ ${chatModelList md: ` | 模型 | 价格(￥) | | --- | --- | -${vectorModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k tokens |`).join('\n')} +${vectorModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k 字符 |`).join('\n')} ` }, { title: '文件预处理模型(QA 拆分)', describe: '', md: ` -| 模型 | 输入价格(￥) | 输出价格(￥) | -| --- | --- | --- | -${qaModelList - ?.map( - (item) => `| ${item.name} | ${item.inputPrice}/1k tokens | ${item.outputPrice}/1k tokens |` - ) - .join('\n')} +| 模型 | 价格(￥) | +| --- | --- | +${qaModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k 字符 |`).join('\n')} ` }, { @@ -84,14 +80,6 @@ ${qgModelList (item) => `| ${item.name} | ${item.inputPrice}/1k tokens | ${item.outputPrice}/1k tokens |` ) .join('\n')}` - }, - { - title: '重排模型(增强检索 & 混合检索)', - describe: '', - md: ` -| 模型 | 价格(￥) | -| --- | --- | -${reRankModelList?.map((item) => `| ${item.name} | ${item.inputPrice}/1k 字符 |`).join('\n')}` }, { title: '语音播放', diff --git a/projects/app/src/global/core/api/datasetReq.d.ts b/projects/app/src/global/core/api/datasetReq.d.ts index 6cad6a2f8..960f56feb 100644 --- a/projects/app/src/global/core/api/datasetReq.d.ts +++ b/projects/app/src/global/core/api/datasetReq.d.ts @@ -2,9 +2,9 @@ import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetTypeEnum -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import type { RequestPaging } from '@/types'; -import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import type { SearchTestItemType } from '@/types/core/dataset'; import { UploadChunkItemType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; diff --git a/projects/app/src/global/core/app/constants.ts b/projects/app/src/global/core/app/constants.ts index 0eb6f7bdf..b5392e6b2 100644 --- a/projects/app/src/global/core/app/constants.ts +++ b/projects/app/src/global/core/app/constants.ts @@ -1,10 +1,10 @@ import { AppSimpleEditConfigTemplateType } from '@fastgpt/global/core/app/type.d'; -import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; export const SimpleModeTemplate_FastGPT_Universal: AppSimpleEditConfigTemplateType = { id: 'fastgpt-universal', - name: '通用模板', - desc: '通用模板\n可完全自行配置AI属性和知识库', + name: 'core.app.template.Common template', + desc: 'core.app.template.Common template tip', systemForm: { aiSettings: { model: true, diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index 5ec3c0180..e942fe637 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -3,7 +3,7 @@ import { DatasetSearchModeEnum, DatasetTypeEnum, TrainingModeEnum -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import { DatasetDataIndexItemType, SearchDataResponseItemType diff --git a/projects/app/src/global/core/prompt/AIChat.ts b/projects/app/src/global/core/prompt/AIChat.ts index 2965df48d..b8c7a2690 100644 --- a/projects/app/src/global/core/prompt/AIChat.ts +++ b/projects/app/src/global/core/prompt/AIChat.ts @@ -53,8 +53,8 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [ 回答要求： - 如果你不清楚答案，你需要澄清。 -- 避免提及你是从 data 获取的知识。 -- 保持答案与 data 中描述的一致。 +- 避免提及你是从 获取的知识。 +- 保持答案与 中描述的一致。 - 使用 Markdown 语法优化回答格式。 - 使用与问题相同的语言回答。 @@ -88,8 +88,8 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [ 3. 如果无关，你直接拒绝回答本次问题。 回答要求： -- 避免提及你是从 data 获取的知识。 -- 保持答案与 data 中描述的一致。 +- 避免提及你是从 获取的知识。 +- 保持答案与 中描述的一致。 - 使用 Markdown 语法优化回答格式。 - 使用与问题相同的语言回答。 diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx index 428397ad7..b329486c1 100644 --- a/projects/app/src/pages/_app.tsx +++ b/projects/app/src/pages/_app.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from 'react'; +import { useEffect, useState } from 'react'; import type { AppProps } from 'next/app'; import Script from 'next/script'; import Head from 'next/head'; diff --git a/projects/app/src/pages/account/components/BillDetail.tsx b/projects/app/src/pages/account/components/BillDetail.tsx index 6ade1f896..c3b7b6cee 100644 --- a/projects/app/src/pages/account/components/BillDetail.tsx +++ b/projects/app/src/pages/account/components/BillDetail.tsx @@ -30,7 +30,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void hasTokens, hasInputTokens, hasOutputTokens, - hasTextLen, + hasCharsLen, hasDuration, hasDataLen } = useMemo(() => { @@ -38,7 +38,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void let hasTokens = false; let hasInputTokens = false; let hasOutputTokens = false; - let hasTextLen = false; + let hasCharsLen = false; let hasDuration = false; let hasDataLen = false; @@ -46,24 +46,21 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void if (item.model !== undefined) { hasModel = true; } - if (item.tokenLen !== undefined) { + if (typeof item.tokenLen === 'number') { hasTokens = true; } - if (item.inputTokens !== undefined) { + if (typeof item.inputTokens === 'number') { hasInputTokens = true; } - if (item.outputTokens !== undefined) { + if (typeof item.outputTokens === 'number') { hasOutputTokens = true; } - if (item.textLen !== undefined) { - hasTextLen = true; + if (typeof item.charsLength === 'number') { + hasCharsLen = true; } - if (item.duration !== undefined) { + if (typeof item.duration === 'number') { hasDuration = true; } - if (item.dataLen !== undefined) { - hasDataLen = true; - } }); return { @@ -71,7 +68,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void hasTokens, hasInputTokens, hasOutputTokens, - hasTextLen, + hasCharsLen, hasDuration, hasDataLen }; @@ -123,9 +120,8 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void {hasTokens && {t('wallet.bill.Token Length')}} {hasInputTokens && {t('wallet.bill.Input Token Length')}} {hasOutputTokens && {t('wallet.bill.Output Token Length')}} - {hasTextLen && {t('wallet.bill.Text Length')}} + {hasCharsLen && {t('wallet.bill.Text Length')}} {hasDuration && {t('wallet.bill.Duration')}} - {hasDataLen && {t('wallet.bill.Data Length')}} 费用(￥) @@ -137,10 +133,8 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void {hasTokens && {item.tokenLen ?? '-'}} {hasInputTokens && {item.inputTokens ?? '-'}} {hasOutputTokens && {item.outputTokens ?? '-'}} - {hasTextLen && {item.textLen ?? '-'}} + {hasCharsLen && {item.charsLength ?? '-'}} {hasDuration && {item.duration ?? '-'}} - {hasDataLen && {item.dataLen ?? '-'}} - {formatStorePrice2Read(item.amount)} ))} diff --git a/projects/app/src/pages/account/components/Info.tsx b/projects/app/src/pages/account/components/Info.tsx index b2f1743ce..5ee0f63bd 100644 --- a/projects/app/src/pages/account/components/Info.tsx +++ b/projects/app/src/pages/account/components/Info.tsx @@ -108,12 +108,12 @@ const UserInfo = () => { }); } catch (err: any) { toast({ - title: typeof err === 'string' ? err : '头像选择异常', + title: typeof err === 'string' ? err : t('common.error.Select avatar failed'), status: 'warning' }); } }, - [onclickSave, toast, userInfo] + [onclickSave, t, toast, userInfo] ); useQuery(['init'], initUserInfo, { diff --git a/projects/app/src/pages/api/admin/initv46-2.ts b/projects/app/src/pages/api/admin/initv46-2.ts deleted file mode 100644 index e99de6a22..000000000 --- a/projects/app/src/pages/api/admin/initv46-2.ts +++ /dev/null @@ -1,184 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; -import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant'; -import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; - -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { defaultQAModels } from '@fastgpt/global/core/ai/model'; - -let success = 0; -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { limit = 50 } = req.body as { limit: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - success = 0; - - try { - await Promise.allSettled([ - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN data_id VARCHAR(50);`), - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN q DROP NOT NULL;`), // q can null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN a DROP NOT NULL;`), // a can null - PgClient.query( - `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id TYPE VARCHAR(50) USING team_id::VARCHAR(50);` - ), // team_id varchar - PgClient.query( - `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id TYPE VARCHAR(50) USING tmb_id::VARCHAR(50);` - ), // tmb_id varchar - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id SET NOT NULL;`), // team_id not null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id SET NOT NULL;`), // tmb_id not null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN dataset_id SET NOT NULL;`), // dataset_id not null - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN collection_id SET NOT NULL;`) // collection_id not null - ]); - } catch (error) {} - - try { - await initPgData(); - } catch (error) {} - - await MongoDataset.updateMany( - {}, - { - agentModel: defaultQAModels[0].model - } - ); - - jsonRes(res, { - data: await init(limit), - message: - '初始化任务已开始，请注意日志进度。可通过 select count(id) from modeldata where data_id is null; 检查是否完全初始化，如果结果为 0 ，则完全初始化。' - }); - } catch (error) { - console.log(error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -type PgItemType = { - id: string; - q: string; - a: string; - dataset_id: string; - collection_id: string; - team_id: string; - tmb_id: string; -}; - -async function initPgData() { - const limit = 10; - const { rows } = await PgClient.query<{ user_id: string }>(` - SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id='null'; -`); - console.log('init pg', rows.length); - let success = 0; - for (let i = 0; i < limit; i++) { - init(i); - } - - async function init(index: number): Promise { - const userId = rows[index]?.user_id; - if (!userId) return; - try { - const tmb = await getUserDefaultTeam({ userId }); - console.log(tmb); - - // update pg - await PgClient.query( - `Update ${PgDatasetTableName} set team_id = '${String(tmb.teamId)}', tmb_id = '${String( - tmb.tmbId - )}' where user_id = '${userId}' AND team_id='null';` - ); - console.log(++success); - init(index + limit); - } catch (error) { - if (error === 'default team not exist') { - return; - } - console.log(error); - await delay(1000); - return init(index); - } - } -} - -async function init(limit: number): Promise { - const { rows: idList } = await PgClient.query<{ id: string }>( - `SELECT id FROM ${PgDatasetTableName} WHERE data_id IS NULL` - ); - - console.log('totalCount', idList.length); - if (idList.length === 0) return; - - for (let i = 0; i < limit; i++) { - initData(i); - } - - async function initData(index: number): Promise { - const dataId = idList[index]?.id; - if (!dataId) { - console.log('done'); - return; - } - // get limit data where data_id is null - const { rows } = await PgClient.query( - `SELECT id,q,a,dataset_id,collection_id,team_id,tmb_id FROM ${PgDatasetTableName} WHERE id=${dataId};` - ); - const data = rows[0]; - if (!data) { - console.log('done'); - return; - } - - let id = ''; - try { - // create mongo data and update data_id - const { _id } = await MongoDatasetData.create({ - teamId: data.team_id.trim(), - tmbId: data.tmb_id.trim(), - datasetId: data.dataset_id, - collectionId: data.collection_id, - q: data.q, - a: data.a, - fullTextToken: '', - indexes: [ - { - defaultIndex: !data.a, - type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk, - dataId: data.id, - text: data.q - } - ] - }); - id = _id; - // update pg data_id - await PgClient.query( - `UPDATE ${PgDatasetTableName} SET data_id='${String(_id)}' WHERE id=${dataId};` - ); - - console.log(++success); - return initData(index + limit); - } catch (error) { - console.log(error); - console.log(data); - - try { - if (id) { - await MongoDatasetData.findByIdAndDelete(id); - } - } catch (error) {} - await delay(500); - return initData(index); - } - } -} diff --git a/projects/app/src/pages/api/admin/initv46-fix.ts b/projects/app/src/pages/api/admin/initv46-fix.ts deleted file mode 100644 index 8ddd8e070..000000000 --- a/projects/app/src/pages/api/admin/initv46-fix.ts +++ /dev/null @@ -1,173 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; -import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; - -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { Types, connectionMongo } from '@fastgpt/service/common/mongo'; -import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; -import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller'; -import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller'; - -let success = 0; -/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { limit = 50 } = req.body as { limit: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - success = 0; - - await init(limit); - await initCollectionFileTeam(limit); - - jsonRes(res, {}); - } catch (error) { - console.log(error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -type PgItemType = { - id: string; - q: string; - a: string; - dataset_id: string; - collection_id: string; - data_id: string; -}; - -async function init(limit: number): Promise { - const { rows } = await PgClient.query<{ id: string; data_id: string }>( - `SELECT id,data_id FROM ${PgDatasetTableName} WHERE team_id = tmb_id` - ); - - console.log('totalCount', rows.length); - - await delay(2000); - - if (rows.length === 0) return; - - for (let i = 0; i < limit; i++) { - initData(i); - } - - async function initData(index: number): Promise { - const item = rows[index]; - if (!item) { - console.log('done'); - return; - } - // get mongo - const mongoData = await MongoDatasetData.findById(item.data_id, '_id teamId tmbId'); - if (!mongoData) { - return initData(index + limit); - } - - try { - // find team owner - const db = connectionMongo?.connection?.db; - const TeamMember = db.collection(TeamMemberCollectionName); - - const tmb = await TeamMember.findOne({ - teamId: new Types.ObjectId(mongoData.teamId), - role: 'owner' - }); - - if (!tmb) { - return initData(index + limit); - } - - // update mongo and pg tmb_id - await MongoDatasetData.findByIdAndUpdate(item.data_id, { - tmbId: tmb._id - }); - await PgClient.query( - `UPDATE ${PgDatasetTableName} SET tmb_id = '${String(tmb._id)}' WHERE id = '${item.id}'` - ); - - console.log(++success); - - return initData(index + limit); - } catch (error) { - console.log(error); - await delay(500); - return initData(index); - } - } -} - -async function initCollectionFileTeam(limit: number) { - /* init user default Team */ - const DatasetFile = getGFSCollection('dataset'); - const matchWhere = { - $or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }] - }; - const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([ - { - $match: matchWhere - }, - { - $group: { - _id: '$metadata.userId', // 按 metadata.userId 分组以去重 - userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId - } - }, - { - $project: { - _id: 0, // 不显示 _id 字段 - userId: 1 // 只显示 userId 字段 - } - } - ]).toArray(); - const users = uniqueUsersWithNoTeamId; - - console.log('un init total', users.length); - // limit 组一次 - const userArr: any[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - - let success = 0; - for await (const item of userArr) { - await Promise.all(item.map((item) => init(item.userId))); - success += limit; - console.log(success); - } - - async function init(userId: string): Promise { - try { - const tmb = await getUserDefaultTeam({ - userId - }); - - await DatasetFile.updateMany( - { - 'metadata.userId': String(userId), - ...matchWhere - }, - { - $set: { - 'metadata.teamId': String(tmb.teamId), - 'metadata.tmbId': String(tmb.tmbId) - } - } - ); - } catch (error) { - if (error === 'team not exist' || error === 'tmbId or userId is required') { - return; - } - console.log(error); - await delay(1000); - return init(userId); - } - } -} diff --git a/projects/app/src/pages/api/admin/initv46.ts b/projects/app/src/pages/api/admin/initv46.ts deleted file mode 100644 index 4af44b89a..000000000 --- a/projects/app/src/pages/api/admin/initv46.ts +++ /dev/null @@ -1,330 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema'; -import { - createDefaultTeam, - getUserDefaultTeam -} from '@fastgpt/service/support/user/team/controller'; -import { MongoUser } from '@fastgpt/service/support/user/schema'; -import { UserModelSchema } from '@fastgpt/global/support/user/type'; -import { delay } from '@fastgpt/global/common/system/utils'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; -import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; -import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; -import { MongoOutLink } from '@fastgpt/service/support/outLink/schema'; -import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema'; -import { MongoApp } from '@fastgpt/service/core/app/schema'; -import { MongoChat } from '@fastgpt/service/core/chat/chatSchema'; -import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; -import { MongoPlugin } from '@fastgpt/service/core/plugin/schema'; -import { POST } from '@fastgpt/service/common/api/plusRequest'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller'; -import { FastGPTProUrl } from '@fastgpt/service/common/system/constants'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { limit = 50, maxSize = 3 } = req.body as { limit: number; maxSize: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - - await initDefaultTeam(limit, maxSize); - await initMongoTeamId(limit); - await initDatasetAndApp(); - await initCollectionFileTeam(limit); - - if (FastGPTProUrl) { - POST('/admin/init46'); - } - - await initPgData(); - - jsonRes(res, { - data: {} - }); - } catch (error) { - console.log(error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -async function initDefaultTeam(limit: number, maxSize: number) { - /* init user default Team */ - const users = await MongoUser.find({}, '_id balance'); - console.log('init user default team', users.length); - // 100 组一次 - const userArr: UserModelSchema[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - let success = 0; - for await (const users of userArr) { - await Promise.all(users.map(init)); - success += limit; - console.log(success); - } - - async function init(user: UserModelSchema): Promise { - try { - await createDefaultTeam({ - userId: user._id, - balance: user.balance, - maxSize - }); - } catch (error) { - console.log(error); - - await delay(1000); - return init(user); - } - } -} -async function initMongoTeamId(limit: number) { - const mongoSchema = [ - { - label: 'MongoPlugin', - schema: MongoPlugin - }, - { - label: 'MongoChat', - schema: MongoChat - }, - { - label: 'MongoChatItem', - schema: MongoChatItem - }, - { - label: 'MongoApp', - schema: MongoApp - }, - { - label: 'MongoDataset', - schema: MongoDataset - }, - { - label: 'MongoDatasetCollection', - schema: MongoDatasetCollection - }, - { - label: 'MongoDatasetTraining', - schema: MongoDatasetTraining - }, - { - label: 'MongoBill', - schema: MongoBill - }, - { - label: 'MongoOutLink', - schema: MongoOutLink - }, - { - label: 'MongoOpenApi', - schema: MongoOpenApi - } - ]; - /* init user default Team */ - - for await (const item of mongoSchema) { - console.log('start init', item.label); - await initTeamTmbId(item.schema); - console.log('finish init', item.label); - } - - async function initTeamTmbId(schema: any) { - const emptyWhere = { - $or: [{ teamId: { $exists: false } }, { teamId: null }] - }; - const uniqueUsersWithNoTeamId = await schema.aggregate([ - { - $match: emptyWhere - }, - { - $group: { - _id: '$userId', // 按 userId 分组以去重 - userId: { $first: '$userId' } // 保留第一个出现的 userId - } - }, - { - $project: { - _id: 0, // 不显示 _id 字段 - userId: 1 // 只显示 userId 字段 - } - } - ]); - const users = uniqueUsersWithNoTeamId; - - console.log('un init total', users.length); - // limit 组一次 - const userArr: any[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - - let success = 0; - for await (const users of userArr) { - await Promise.all(users.map((item) => init(item.userId))); - success += limit; - console.log(success); - } - - async function init(userId: string): Promise { - try { - const tmb = await getUserDefaultTeam({ userId }); - - await schema.updateMany( - { - userId, - ...emptyWhere - }, - { - teamId: tmb.teamId, - tmbId: tmb.tmbId - } - ); - } catch (error) { - if (error === 'team not exist' || error === 'tmbId or userId is required') { - return; - } - console.log(error); - await delay(1000); - return init(userId); - } - } - } -} -async function initDatasetAndApp() { - await MongoDataset.updateMany( - {}, - { - $set: { - permission: PermissionTypeEnum.private - } - } - ); - await MongoApp.updateMany( - {}, - { - $set: { - permission: PermissionTypeEnum.private - } - } - ); -} -async function initCollectionFileTeam(limit: number) { - /* init user default Team */ - const DatasetFile = getGFSCollection('dataset'); - const matchWhere = { - $or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }] - }; - const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([ - { - $match: matchWhere - }, - { - $group: { - _id: '$metadata.userId', // 按 metadata.userId 分组以去重 - userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId - } - }, - { - $project: { - _id: 0, // 不显示 _id 字段 - userId: 1 // 只显示 userId 字段 - } - } - ]).toArray(); - const users = uniqueUsersWithNoTeamId; - - console.log('un init total', users.length); - // limit 组一次 - const userArr: any[][] = []; - for (let i = 0; i < users.length; i += limit) { - userArr.push(users.slice(i, i + limit)); - } - - let success = 0; - for await (const item of userArr) { - await Promise.all(item.map((item) => init(item.userId))); - success += limit; - console.log(success); - } - - async function init(userId: string): Promise { - try { - const tmb = await getUserDefaultTeam({ - userId - }); - - await DatasetFile.updateMany( - { - 'metadata.userId': String(userId), - ...matchWhere - }, - { - $set: { - 'metadata.teamId': String(tmb.teamId), - 'metadata.tmbId': String(tmb.tmbId) - } - } - ); - } catch (error) { - if (error === 'team not exist' || error === 'tmbId or userId is required') { - return; - } - console.log(error); - await delay(1000); - return init(userId); - } - } -} -async function initPgData() { - const limit = 10; - // add column - try { - await Promise.allSettled([ - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id VARCHAR(50);`), - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id VARCHAR(50);`), - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`) - ]); - } catch (error) { - console.log(error); - console.log('column exists'); - } - - const { rows } = await PgClient.query<{ user_id: string }>(` - SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id IS NULL; -`); - console.log('init pg', rows.length); - let success = 0; - for (let i = 0; i < limit; i++) { - init(i); - } - async function init(index: number): Promise { - const userId = rows[index]?.user_id; - if (!userId) return; - try { - const tmb = await getUserDefaultTeam({ userId }); - // update pg - await PgClient.query( - `Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;` - ); - console.log(++success); - init(index + limit); - } catch (error) { - if (error === 'default team not exist') { - return; - } - console.log(error); - await delay(1000); - return init(index); - } - } -} diff --git a/projects/app/src/pages/api/admin/initv463.ts b/projects/app/src/pages/api/admin/initv463.ts index 446d4f69f..3142b98d0 100644 --- a/projects/app/src/pages/api/admin/initv463.ts +++ b/projects/app/src/pages/api/admin/initv463.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; let success = 0; diff --git a/projects/app/src/pages/api/admin/initv467.ts b/projects/app/src/pages/api/admin/initv467.ts new file mode 100644 index 000000000..e8cc89270 --- /dev/null +++ b/projects/app/src/pages/api/admin/initv467.ts @@ -0,0 +1,106 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { PgClient } from '@fastgpt/service/common/vectorStore/pg'; +import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants'; +import { MongoImage } from '@fastgpt/service/common/file/image/schema'; +import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type'; +import { delay } from '@fastgpt/global/common/system/utils'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; + +let success = 0; +let deleteImg = 0; +/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { test = false } = req.body as { test: boolean }; + await authCert({ req, authRoot: true }); + await connectToDatabase(); + success = 0; + deleteImg = 0; + + // 取消 pg tmb_id 和 data_id 的null + await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id DROP NOT NULL;`); + await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN data_id DROP NOT NULL;`); + + // 重新绑定 images 和 collections + const images = await MongoImage.find( + { 'metadata.fileId': { $exists: true } }, + '_id metadata' + ).lean(); + + // 去除 fileId 相同的数据 + const fileIdMap = new Map(); + images.forEach((image) => { + // @ts-ignore + const fileId = image.metadata?.fileId; + if (!fileIdMap.has(fileId) && fileId) { + fileIdMap.set(fileId, image); + } + }); + const images2 = Array.from(fileIdMap.values()); + + console.log('total image list', images2.length); + + for await (const image of images2) { + await initImages(image, test); + } + + jsonRes(res, { + data: success, + message: 'success' + }); + } catch (error) { + console.log(error); + + jsonRes(res, { + code: 500, + error + }); + } +} +export const initImages = async (image: MongoImageSchemaType, test: boolean): Promise => { + try { + //@ts-ignore + const fileId = image.metadata.fileId as string; + if (!fileId) return; + + // 找到集合 + const collection = await MongoDatasetCollection.findOne({ fileId }, '_id metadata').lean(); + + if (!collection) { + deleteImg++; + console.log('deleteImg', deleteImg); + + if (test) return; + return MongoImage.deleteOne({ _id: image._id }); + } + + const relatedImageId = getNanoid(24); + + // update image + if (!test) { + await Promise.all([ + MongoImage.updateMany( + { 'metadata.fileId': fileId }, + { $set: { 'metadata.relatedId': relatedImageId } } + ), + MongoDatasetCollection.findByIdAndUpdate(collection._id, { + $set: { + 'metadata.relatedImgId': relatedImageId + } + }) + ]); + } + + success++; + console.log('success', success); + } catch (error) { + console.log(error); + + await delay(1000); + return initImages(image, test); + } +}; diff --git a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts b/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts deleted file mode 100644 index 69e3d6bd8..000000000 --- a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { - delFileByFileIdList, - getGFSCollection -} from '@fastgpt/service/common/file/gridfs/controller'; -import { addLog } from '@fastgpt/service/common/system/log'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { delay } from '@fastgpt/global/common/system/utils'; - -/* - check dataset.files data. If there is no match in dataset.collections, delete it -*/ -let deleteFileAmount = 0; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { - startDay = 10, - endDay = 3, - limit = 30 - } = req.body as { startDay?: number; endDay?: number; limit?: number }; - await authCert({ req, authRoot: true }); - await connectToDatabase(); - - // start: now - maxDay, end: now - 3 day - const start = new Date(Date.now() - startDay * 24 * 60 * 60 * 1000); - const end = new Date(Date.now() - endDay * 24 * 60 * 60 * 1000); - deleteFileAmount = 0; - - checkFiles(start, end, limit); - - jsonRes(res, { - message: 'success' - }); - } catch (error) { - addLog.error(`check valid dataset files error`, error); - - jsonRes(res, { - code: 500, - error - }); - } -} - -export async function checkFiles(start: Date, end: Date, limit: number) { - const collection = getGFSCollection('dataset'); - const where = { - uploadDate: { $gte: start, $lte: end } - }; - - // 1. get all _id - const ids = await collection - .find(where, { - projection: { - _id: 1 - } - }) - .toArray(); - console.log('total files', ids.length); - - for (let i = 0; i < limit; i++) { - check(i); - } - - async function check(index: number): Promise { - const id = ids[index]; - if (!id) { - console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`); - - return; - } - try { - const { _id } = id; - - // 2. find fileId in dataset.collections - const hasCollection = await MongoDatasetCollection.countDocuments({ fileId: _id }); - - // 3. if not found, delete file - if (hasCollection === 0) { - await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] }); - console.log('delete file', _id); - deleteFileAmount++; - } - index % 100 === 0 && console.log(index); - return check(index + limit); - } catch (error) { - console.log(error); - await delay(2000); - return check(index); - } - } -} diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts index 76101589d..3f3d135e5 100644 --- a/projects/app/src/pages/api/common/file/upload.ts +++ b/projects/app/src/pages/api/common/file/upload.ts @@ -18,32 +18,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< try { const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); - const { files, bucketName, metadata } = await upload.doUpload(req, res); - - filePaths = files.map((file) => file.path); + const { file, bucketName, metadata } = await upload.doUpload(req, res); + filePaths = [file.path]; await connectToDatabase(); if (!bucketName) { throw new Error('bucketName is empty'); } - const upLoadResults = await Promise.all( - files.map((file) => - uploadFile({ - teamId, - tmbId, - bucketName, - path: file.path, - filename: file.originalname, - metadata: { - ...metadata, - contentType: file.mimetype, - userId - } - }) - ) - ); + const upLoadResults = await uploadFile({ + teamId, + tmbId, + bucketName, + path: file.path, + filename: file.originalname, + contentType: file.mimetype, + metadata: metadata + }); jsonRes(res, { data: upLoadResults diff --git a/projects/app/src/pages/api/common/system/getInitData.ts b/projects/app/src/pages/api/common/system/getInitData.ts index 8eb660f35..7bfe9c072 100644 --- a/projects/app/src/pages/api/common/system/getInitData.ts +++ b/projects/app/src/pages/api/common/system/getInitData.ts @@ -55,7 +55,8 @@ const defaultFeConfigs: FastGPTFeConfigsType = { websiteSyncLimitMinuted: 0 }, scripts: [], - favicon: '/favicon.ico' + favicon: '/favicon.ico', + uploadFileMaxSize: 500 }; export async function getInitConfig() { diff --git a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts index ff93cac97..6f2a469db 100644 --- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts +++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts @@ -7,7 +7,7 @@ import { jsonRes } from '@fastgpt/service/common/response'; import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d'; import type { ModuleItemType } from '@fastgpt/global/core/module/type'; import { FormatForm2ModulesProps } from '@fastgpt/global/core/app/api'; -import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants'; import { getExtractModel } from '@/service/core/ai/model'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -37,7 +37,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { return [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -49,7 +49,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -58,7 +58,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ @@ -93,7 +93,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -189,7 +189,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: @@ -268,7 +268,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { const modules: ModuleItemType[] = [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -280,7 +280,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -289,17 +289,13 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ { moduleId: 'vuc92c', key: 'userChatInput' - }, - { - moduleId: 'chatModule', - key: 'userChatInput' } ] } @@ -307,7 +303,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { }, { moduleId: 'datasetSearch', - name: '知识库搜索', + name: 'core.module.template.Dataset search', avatar: '/imgs/module/db.png', flowType: 'datasetSearchNode', showStatus: true, @@ -447,6 +443,18 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { valueType: 'boolean', type: 'source', targets: [] + }, + { + key: 'userChatInput', + label: 'core.module.input.label.user question', + type: 'hidden', + valueType: 'string', + targets: [ + { + moduleId: 'chatModule', + key: 'userChatInput' + } + ] } ] }, @@ -473,7 +481,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -569,7 +577,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: diff --git a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts index fa75ca327..140df7cb3 100644 --- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts +++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts @@ -33,7 +33,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { return [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -45,7 +45,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -54,7 +54,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ @@ -89,7 +89,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -185,7 +185,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: @@ -264,7 +264,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { const modules: ModuleItemType[] = [ { moduleId: 'userChatInput', - name: '用户问题(对话入口)', + name: 'core.module.template.Chat entrance', avatar: '/imgs/module/userChatInput.png', flowType: 'questionInput', position: { @@ -276,7 +276,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { key: 'userChatInput', type: 'systemInput', valueType: 'string', - label: '用户问题', + label: 'core.module.input.label.user question', showTargetInApp: false, showTargetInPlugin: false, connected: false @@ -285,17 +285,13 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { outputs: [ { key: 'userChatInput', - label: '用户问题', + label: 'core.module.input.label.user question', type: 'source', valueType: 'string', targets: [ { moduleId: 'vuc92c', key: 'userChatInput' - }, - { - moduleId: 'chatModule', - key: 'userChatInput' } ] } @@ -303,7 +299,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { }, { moduleId: 'datasetSearch', - name: '知识库搜索', + name: 'core.module.template.Dataset search', avatar: '/imgs/module/db.png', flowType: 'datasetSearchNode', showStatus: true, @@ -457,6 +453,18 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { valueType: 'boolean', type: 'source', targets: [] + }, + { + key: 'userChatInput', + label: 'core.module.input.label.user question', + type: 'hidden', + valueType: 'string', + targets: [ + { + moduleId: 'chatModule', + key: 'userChatInput' + } + ] } ] }, @@ -483,7 +491,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'model', type: 'selectChatModel', - label: '对话模型', + label: 'core.module.input.label.aiModel', required: true, valueType: 'string', showTargetInApp: false, @@ -579,7 +587,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] { { key: 'systemPrompt', type: 'textarea', - label: '系统提示词', + label: 'core.ai.Prompt', max: 300, valueType: 'string', description: diff --git a/projects/app/src/pages/api/core/app/getChatLogs.ts b/projects/app/src/pages/api/core/app/getChatLogs.ts index 66f198509..8e74264ee 100644 --- a/projects/app/src/pages/api/core/app/getChatLogs.ts +++ b/projects/app/src/pages/api/core/app/getChatLogs.ts @@ -8,6 +8,7 @@ import { Types } from '@fastgpt/service/common/mongo'; import { addDays } from 'date-fns'; import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d'; import { authApp } from '@fastgpt/service/support/permission/auth/app'; +import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -28,8 +29,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' }); const where = { - appId: new Types.ObjectId(appId), teamId: new Types.ObjectId(teamId), + appId: new Types.ObjectId(appId), updateTime: { $gte: new Date(dateStart), $lte: new Date(dateEnd) @@ -41,18 +42,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) { $match: where }, { $lookup: { - from: 'chatitems', - let: { chat_id: '$chatId' }, + from: ChatItemCollectionName, + let: { chatId: '$chatId' }, pipeline: [ { $match: { $expr: { $and: [ - { $eq: ['$chatId', '$$chat_id'] }, - { $eq: ['$appId', new Types.ObjectId(appId)] } + { $eq: ['$appId', new Types.ObjectId(appId)] }, + { $eq: ['$chatId', '$$chatId'] } ] } } + }, + { + $project: { + userGoodFeedback: 1, + userBadFeedback: 1, + customFeedbacks: 1, + adminFeedback: 1 + } } ], as: 'chatitems' diff --git a/projects/app/src/pages/api/core/chat/clearHistories.ts b/projects/app/src/pages/api/core/chat/clearHistories.ts index 05cbb1ca1..2c1ffc662 100644 --- a/projects/app/src/pages/api/core/chat/clearHistories.ts +++ b/projects/app/src/pages/api/core/chat/clearHistories.ts @@ -35,16 +35,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) return Promise.reject('Param are error'); })(); - console.log(match); // find chatIds const list = await MongoChat.find(match, 'chatId').lean(); const idList = list.map((item) => item.chatId); await MongoChatItem.deleteMany({ + appId, chatId: { $in: idList } }); await MongoChat.deleteMany({ + appId, chatId: { $in: idList } }); diff --git a/projects/app/src/pages/api/core/chat/delHistory.ts b/projects/app/src/pages/api/core/chat/delHistory.ts index 39baca624..d4623b49f 100644 --- a/projects/app/src/pages/api/core/chat/delHistory.ts +++ b/projects/app/src/pages/api/core/chat/delHistory.ts @@ -23,9 +23,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }); await MongoChatItem.deleteMany({ + appId, chatId }); await MongoChat.findOneAndRemove({ + appId, chatId }); diff --git a/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts b/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts index dcc8cac4a..36ae9ad5b 100644 --- a/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts +++ b/projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts @@ -27,6 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await MongoChatItem.findOneAndUpdate( { + appId, + chatId, dataId: chatItemId }, { diff --git a/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts b/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts index 9c3678d32..619a66b21 100644 --- a/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts +++ b/projects/app/src/pages/api/core/chat/feedback/closeCustom.ts @@ -2,14 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import type { - AdminUpdateFeedbackParams, - CloseCustomFeedbackParams -} from '@/global/core/chat/api.d'; +import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d'; import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; import { autChatCrud } from '@/service/support/permission/auth/chat'; -/* 初始化我的聊天框，需要身份验证 */ +/* remove custom feedback */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); @@ -29,13 +26,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await authCert({ req, authToken: true }); await MongoChatItem.findOneAndUpdate( - { dataId: chatItemId }, + { appId, chatId, dataId: chatItemId }, { $unset: { [`customFeedbacks.${index}`]: 1 } } ); - await MongoChatItem.findOneAndUpdate( - { dataId: chatItemId }, - { $pull: { customFeedbacks: null } } - ); jsonRes(res); } catch (err) { diff --git a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts index d1655e738..6cdd98675 100644 --- a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts +++ b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts @@ -29,6 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await MongoChatItem.findOneAndUpdate( { + appId, chatId, dataId: chatItemId }, diff --git a/projects/app/src/pages/api/core/chat/getHistories.ts b/projects/app/src/pages/api/core/chat/getHistories.ts index fe68beea8..c81f8cae6 100644 --- a/projects/app/src/pages/api/core/chat/getHistories.ts +++ b/projects/app/src/pages/api/core/chat/getHistories.ts @@ -31,8 +31,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) if (appId) { const { tmbId } = await authCert({ req, authToken: true }); return { - appId, tmbId, + appId, source: ChatSourceEnum.online }; } diff --git a/projects/app/src/pages/api/core/chat/init.ts b/projects/app/src/pages/api/core/chat/init.ts index 9f759f0ee..4bbdace8f 100644 --- a/projects/app/src/pages/api/core/chat/init.ts +++ b/projects/app/src/pages/api/core/chat/init.ts @@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) appId, per: 'r' }), - chatId ? MongoChat.findOne({ chatId }) : undefined + chatId ? MongoChat.findOne({ appId, chatId }) : undefined ]); // auth chat permission @@ -41,6 +41,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) // get app and history const { history } = await getChatItems({ + appId, chatId, limit: 30, field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${ diff --git a/projects/app/src/pages/api/core/chat/item/delete.ts b/projects/app/src/pages/api/core/chat/item/delete.ts index 7a1b8c043..6d7318707 100644 --- a/projects/app/src/pages/api/core/chat/item/delete.ts +++ b/projects/app/src/pages/api/core/chat/item/delete.ts @@ -25,8 +25,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }); await MongoChatItem.deleteOne({ - dataId: contentId, - chatId + appId, + chatId, + dataId: contentId }); jsonRes(res); diff --git a/projects/app/src/pages/api/core/chat/item/getSpeech.ts b/projects/app/src/pages/api/core/chat/item/getSpeech.ts index 4c8629a19..17d9a647b 100644 --- a/projects/app/src/pages/api/core/chat/item/getSpeech.ts +++ b/projects/app/src/pages/api/core/chat/item/getSpeech.ts @@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) try { pushAudioSpeechBill({ model: model, - textLen: input.length, + charsLength: input.length, tmbId, teamId, source: authType2BillSource({ authType }) diff --git a/projects/app/src/pages/api/core/chat/outLink/init.ts b/projects/app/src/pages/api/core/chat/outLink/init.ts index 134203527..27bb33138 100644 --- a/projects/app/src/pages/api/core/chat/outLink/init.ts +++ b/projects/app/src/pages/api/core/chat/outLink/init.ts @@ -26,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) // auth app permission const [tmb, chat, app] = await Promise.all([ MongoTeamMember.findById(shareChat.tmbId, '_id userId').populate('userId', 'avatar').lean(), - MongoChat.findOne({ chatId, shareId }).lean(), + MongoChat.findOne({ appId, chatId, shareId }).lean(), MongoApp.findById(appId).lean() ]); @@ -40,6 +40,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) } const { history } = await getChatItems({ + appId: app._id, chatId, limit: 30, field: `dataId obj value userGoodFeedback userBadFeedback ${ diff --git a/projects/app/src/pages/api/core/chat/updateHistory.ts b/projects/app/src/pages/api/core/chat/updateHistory.ts index 8e983fd7d..634336c9a 100644 --- a/projects/app/src/pages/api/core/chat/updateHistory.ts +++ b/projects/app/src/pages/api/core/chat/updateHistory.ts @@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }); await MongoChat.findOneAndUpdate( - { chatId }, + { appId, chatId }, { ...(customTitle !== undefined && { customTitle }), ...(top !== undefined && { top }) diff --git a/projects/app/src/pages/api/core/dataset/allDataset.ts b/projects/app/src/pages/api/core/dataset/allDataset.ts index e6dc77c32..e1d854018 100644 --- a/projects/app/src/pages/api/core/dataset/allDataset.ts +++ b/projects/app/src/pages/api/core/dataset/allDataset.ts @@ -6,7 +6,7 @@ import { getVectorModel } from '@/service/core/ai/model'; import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; /* get all dataset by teamId or tmbId */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { diff --git a/projects/app/src/pages/api/core/dataset/collection/create/file.ts b/projects/app/src/pages/api/core/dataset/collection/create/file.ts new file mode 100644 index 000000000..e208ead8c --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/create/file.ts @@ -0,0 +1,85 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller'; +import { getUploadModel } from '@fastgpt/service/common/file/multer'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api'; +import { removeFilesByPaths } from '@fastgpt/service/common/file/utils'; +import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; +import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; + +/** + * Creates the multer uploader + */ +const upload = getUploadModel({ + maxSize: 500 * 1024 * 1024 +}); + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + let filePaths: string[] = []; + + const { datasetId } = req.query as { datasetId: string }; + + try { + await connectToDatabase(); + + const { teamId, tmbId } = await authDataset({ + req, + authToken: true, + authApiKey: true, + per: 'w', + datasetId + }); + + const { file, bucketName, data } = await upload.doUpload( + req, + res + ); + filePaths = [file.path]; + + if (!file || !bucketName) { + throw new Error('file is empty'); + } + + const { fileMetadata, collectionMetadata, ...collectionData } = data; + + // upload file and create collection + const fileId = await uploadFile({ + teamId, + tmbId, + bucketName, + path: file.path, + filename: file.originalname, + contentType: file.mimetype, + metadata: fileMetadata + }); + + // create collection + const collectionId = await createOneCollection({ + ...collectionData, + metadata: collectionMetadata, + teamId, + tmbId, + type: DatasetCollectionTypeEnum.file, + fileId + }); + + jsonRes(res, { + data: collectionId + }); + } catch (error) { + jsonRes(res, { + code: 500, + error + }); + } + + removeFilesByPaths(filePaths); +} + +export const config = { + api: { + bodyParser: false + } +}; diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts b/projects/app/src/pages/api/core/dataset/collection/create/link.ts similarity index 96% rename from projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts rename to projects/app/src/pages/api/core/dataset/collection/create/link.ts index 7a4f74807..7ccd0bcc6 100644 --- a/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/link.ts @@ -7,7 +7,10 @@ import { connectToDatabase } from '@/service/mongo'; import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; -import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + TrainingModeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts b/projects/app/src/pages/api/core/dataset/collection/create/text.ts similarity index 87% rename from projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts rename to projects/app/src/pages/api/core/dataset/collection/create/text.ts index 6e131c4dc..59f8267fd 100644 --- a/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/text.ts @@ -7,11 +7,14 @@ import { connectToDatabase } from '@/service/mongo'; import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; -import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { + TrainingModeEnum, + DatasetCollectionTypeEnum +} from '@fastgpt/global/core/dataset/constants'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; -import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; +import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller'; import { hashStr } from '@fastgpt/global/common/string/tools'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -39,8 +42,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< text, chunkLen: chunkSize, overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0, - customReg: chunkSplitter ? [chunkSplitter] : [], - countTokens: false + customReg: chunkSplitter ? [chunkSplitter] : [] }); // 2. check dataset limit @@ -67,7 +69,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // 4. push chunks to training queue - const insertResults = await pushDataToDatasetCollection({ + const insertResults = await pushDataToTrainingQueue({ teamId, tmbId, collectionId, diff --git a/projects/app/src/pages/api/core/dataset/collection/delete.ts b/projects/app/src/pages/api/core/dataset/collection/delete.ts index 9544badec..d4c7835c4 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delete.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils'; -import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller'; +import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< throw new Error('CollectionIdId is required'); } - await authDatasetCollection({ + const { teamId, collection } = await authDatasetCollection({ req, authToken: true, authApiKey: true, @@ -24,13 +24,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // find all delete id - const collections = await findCollectionAndChild(collectionId, '_id fileId'); - const delIdList = collections.map((item) => item._id); + const collections = await findCollectionAndChild({ + teamId, + datasetId: collection.datasetId._id, + collectionId, + fields: '_id teamId fileId metadata' + }); // delete - await delCollectionRelevantData({ - collectionIds: delIdList, - fileIds: collections.map((item) => item?.fileId || '').filter(Boolean) + await delCollectionAndRelatedSources({ + collections }); jsonRes(res); diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 4ffb1e94c..844b5ce02 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -7,7 +7,7 @@ import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type. import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; import { PagingData } from '@/types'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { startQueue } from '@/service/utils/tools'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; @@ -87,7 +87,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< { $match: { $expr: { - $eq: ['$collectionId', '$$id'] + $and: [{ $eq: ['$teamId', match.teamId] }, { $eq: ['$collectionId', '$$id'] }] } } }, @@ -105,7 +105,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< { $match: { $expr: { - $eq: ['$collectionId', '$$id'] + $and: [ + { $eq: ['$teamId', match.teamId] }, + { $eq: ['$datasetId', match.datasetId] }, + { $eq: ['$collectionId', '$$id'] } + ] } } }, diff --git a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts index b7a1b9042..4aba219f6 100644 --- a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts @@ -6,11 +6,11 @@ import { getCollectionAndRawText, reloadCollectionChunks } from '@fastgpt/service/core/dataset/collection/utils'; -import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller'; +import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller'; import { DatasetCollectionSyncResultEnum, DatasetCollectionTypeEnum -} from '@fastgpt/global/core/dataset/constant'; +} from '@fastgpt/global/core/dataset/constants'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants'; @@ -27,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< throw new Error('CollectionIdId is required'); } - const { collection, tmbId } = await authDatasetCollection({ + const { collection, teamId, tmbId } = await authDatasetCollection({ req, authToken: true, collectionId, @@ -87,9 +87,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // delete old collection - await delCollectionRelevantData({ - collectionIds: [collection._id], - fileIds: collection.fileId ? [collection.fileId] : [] + await delCollectionAndRelatedSources({ + collections: [collection] }); jsonRes(res, { diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index d7533443d..e2e200c67 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -5,7 +5,7 @@ import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import type { CreateDatasetParams } from '@/global/core/dataset/api.d'; import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller'; import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { diff --git a/projects/app/src/pages/api/core/dataset/data/delete.ts b/projects/app/src/pages/api/core/dataset/data/delete.ts index 5bf4df034..b945d3e56 100644 --- a/projects/app/src/pages/api/core/dataset/data/delete.ts +++ b/projects/app/src/pages/api/core/dataset/data/delete.ts @@ -3,7 +3,8 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; import { connectToDatabase } from '@/service/mongo'; import { authDatasetData } from '@/service/support/permission/auth/dataset'; -import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/controller'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -17,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } // 凭证校验 - const { datasetData } = await authDatasetData({ + const { teamId, datasetData } = await authDatasetData({ req, authToken: true, authApiKey: true, @@ -25,11 +26,20 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex per: 'w' }); - await delDatasetDataByDataId({ - collectionId: datasetData.collectionId, - mongoDataId: dataId + // update mongo data update time + await MongoDatasetData.findByIdAndUpdate(dataId, { + updateTime: new Date() }); + // delete vector data + await deleteDatasetDataVector({ + teamId, + idList: datasetData.indexes.map((item) => item.dataId) + }); + + // delete mongo data + await MongoDatasetData.findByIdAndDelete(dataId); + jsonRes(res, { data: 'success' }); diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts index 0a794bfe3..7196383e6 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertData.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts @@ -71,12 +71,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // Duplicate data check await hasSameValue({ + teamId, collectionId, q: formatQ, a: formatA }); - const { insertId, tokens } = await insertData2Dataset({ + const { insertId, charsLength } = await insertData2Dataset({ teamId, tmbId, datasetId, @@ -91,7 +92,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model: vectorModelData.model }); diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index e014ec7a4..a22eed4d5 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -20,11 +20,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< pageSize = Math.min(pageSize, 30); // 凭证校验 - await authDatasetCollection({ req, authToken: true, authApiKey: true, collectionId, per: 'r' }); + const { teamId, collection } = await authDatasetCollection({ + req, + authToken: true, + authApiKey: true, + collectionId, + per: 'r' + }); searchText = searchText.replace(/'/g, ''); const match = { + teamId, + datasetId: collection.datasetId._id, collectionId, ...(searchText ? { diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index 5e05a7fde..146a21e52 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -3,13 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; -import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; import type { PushDataResponse } from '@/global/core/api/datasetRes.d'; import type { PushDatasetDataProps } from '@/global/core/dataset/api.d'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; -import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; +import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -41,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex }); jsonRes(res, { - data: await pushDataToDatasetCollection({ + data: await pushDataToTrainingQueue({ ...req.body, teamId, tmbId diff --git a/projects/app/src/pages/api/core/dataset/data/update.ts b/projects/app/src/pages/api/core/dataset/data/update.ts index 41279637a..7f7102d4b 100644 --- a/projects/app/src/pages/api/core/dataset/data/update.ts +++ b/projects/app/src/pages/api/core/dataset/data/update.ts @@ -31,7 +31,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // auth team balance await authTeamBalance(teamId); - const { tokens } = await updateData2Dataset({ + const { charsLength } = await updateData2Dataset({ dataId: id, q, a, @@ -42,7 +42,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model: vectorModel }); diff --git a/projects/app/src/pages/api/core/dataset/delete.ts b/projects/app/src/pages/api/core/dataset/delete.ts index 6b8f796c6..af030a7fb 100644 --- a/projects/app/src/pages/api/core/dataset/delete.ts +++ b/projects/app/src/pages/api/core/dataset/delete.ts @@ -2,32 +2,35 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; -import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/data/controller'; -import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller'; +import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller'; +import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { id } = req.query as { + const { id: datasetId } = req.query as { id: string; }; - if (!id) { + if (!datasetId) { throw new Error('缺少参数'); } // auth owner - await authDataset({ req, authToken: true, datasetId: id, per: 'owner' }); + const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'owner' }); - const deletedIds = await findDatasetIdTreeByTopDatasetId(id); + const datasets = await findDatasetAndAllChildren({ + teamId, + datasetId + }); // delete all dataset.data and pg data - await delDatasetRelevantData({ datasetIds: deletedIds }); + await delDatasetRelevantData({ datasets }); // delete dataset data await MongoDataset.deleteMany({ - _id: { $in: deletedIds } + _id: { $in: datasets.map((d) => d._id) } }); jsonRes(res); diff --git a/projects/app/src/pages/api/core/dataset/exportAll.ts b/projects/app/src/pages/api/core/dataset/exportAll.ts index 44ef68650..b0d9453e0 100644 --- a/projects/app/src/pages/api/core/dataset/exportAll.ts +++ b/projects/app/src/pages/api/core/dataset/exportAll.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { addLog } from '@fastgpt/service/common/system/log'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; -import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller'; +import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; import { checkExportDatasetLimit, @@ -30,7 +30,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes }); - const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId); + const datasets = await findDatasetAndAllChildren({ + teamId, + datasetId, + fields: '_id' + }); res.setHeader('Content-Type', 'text/csv; charset=utf-8;'); res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; '); @@ -42,7 +46,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex a: string; }>( { - datasetId: { $in: exportIds } + teamId, + datasetId: { $in: datasets.map((d) => d._id) } }, 'q a' ) diff --git a/projects/app/src/pages/api/core/dataset/list.ts b/projects/app/src/pages/api/core/dataset/list.ts index 9ebea8e33..07f54546c 100644 --- a/projects/app/src/pages/api/core/dataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/list.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d'; -import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; diff --git a/projects/app/src/pages/api/core/dataset/searchTest.ts b/projects/app/src/pages/api/core/dataset/searchTest.ts index b0fb6246d..b0cbb3a43 100644 --- a/projects/app/src/pages/api/core/dataset/searchTest.ts +++ b/projects/app/src/pages/api/core/dataset/searchTest.ts @@ -47,7 +47,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // model: global.chatModels[0].model // }); - const { searchRes, tokens, ...result } = await searchDatasetData({ + const { searchRes, charsLength, ...result } = await searchDatasetData({ + teamId, rawQuery: text, queries: [text], model: dataset.vectorModel, @@ -62,7 +63,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex const { total } = pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model: dataset.vectorModel, source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt }); diff --git a/projects/app/src/pages/api/plugins/customFeedback/index.ts b/projects/app/src/pages/api/plugins/customFeedback/index.ts index f6e401ca6..cae9ede8a 100644 --- a/projects/app/src/pages/api/plugins/customFeedback/index.ts +++ b/projects/app/src/pages/api/plugins/customFeedback/index.ts @@ -12,6 +12,7 @@ type Props = HttpBodyType<{ export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { const { + appId, chatId, responseChatItemId: chatItemId, data: { defaultFeedback, customFeedback } @@ -30,6 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // wait the chat finish setTimeout(() => { addCustomFeedbacks({ + appId, chatId, chatItemId, feedbacks: [feedback] diff --git a/projects/app/src/pages/api/support/user/account/loginByPassword.ts b/projects/app/src/pages/api/support/user/account/loginByPassword.ts index 197ed8700..44917e3e1 100644 --- a/projects/app/src/pages/api/support/user/account/loginByPassword.ts +++ b/projects/app/src/pages/api/support/user/account/loginByPassword.ts @@ -10,7 +10,7 @@ import { UserStatusEnum } from '@fastgpt/global/support/user/constant'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { username, password, tmbId = '' } = req.body as PostLoginProps; + const { username, password } = req.body as PostLoginProps; if (!username || !password) { throw new Error('缺少参数'); @@ -40,7 +40,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) throw new Error('密码错误'); } - const userDetail = await getUserDetail({ tmbId, userId: user._id }); + const userDetail = await getUserDetail({ + tmbId: user?.lastLoginTmbId, + userId: user._id + }); + + MongoUser.findByIdAndUpdate(user._id, { + lastLoginTmbId: userDetail.team.tmbId + }); const token = createJWT(userDetail); setCookie(res, token); diff --git a/projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts b/projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts new file mode 100644 index 000000000..4ead99cd3 --- /dev/null +++ b/projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts @@ -0,0 +1,37 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + const { size } = req.query as { + size: string; + }; + + // 凭证校验 + const { teamId } = await authCert({ req, authToken: true }); + + if (!size) { + return jsonRes(res); + } + + const numberSize = Number(size); + + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: numberSize + }); + + jsonRes(res); + } catch (err) { + res.status(500); + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts b/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts index b7db92ed4..7a8ba7c13 100644 --- a/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts +++ b/projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts @@ -1,29 +1,32 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants'; import { CreateTrainingBillProps } from '@fastgpt/global/support/wallet/bill/api.d'; import { getQAModel, getVectorModel } from '@/service/core/ai/model'; import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { name, vectorModel, agentModel } = req.body as CreateTrainingBillProps; + const { name, datasetId } = req.body as CreateTrainingBillProps; - const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true }); - - const vectorModelData = getVectorModel(vectorModel); - const agentModelData = getQAModel(agentModel); + const { teamId, tmbId, dataset } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: 'w' + }); const { billId } = await createTrainingBill({ teamId, tmbId, appName: name, billSource: BillSourceEnum.training, - vectorModel: vectorModelData.name, - agentModel: agentModelData.name + vectorModel: getVectorModel(dataset.vectorModel).name, + agentModel: getQAModel(dataset.agentModel).name }); jsonRes(res, { diff --git a/projects/app/src/pages/api/v1/audio/transcriptions.ts b/projects/app/src/pages/api/v1/audio/transcriptions.ts index 22fd91dce..189ab13c4 100644 --- a/projects/app/src/pages/api/v1/audio/transcriptions.ts +++ b/projects/app/src/pages/api/v1/audio/transcriptions.ts @@ -17,11 +17,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex try { const { - files, - metadata: { duration, shareId } + file, + data: { duration } } = await upload.doUpload<{ duration: number; shareId?: string }>(req, res); - filePaths = files.map((file) => file.path); + filePaths = [file.path]; const { teamId, tmbId } = await authCert({ req, authToken: true }); @@ -29,8 +29,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex throw new Error('whisper model not found'); } - const file = files[0]; - if (!file) { throw new Error('file not found'); } diff --git a/projects/app/src/pages/api/v1/chat/completions.ts b/projects/app/src/pages/api/v1/chat/completions.ts index 2ad353878..f703072ab 100644 --- a/projects/app/src/pages/api/v1/chat/completions.ts +++ b/projects/app/src/pages/api/v1/chat/completions.ts @@ -195,7 +195,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex }); // get and concat history - const { history } = await getChatItems({ chatId, limit: 30, field: `dataId obj value` }); + const { history } = await getChatItems({ + appId: app._id, + chatId, + limit: 30, + field: `dataId obj value` + }); const concatHistories = history.concat(chatMessages); const responseChatItemId: string | undefined = messages[messages.length - 1].dataId; diff --git a/projects/app/src/pages/api/v1/embeddings.ts b/projects/app/src/pages/api/v1/embeddings.ts index 4371c2609..e1e1b283d 100644 --- a/projects/app/src/pages/api/v1/embeddings.ts +++ b/projects/app/src/pages/api/v1/embeddings.ts @@ -33,7 +33,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex await authTeamBalance(teamId); - const { tokens, vectors } = await getVectorsByText({ input: query, model }); + const { charsLength, vectors } = await getVectorsByText({ input: query, model }); res.json({ object: 'list', @@ -44,15 +44,15 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex })), model, usage: { - prompt_tokens: tokens, - total_tokens: tokens + prompt_tokens: charsLength, + total_tokens: charsLength } }); const { total } = pushGenerateVectorBill({ teamId, tmbId, - tokens, + charsLength, model, billId, source: getBillSourceByAuthType({ authType }) diff --git a/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx b/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx index 3b797d97a..f1932ec3f 100644 --- a/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx +++ b/projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx @@ -62,7 +62,7 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ }); if (unconnected) { - const msg = `【${t(item.name)}】存在未填或未连接参数`; + const msg = t('core.module.Unlink tip', { name: t(item.name) }); toast({ status: 'warning', @@ -82,8 +82,8 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ permission: undefined }); }, - successToast: '保存配置成功', - errorToast: '保存配置异常', + successToast: t('common.Save Success'), + errorToast: t('common.Save Failed'), onSuccess() { ChatTestRef.current?.resetChatTest(); } @@ -98,22 +98,23 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ alignItems={'center'} userSelect={'none'} > - - } - borderColor={'myGray.300'} - variant={'whiteBase'} - aria-label={''} - onClick={openConfirmOut(async () => { - const modules = await flow2ModulesAndCheck(); - if (modules) { - await onclickSave(modules); - } - onClose(); - }, onClose)} - /> - + } + borderRadius={'50%'} + w={'26px'} + h={'26px'} + borderColor={'myGray.300'} + variant={'whiteBase'} + aria-label={''} + onClick={openConfirmOut(async () => { + const modules = await flow2ModulesAndCheck(); + if (modules) { + await onclickSave(modules); + } + onClose(); + }, onClose)} + /> {app.name} @@ -154,7 +155,7 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ onClick={() => setTestModules(undefined)} /> ) : ( - + } @@ -171,9 +172,9 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({ )} - + } + icon={} size={'smSquare'} isLoading={isLoading} aria-label={'save'} diff --git a/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx b/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx index 11dbd0761..b1298c9cd 100644 --- a/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx +++ b/projects/app/src/pages/app/detail/components/FlowEdit/index.tsx @@ -44,12 +44,18 @@ const Render = ({ app, onClose }: Props) => { initData(JSON.parse(JSON.stringify(app.modules))); }, [app.modules]); - return } />; + const memoRender = useMemo(() => { + return } />; + }, [app, moduleTemplates.length, onClose]); + + return memoRender; }; export default React.memo(function FlowEdit(props: Props) { + const filterAppIds = useMemo(() => [props.app._id], [props.app._id]); + return ( - + ); diff --git a/projects/app/src/pages/app/detail/components/InfoModal.tsx b/projects/app/src/pages/app/detail/components/InfoModal.tsx index 6786ea4b8..3185c741b 100644 --- a/projects/app/src/pages/app/detail/components/InfoModal.tsx +++ b/projects/app/src/pages/app/detail/components/InfoModal.tsx @@ -52,7 +52,7 @@ const InfoModal = ({ }); const [refresh, setRefresh] = useState(false); - // 提交保存模型修改 + // submit config const { mutate: saveSubmitSuccess, isLoading: btnLoading } = useRequest({ mutationFn: async (data: AppSchema) => { await updateAppDetail(data._id, { @@ -66,18 +66,17 @@ const InfoModal = ({ onSuccess && onSuccess(); onClose(); toast({ - title: '更新成功', + title: t('common.Update Success'), status: 'success' }); }, - errorToast: '更新失败' + errorToast: t('common.Update Failed') }); - // 提交保存表单失败 const saveSubmitError = useCallback(() => { // deep search message const deepSearch = (obj: any): string => { - if (!obj) return '提交表单错误'; + if (!obj) return t('common.Submit failed'); if (!!obj.message) { return obj.message; } @@ -89,7 +88,7 @@ const InfoModal = ({ duration: 4000, isClosable: true }); - }, [errors, toast]); + }, [errors, t, toast]); const saveUpdateModel = useCallback( () => handleSubmit((data) => saveSubmitSuccess(data), saveSubmitError)(), @@ -111,12 +110,12 @@ const InfoModal = ({ setRefresh((state) => !state); } catch (err: any) { toast({ - title: getErrText(err, '头像选择异常'), + title: getErrText(err, t('common.error.Select avatar failed')), status: 'warning' }); } }, - [setValue, toast] + [setValue, t, toast] ); return ( @@ -127,7 +126,7 @@ const InfoModal = ({ title={t('core.app.setting')} > - 头像 & 名称 + {t('core.app.Name and avatar')} onOpenSelectFile()} /> - 应用介绍 + {t('core.app.App intro')} {/* 该介绍主要用于记忆和在应用市场展示 @@ -158,7 +157,7 @@ const InfoModal = ({