From bfd8be5df0f7de87595ca976c9d8d59f62f2721a Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Wed, 15 Nov 2023 11:36:25 +0800 Subject: [PATCH] v4.6-3 (#471) --- .github/workflows/fastgpt-image-personal.yml | 52 ++ .github/workflows/fastgpt-image.yml | 5 +- .github/workflows/preview-image.yml | 3 +- .../content/docs/installation/upgrading/46.md | 33 +- packages/global/common/string/textSplitter.ts | 131 +++++ .../common/string}/tiktoken/cl100k_base.json | 0 .../global/common/string}/tiktoken/index.ts | 17 +- .../global/common/string/tiktoken/type.d.ts | 5 + packages/global/common/string/tools.ts | 19 +- packages/global/core/ai/api.d.ts | 5 + .../global/core/chat/adapt.ts | 8 +- packages/global/core/dataset/api.d.ts | 20 + packages/global/core/dataset/constant.ts | 65 ++- packages/global/core/dataset/controller.d.ts | 27 + packages/global/core/dataset/type.d.ts | 78 +-- packages/global/core/dataset/utils.ts | 13 +- packages/global/core/module/api.d.ts | 3 + packages/global/package.json | 3 +- packages/global/support/user/team/constant.ts | 8 +- .../global/support/user/team/controller.d.ts | 5 +- packages/global/support/user/team/type.d.ts | 4 +- packages/global/support/wallet/bill/type.d.ts | 2 +- packages/service/common/api/plusRequest.ts | 6 +- packages/service/common/pg/index.ts | 3 +- .../service/core/chat/utils.ts | 19 +- .../service/core/dataset/collection/schema.ts | 3 +- packages/service/core/dataset/controller.ts | 20 + packages/service/core/dataset/data/schema.ts | 78 +++ .../service/core/dataset/training/schema.ts | 25 +- .../support/activity/promotion/schema.ts | 1 + packages/service/support/outLink/tools.ts | 2 +- .../service/support/permission/auth/common.ts | 21 + .../support/permission/auth/dataset.ts | 4 +- .../support/permission/auth/openapi.ts | 1 - .../service/support/permission/auth/user.ts | 8 +- .../service/support/user/team/controller.ts | 62 +-- .../service/support/wallet/bill/schema.ts | 1 + pnpm-lock.yaml | 42 +- projects/app/next-i18next.config.js | 5 +- projects/app/package.json | 3 - projects/app/public/docs/versionIntro.md | 17 +- projects/app/public/locales/en/common.json | 26 +- projects/app/public/locales/zh/common.json | 26 +- .../app/src/components/ChatBox/QuoteModal.tsx | 15 +- .../src/components/ChatBox/ResponseTags.tsx | 2 +- .../ChatBox/SelectMarkCollection.tsx | 13 +- .../components/ChatBox/WholeResponseModal.tsx | 2 +- projects/app/src/components/ChatBox/index.tsx | 4 +- .../src/components/CommunityModal/index.tsx | 2 +- .../app/src/components/Language/index.tsx | 48 -- .../app/src/components/Layout/navbarPhone.tsx | 2 +- projects/app/src/components/Tabs/index.tsx | 2 +- .../components/common/ParentPaths/index.tsx | 2 +- .../common/Textarea/TagTextarea.tsx | 2 +- .../components/core/dataset/SelectModal.tsx | 2 +- .../core/module/AIChatSettingsModal.tsx | 2 +- .../core/module/DatasetSelectModal.tsx | 4 +- .../core/module/Flow/ImportSettings.tsx | 2 +- .../core/module/Flow/SelectAppModal.tsx | 2 +- .../core/module/Flow/TemplateList.tsx | 2 +- .../components/modules/FieldEditModal.tsx | 2 +- .../Flow/components/modules/NodeCard.tsx | 2 +- .../Flow/components/nodes/NodeCQNode.tsx | 2 +- .../Flow/components/render/RenderInput.tsx | 4 +- .../src/components/support/apikey/Table.tsx | 2 +- .../support/permission/IconText/index.tsx | 2 +- .../support/permission/Radio/index.tsx | 2 +- .../user/team/TeamManageModal/EditModal.tsx | 2 +- .../user/team/TeamManageModal/InviteModal.tsx | 6 +- .../user/team/TeamManageModal/index.tsx | 10 +- .../support/user/team/TeamMenu/index.tsx | 2 +- .../user/team/UpdateInviteModal/index.tsx | 2 +- projects/app/src/constants/dataset.ts | 36 +- .../app/src/global/common/string/tools.ts | 108 ---- .../app/src/global/core/api/datasetReq.d.ts | 26 +- projects/app/src/global/core/dataset/api.d.ts | 35 ++ .../core/dataset/{response.d.ts => type.d.ts} | 12 +- projects/app/src/pages/_app.tsx | 24 +- projects/app/src/pages/_document.tsx | 3 +- projects/app/src/pages/_error.tsx | 5 +- .../pages/account/components/ApiKeyTable.tsx | 2 +- .../pages/account/components/BillDetail.tsx | 4 +- .../pages/account/components/BillTable.tsx | 4 +- .../app/src/pages/account/components/Info.tsx | 47 +- .../account/components/OpenAIAccountModal.tsx | 2 +- .../src/pages/account/components/PayModal.tsx | 2 +- .../account/components/UpdatePswModal.tsx | 2 +- projects/app/src/pages/account/index.tsx | 119 +++-- projects/app/src/pages/api/admin/initv46-2.ts | 169 +++++++ projects/app/src/pages/api/admin/initv46.ts | 22 +- .../app/src/pages/api/common/file/upload.ts | 2 - .../api/core/ai/agent/createQuestionGuide.ts | 9 +- .../src/pages/api/core/dataset/allDataset.ts | 4 +- .../api/core/dataset/collection/delById.ts | 8 +- .../api/core/dataset/collection/detail.ts | 5 +- .../pages/api/core/dataset/collection/list.ts | 114 +++-- .../api/core/dataset/collection/paths.ts | 4 +- .../app/src/pages/api/core/dataset/create.ts | 2 +- .../data/{delDataById.ts => delete.ts} | 11 +- .../data/{getDataById.ts => detail.ts} | 5 +- .../pages/api/core/dataset/data/exportAll.ts | 132 ----- .../pages/api/core/dataset/data/insertData.ts | 15 +- .../dataset/data/{getDataList.ts => list.ts} | 41 +- .../pages/api/core/dataset/data/pushData.ts | 33 +- .../dataset/data/{updateData.ts => update.ts} | 42 +- .../app/src/pages/api/core/dataset/delete.ts | 30 +- .../app/src/pages/api/core/dataset/detail.ts | 3 +- .../src/pages/api/core/dataset/exportAll.ts | 117 +++++ .../app/src/pages/api/core/dataset/list.ts | 3 +- .../app/src/pages/api/core/dataset/paths.ts | 6 +- .../src/pages/api/core/dataset/searchTest.ts | 2 +- .../dataset/{data => training}/getQueueLen.ts | 0 .../app/src/pages/api/plusApi/[...path].ts | 7 +- .../app/src/pages/api/system/getInitData.ts | 6 +- .../src/pages/api/user/account/tokenLogin.ts | 4 +- .../app/src/pages/api/v1/chat/completions.ts | 4 +- .../app/detail/components/BasicEdit/index.tsx | 5 +- .../pages/app/detail/components/InfoModal.tsx | 2 +- .../app/detail/components/OutLink/API.tsx | 2 +- .../app/detail/components/OutLink/Share.tsx | 2 +- .../pages/app/detail/components/QGSwitch.tsx | 2 +- .../pages/app/detail/components/TTSSelect.tsx | 2 +- projects/app/src/pages/app/list/index.tsx | 1 + .../chat/components/ChatHistorySlider.tsx | 2 +- .../src/pages/chat/components/SliderApps.tsx | 2 +- projects/app/src/pages/chat/index.tsx | 2 +- projects/app/src/pages/components/Footer.tsx | 2 +- projects/app/src/pages/components/Hero.tsx | 2 +- .../dataset/component/EditFolderModal.tsx | 2 +- .../detail/components/CollectionCard.tsx | 14 +- .../dataset/detail/components/DataCard.tsx | 28 +- .../detail/components/Import/FileSelect.tsx | 13 +- .../detail/components/Import/ImportModal.tsx | 16 +- .../detail/components/Import/Provider.tsx | 14 +- .../dataset/detail/components/Import/QA.tsx | 7 +- .../pages/dataset/detail/components/Info.tsx | 10 +- .../detail/components/InputDataModal.tsx | 465 +++++++++++------- .../pages/dataset/detail/components/Test.tsx | 21 +- .../app/src/pages/dataset/detail/index.tsx | 8 +- .../dataset/list/component/CreateModal.tsx | 2 +- .../dataset/list/component/MoveModal.tsx | 2 +- projects/app/src/pages/dataset/list/index.tsx | 33 +- .../app/src/pages/plugin/edit/Preview.tsx | 2 +- projects/app/src/pages/plugin/edit/index.tsx | 2 +- .../pages/plugin/list/component/EditModal.tsx | 2 +- projects/app/src/pages/tools/index.tsx | 2 +- .../service/core/dataset/data/controller.ts | 351 +++++++------ .../app/src/service/core/dataset/data/pg.ts | 281 +++++++++++ .../src/service/core/dataset/data/utils.ts | 78 --- projects/app/src/service/events/generateQA.ts | 73 ++- .../app/src/service/events/generateVector.ts | 30 +- .../moduleDispatch/agent/classifyQuestion.ts | 6 +- .../service/moduleDispatch/agent/extract.ts | 6 +- .../src/service/moduleDispatch/chat/oneapi.ts | 9 +- .../service/moduleDispatch/dataset/search.ts | 6 +- .../app/src/service/moduleDispatch/index.ts | 2 +- .../service/moduleDispatch/tools/runApp.ts | 2 +- projects/app/src/service/mongo.ts | 8 +- .../support/permission/auth/dataset.ts | 41 +- .../src/service/support/user/controller.ts | 15 +- .../src/service/support/wallet/bill/push.ts | 2 +- .../app/src/service/utils/chat/saveChat.ts | 6 +- .../app/src/types/core/dataset/index.d.ts | 16 - projects/app/src/types/index.d.ts | 2 - projects/app/src/utils/adapt.ts | 2 +- projects/app/src/web/common/api/request.ts | 47 +- .../web/common/file/hooks/useSelectFile.tsx | 2 +- .../app/src/web/common/hooks/useCopyData.tsx | 2 +- .../app/src/web/common/hooks/useRequest.tsx | 2 +- .../app/src/web/common/hooks/useSpeech.ts | 2 +- projects/app/src/web/common/utils/i18n.ts | 39 +- .../app/src/web/core/app/basicSettings.ts | 2 +- projects/app/src/web/core/dataset/api.ts | 62 +-- .../dataset/components/SelectCollections.tsx | 2 +- .../app/src/web/core/dataset/store/dataset.ts | 4 +- .../src/web/core/dataset/store/markdata.ts | 1 - projects/app/src/web/core/dataset/utils.ts | 10 +- projects/app/src/web/styles/default.scss | 4 + projects/app/src/web/styles/theme.ts | 5 +- projects/app/src/web/support/user/api.ts | 3 +- projects/app/src/web/support/user/team/api.ts | 3 +- 181 files changed, 2499 insertions(+), 1552 deletions(-) create mode 100644 .github/workflows/fastgpt-image-personal.yml create mode 100644 packages/global/common/string/textSplitter.ts rename {projects/app/src/global/common => packages/global/common/string}/tiktoken/cl100k_base.json (100%) rename {projects/app/src/global/common => packages/global/common/string}/tiktoken/index.ts (79%) create mode 100644 packages/global/common/string/tiktoken/type.d.ts create mode 100644 packages/global/core/ai/api.d.ts rename projects/app/src/utils/common/adapt/message.ts => packages/global/core/chat/adapt.ts (80%) create mode 100644 packages/global/core/dataset/api.d.ts create mode 100644 packages/global/core/dataset/controller.d.ts rename projects/app/src/service/common/tiktoken.ts => packages/service/core/chat/utils.ts (74%) create mode 100644 packages/service/core/dataset/data/schema.ts delete mode 100644 projects/app/src/components/Language/index.tsx delete mode 100644 projects/app/src/global/common/string/tools.ts create mode 100644 projects/app/src/global/core/dataset/api.d.ts rename projects/app/src/global/core/dataset/{response.d.ts => type.d.ts} (72%) create mode 100644 projects/app/src/pages/api/admin/initv46-2.ts rename projects/app/src/pages/api/core/dataset/data/{delDataById.ts => delete.ts} (78%) rename projects/app/src/pages/api/core/dataset/data/{getDataById.ts => detail.ts} (88%) delete mode 100644 projects/app/src/pages/api/core/dataset/data/exportAll.ts rename projects/app/src/pages/api/core/dataset/data/{getDataList.ts => list.ts} (57%) rename projects/app/src/pages/api/core/dataset/data/{updateData.ts => update.ts} (60%) create mode 100644 projects/app/src/pages/api/core/dataset/exportAll.ts rename projects/app/src/pages/api/core/dataset/{data => training}/getQueueLen.ts (100%) create mode 100644 projects/app/src/service/core/dataset/data/pg.ts delete mode 100644 projects/app/src/types/core/dataset/index.d.ts diff --git a/.github/workflows/fastgpt-image-personal.yml b/.github/workflows/fastgpt-image-personal.yml new file mode 100644 index 000000000..f0edad535 --- /dev/null +++ b/.github/workflows/fastgpt-image-personal.yml @@ -0,0 +1,52 @@ +name: Build FastGPT images in Personal warehouse +on: + workflow_dispatch: + push: + paths: + - 'projects/app/**' + - 'packages/**' + branches: + - 'main' +jobs: + build-fastgpt-images: + runs-on: ubuntu-20.04 + if: github.repository != 'labring/FastGPT' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GH_PAT }} + - name: Set DOCKER_REPO_TAGGED based on branch or tag + run: | + echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt:latest" >> $GITHUB_ENV + - name: Build and publish image for main branch or tag push event + env: + DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }} + run: | + docker buildx build \ + --build-arg name=app \ + --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \ + --label "org.opencontainers.image.description=fastgpt image" \ + --push \ + --cache-from=type=local,src=/tmp/.buildx-cache \ + --cache-to=type=local,dest=/tmp/.buildx-cache \ + -t ${DOCKER_REPO_TAGGED} \ + -f Dockerfile \ + . diff --git a/.github/workflows/fastgpt-image.yml b/.github/workflows/fastgpt-image.yml index 09a4c2a5a..baf1d16c6 100644 --- a/.github/workflows/fastgpt-image.yml +++ b/.github/workflows/fastgpt-image.yml @@ -5,8 +5,6 @@ on: paths: - 'projects/app/**' - 'packages/**' - branches: - - 'main' tags: - 'v*.*.*' jobs: @@ -53,9 +51,8 @@ jobs: docker buildx build \ --build-arg name=app \ --platform linux/amd64,linux/arm64 \ - --label "org.opencontainers.image.source= https://github.com/ ${{ github.repository_owner }}/FastGPT" \ + --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \ --label "org.opencontainers.image.description=fastgpt image" \ - --label "org.opencontainers.image.licenses=Apache" \ --push \ --cache-from=type=local,src=/tmp/.buildx-cache \ --cache-to=type=local,dest=/tmp/.buildx-cache \ diff --git a/.github/workflows/preview-image.yml b/.github/workflows/preview-image.yml index cbcc80b88..e2195fa15 100644 --- a/.github/workflows/preview-image.yml +++ b/.github/workflows/preview-image.yml @@ -24,7 +24,7 @@ jobs: with: driver-opts: network=host - name: Cache Docker layers - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -48,6 +48,7 @@ jobs: --label "org.opencontainers.image.source= https://github.com/ ${{ github.repository_owner }}/FastGPT" \ --label "org.opencontainers.image.description=fastgpt-pr image" \ --label "org.opencontainers.image.licenses=Apache" \ + --push \ --cache-from=type=local,src=/tmp/.buildx-cache \ --cache-to=type=local,dest=/tmp/.buildx-cache \ -t ${DOCKER_REPO_TAGGED} \ diff --git a/docSite/content/docs/installation/upgrading/46.md b/docSite/content/docs/installation/upgrading/46.md index 36f7dd9b7..4ebc6fcee 100644 --- a/docSite/content/docs/installation/upgrading/46.md +++ b/docSite/content/docs/installation/upgrading/46.md @@ -7,20 +7,20 @@ toc: true weight: 836 --- -未正式发布。 +# V4.6 版本加入了简单的团队功能，可以邀请其他用户进来管理资源。该版本升级后无法执行旧的升级脚本，且无法回退。 -V4.6 版本加入了简单的团队功能，可以邀请其他用户进来管理资源。该版本升级后无法执行旧的升级脚本，且无法回退。 +# 1. 更新镜像并变更配置文件 -## 1. 更新镜像并变更配置文件 - -更新镜像至 latest 或者 v4.6 版本。商业版镜像更新至 V0.2. +更新镜像至 latest 或者 v4.6 版本。商业版镜像更新至 V0.2.1 最新配置可参考: [V46版本最新 config.json](/docs/development/configuration)，商业镜像配置文件也更新，参考最新的飞书文档。 -## 2. 执行初始化 API +# 2. 执行初始化 API -发起 1 个 HTTP 请求（{{rootkey}} 替换成环境变量里的`rootkey`，{{host}}替换成自己域名） +发起 2 个 HTTP 请求（{{rootkey}} 替换成环境变量里的`rootkey`，{{host}}替换成自己域名） + +**该初始化接口可能速度很慢，返回超时不用管，注意看日志即可，需要注意的是，需确保initv46成功后，在执行initv46-2** 1. https://xxxxx/api/admin/initv46 @@ -30,16 +30,25 @@ curl --location --request POST 'https://{{host}}/api/admin/initv46' \ --header 'Content-Type: application/json' ``` +2. https://xxxxx/api/admin/initv46-2 + +```bash +curl --location --request POST 'https://{{host}}/api/admin/initv46-2' \ +--header 'rootkey: {{rootkey}}' \ +--header 'Content-Type: application/json' +``` + 初始化内容： 1. 创建默认团队 2. 初始化 Mongo 所有资源的团队字段 3. 初始化 Pg 的字段 - -**该初始化接口可能速度很慢，返回超时不用管，注意看日志即可** +4. 初始化 Mongo Data -## 功能介绍 - -### Fast GPT V4.6 +# V4.6功能介绍 1. 新增 - 团队空间 +2. 新增 - 多路向量（多个向量映射一组数据） +3. 新增 - tts语音 +4. 线上环境新增 - ReRank向量召回，提高召回精度 +5. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈 diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts new file mode 100644 index 000000000..5737347f2 --- /dev/null +++ b/packages/global/common/string/textSplitter.ts @@ -0,0 +1,131 @@ +import { getErrText } from '../error/utils'; +import { countPromptTokens } from './tiktoken'; + +/** + * text split into chunks + * maxLen - one chunk len. max: 3500 + * overlapLen - The size of the before and after Text + * maxLen > overlapLen + * markdown + */ +export const splitText2Chunks = (props: { text: string; maxLen: number; overlapLen?: number }) => { + const { text = '', maxLen, overlapLen = Math.floor(maxLen * 0.2) } = props; + const tempMarker = 'SPLIT_HERE_SPLIT_HERE'; + + const stepReg: Record = { + 0: /^(#\s[^\n]+)\n/gm, + 1: /^(##\s[^\n]+)\n/gm, + 2: /^(###\s[^\n]+)\n/gm, + 3: /^(####\s[^\n]+)\n/gm, + + 4: /(\n\n)/g, + 5: /([\n])/g, + 6: /[。]|(?!<[^a-zA-Z])\.\s/g, + 7: /([！？]|!\s|\?\s)/g, + 8: /([；]|;\s)/g, + 9: /([，]|,\s)/g + }; + + const splitTextRecursively = ({ + text = '', + step, + lastChunk, + overlayChunk + }: { + text: string; + step: number; + lastChunk: string; + overlayChunk: string; + }) => { + if (text.length <= maxLen) { + return [text]; + } + const reg = stepReg[step]; + const isMarkdownSplit = step < 4; + + if (!reg) { + // use slice-maxLen to split text + const chunks: string[] = []; + let chunk = ''; + for (let i = 0; i < text.length; i += maxLen - overlapLen) { + chunk = text.slice(i, i + maxLen); + chunks.push(chunk); + } + return chunks; + } + + // split text by special char + const splitTexts = text + .replace(reg, isMarkdownSplit ? `${tempMarker}$1` : `$1${tempMarker}`) + .split(`${tempMarker}`) + .filter((part) => part); + + let chunks: string[] = []; + for (let i = 0; i < splitTexts.length; i++) { + let text = splitTexts[i]; + let chunkToken = countPromptTokens(lastChunk, ''); + const textToken = countPromptTokens(text, ''); + + // next chunk is too large / new chunk is too large(The current chunk must be smaller than maxLen) + if (textToken >= maxLen || chunkToken + textToken > maxLen * 1.4) { + // last chunk is too large, push it to chunks, not add to next chunk + if (chunkToken > maxLen * 0.7) { + chunks.push(lastChunk); + lastChunk = ''; + overlayChunk = ''; + } + // chunk is small, insert to next chunks + const innerChunks = splitTextRecursively({ + text, + step: step + 1, + lastChunk, + overlayChunk + }); + if (innerChunks.length === 0) continue; + chunks = chunks.concat(innerChunks); + lastChunk = ''; + overlayChunk = ''; + continue; + } + + // size less than maxLen, push text to last chunk + lastChunk += text; + chunkToken += textToken; // Definitely less than 1.4 * maxLen + + // size over lapLen, push it to next chunk + if ( + overlapLen !== 0 && + !isMarkdownSplit && + chunkToken >= maxLen - overlapLen && + textToken < overlapLen + ) { + overlayChunk += text; + } + if (chunkToken >= maxLen) { + chunks.push(lastChunk); + lastChunk = overlayChunk; + overlayChunk = ''; + } + } + + /* If the last chunk is independent, it needs to be push chunks. */ + if (lastChunk && chunks[chunks.length - 1] && !chunks[chunks.length - 1].endsWith(lastChunk)) { + chunks.push(lastChunk); + } + + return chunks; + }; + + try { + const chunks = splitTextRecursively({ text, step: 0, lastChunk: '', overlayChunk: '' }); + + const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0); + + return { + chunks, + tokens + }; + } catch (err) { + throw new Error(getErrText(err)); + } +}; diff --git a/projects/app/src/global/common/tiktoken/cl100k_base.json b/packages/global/common/string/tiktoken/cl100k_base.json similarity index 100% rename from projects/app/src/global/common/tiktoken/cl100k_base.json rename to packages/global/common/string/tiktoken/cl100k_base.json diff --git a/projects/app/src/global/common/tiktoken/index.ts b/packages/global/common/string/tiktoken/index.ts similarity index 79% rename from projects/app/src/global/common/tiktoken/index.ts rename to packages/global/common/string/tiktoken/index.ts index 61e521cbb..c9586c22e 100644 --- a/projects/app/src/global/common/tiktoken/index.ts +++ b/packages/global/common/string/tiktoken/index.ts @@ -1,8 +1,8 @@ /* Only the token of gpt-3.5-turbo is used */ -import type { ChatItemType } from '@fastgpt/global/core/chat/type'; +import type { ChatItemType } from '../../../core/chat/type'; import { Tiktoken } from 'js-tiktoken/lite'; -import { adaptChat2GptMessages } from '@/utils/common/adapt/message'; -import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constant'; +import { adaptChat2GptMessages } from '../../../core/chat/adapt'; +import { ChatCompletionRequestMessageRoleEnum } from '../../../core/ai/constant'; import encodingJson from './cl100k_base.json'; /* init tikToken obj */ @@ -55,17 +55,6 @@ export function countMessagesTokens({ messages }: { messages: ChatItemType[] }) return totalTokens; } -export function sliceTextByTokens({ text, length }: { text: string; length: number }) { - const enc = getTikTokenEnc(); - - try { - const encodeText = enc.encode(text); - return enc.decode(encodeText.slice(0, length)); - } catch (error) { - return text.slice(0, length); - } -} - /* slice messages from top to bottom by maxTokens */ export function sliceMessagesTB({ messages, diff --git a/packages/global/common/string/tiktoken/type.d.ts b/packages/global/common/string/tiktoken/type.d.ts new file mode 100644 index 000000000..0531c0735 --- /dev/null +++ b/packages/global/common/string/tiktoken/type.d.ts @@ -0,0 +1,5 @@ +import type { Tiktoken } from 'js-tiktoken'; + +declare global { + var TikToken: Tiktoken; +} diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts index 0621b4f80..61cdf44eb 100644 --- a/packages/global/common/string/tools.ts +++ b/packages/global/common/string/tools.ts @@ -1,13 +1,15 @@ import crypto from 'crypto'; +/* check string is a web link */ export function strIsLink(str?: string) { if (!str) return false; if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true; return false; } -export const hashStr = (psw: string) => { - return crypto.createHash('sha256').update(psw).digest('hex'); +/* hash string */ +export const hashStr = (str: string) => { + return crypto.createHash('sha256').update(str).digest('hex'); }; /* simple text, remove chinese space and extra \n */ @@ -20,3 +22,16 @@ export const simpleText = (text: string) => { return text; }; + +/* + replace {{variable}} to value +*/ +export function replaceVariable(text: string, obj: Record) { + for (const key in obj) { + const val = obj[key]; + if (!['string', 'number'].includes(typeof val)) continue; + + text = text.replace(new RegExp(`{{(${key})}}`, 'g'), String(val)); + } + return text || ''; +} diff --git a/packages/global/core/ai/api.d.ts b/packages/global/core/ai/api.d.ts new file mode 100644 index 000000000..33eb945fe --- /dev/null +++ b/packages/global/core/ai/api.d.ts @@ -0,0 +1,5 @@ +export type PostReRankProps = { + query: string; + inputs: { id: string; text: string }[]; +}; +export type PostReRankResponse = { id: string; score: number }[]; diff --git a/projects/app/src/utils/common/adapt/message.ts b/packages/global/core/chat/adapt.ts similarity index 80% rename from projects/app/src/utils/common/adapt/message.ts rename to packages/global/core/chat/adapt.ts index 88524851e..c63417bcd 100644 --- a/projects/app/src/utils/common/adapt/message.ts +++ b/packages/global/core/chat/adapt.ts @@ -1,7 +1,7 @@ -import type { ChatItemType } from '@fastgpt/global/core/chat/type.d'; -import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; -import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constant'; -import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d'; +import type { ChatItemType } from '../../core/chat/type.d'; +import { ChatRoleEnum } from '../../core/chat/constants'; +import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constant'; +import type { ChatMessageItemType } from '../../core/ai/type.d'; const chat2Message = { [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant, diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts new file mode 100644 index 000000000..4d5049389 --- /dev/null +++ b/packages/global/core/dataset/api.d.ts @@ -0,0 +1,20 @@ +import { DatasetDataIndexItemType } from './type'; + +/* ================= dataset ===================== */ + +/* ================= collection ===================== */ + +/* ================= data ===================== */ +export type PgSearchRawType = { + id: string; + team_id: string; + tmb_id: string; + collection_id: string; + data_id: string; + score: number; +}; +export type PushDatasetDataChunkProps = { + q: string; // embedding content + a?: string; // bonus content + indexes?: Omit[]; +}; diff --git a/packages/global/core/dataset/constant.ts b/packages/global/core/dataset/constant.ts index 822ffc54d..9de474444 100644 --- a/packages/global/core/dataset/constant.ts +++ b/packages/global/core/dataset/constant.ts @@ -36,29 +36,54 @@ export const DatasetCollectionTypeMap = { } }; -export enum TrainingModeEnum { - 'qa' = 'qa', - 'index' = 'index' +export enum DatasetDataIndexTypeEnum { + chunk = 'chunk', + qa = 'qa', + summary = 'summary', + hypothetical = 'hypothetical', + custom = 'custom' } -export const TrainingTypeMap = { - [TrainingModeEnum.qa]: 'qa', - [TrainingModeEnum.index]: 'index' -}; - -export enum DatasetSpecialIdEnum { - manual = 'manual', - mark = 'mark' -} -export const datasetSpecialIdMap = { - [DatasetSpecialIdEnum.manual]: { - name: 'kb.Manual Data', - sourceName: 'kb.Manual Input' +export const DatasetDataIndexTypeMap = { + [DatasetDataIndexTypeEnum.chunk]: { + name: 'dataset.data.indexes.chunk' }, - [DatasetSpecialIdEnum.mark]: { - name: 'kb.Mark Data', - sourceName: 'kb.Manual Mark' + [DatasetDataIndexTypeEnum.summary]: { + name: 'dataset.data.indexes.summary' + }, + [DatasetDataIndexTypeEnum.hypothetical]: { + name: 'dataset.data.indexes.hypothetical' + }, + [DatasetDataIndexTypeEnum.qa]: { + name: 'dataset.data.indexes.qa' + }, + [DatasetDataIndexTypeEnum.custom]: { + name: 'dataset.data.indexes.custom' } }; -export const datasetSpecialIds: string[] = [DatasetSpecialIdEnum.manual, DatasetSpecialIdEnum.mark]; + +export enum TrainingModeEnum { + 'chunk' = 'chunk', + 'qa' = 'qa' + // 'hypothetical' = 'hypothetical', + // 'summary' = 'summary', + // 'multipleIndex' = 'multipleIndex' +} +export const TrainingTypeMap = { + [TrainingModeEnum.chunk]: { + name: 'chunk' + }, + [TrainingModeEnum.qa]: { + name: 'qa' + } + // [TrainingModeEnum.hypothetical]: { + // name: 'hypothetical' + // }, + // [TrainingModeEnum.summary]: { + // name: 'summary' + // }, + // [TrainingModeEnum.multipleIndex]: { + // name: 'multipleIndex' + // } +}; export const FolderAvatarSrc = '/imgs/files/folder.svg'; diff --git a/packages/global/core/dataset/controller.d.ts b/packages/global/core/dataset/controller.d.ts new file mode 100644 index 000000000..a83607de9 --- /dev/null +++ b/packages/global/core/dataset/controller.d.ts @@ -0,0 +1,27 @@ +import type { DatasetDataIndexItemType, DatasetDataSchemaType } from './type'; + +export type CreateDatasetDataProps = { + teamId: string; + tmbId: string; + datasetId: string; + collectionId: string; + q: string; + a?: string; + indexes?: Omit[]; +}; + +export type UpdateDatasetDataProps = { + dataId: string; + q?: string; + a?: string; + indexes?: (Omit & { + dataId?: string; // pg data id + })[]; +}; + +export type PatchIndexesProps = { + type: 'create' | 'update' | 'delete'; + index: Omit & { + dataId?: string; + }; +}; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 3f4173d69..a1a7700da 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -1,6 +1,14 @@ +import type { VectorModelItemType } from '../../core/ai/model.d'; import { PermissionTypeEnum } from '../../support/permission/constant'; -import { DatasetCollectionTypeEnum, DatasetTypeEnum, TrainingModeEnum } from './constant'; +import { PushDatasetDataChunkProps } from './api'; +import { + DatasetCollectionTypeEnum, + DatasetDataIndexTypeEnum, + DatasetTypeEnum, + TrainingModeEnum +} from './constant'; +/* schema */ export type DatasetSchemaType = { _id: string; parentId: string; @@ -33,13 +41,33 @@ export type DatasetCollectionSchemaType = { }; }; +export type DatasetDataIndexItemType = { + defaultIndex: boolean; + dataId: string; // pg data id + type: `${DatasetDataIndexTypeEnum}`; + text: string; +}; +export type DatasetDataSchemaType = { + _id: string; + userId: string; + teamId: string; + tmbId: string; + datasetId: string; + collectionId: string; + datasetId: string; + collectionId: string; + q: string; // large chunks or question + a: string; // answer or custom content + indexes: DatasetDataIndexItemType[]; +}; + export type DatasetTrainingSchemaType = { _id: string; userId: string; teamId: string; tmbId: string; datasetId: string; - datasetCollectionId: string; + collectionId: string; billId: string; expireAt: Date; lockTime: Date; @@ -48,6 +76,7 @@ export type DatasetTrainingSchemaType = { prompt: string; q: string; a: string; + indexes: Omit[]; }; export type CollectionWithDatasetType = Omit & { @@ -55,41 +84,31 @@ export type CollectionWithDatasetType = Omit & { + vectorModel: VectorModelItemType; + isOwner: boolean; canWrite: boolean; }; +/* ================= collection ===================== */ +export type DatasetCollectionItemType = CollectionWithDatasetType & { + canWrite: boolean; + sourceName: string; + sourceId?: string; +}; + /* ================= data ===================== */ -export type PgRawDataItemType = { - id: string; - q: string; - a: string; - team_id: string; - tmb_id: string; - dataset_id: string; - collection_id: string; -}; -export type PgDataItemType = { - id: string; - q: string; - a: string; - teamId: string; - tmbId: string; - datasetId: string; - collectionId: string; -}; -export type DatasetChunkItemType = { - q: string; - a: string; -}; -export type DatasetDataItemType = DatasetChunkItemType & { +export type DatasetDataItemType = { id: string; datasetId: string; collectionId: string; sourceName: string; sourceId?: string; + q: string; + a: string; + indexes: DatasetDataIndexItemType[]; + isOwner: boolean; + canWrite: boolean; }; /* --------------- file ---------------------- */ @@ -109,9 +128,6 @@ export type DatasetFileSchema = { }; /* ============= search =============== */ -export type SearchDataResultItemType = PgRawDataItemType & { - score: number; -}; export type SearchDataResponseItemType = DatasetDataItemType & { score: number; }; diff --git a/packages/global/core/dataset/utils.ts b/packages/global/core/dataset/utils.ts index 17ce0034d..da848ad5a 100644 --- a/packages/global/core/dataset/utils.ts +++ b/packages/global/core/dataset/utils.ts @@ -1,4 +1,4 @@ -import { DatasetCollectionTypeEnum } from './constant'; +import { DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant'; import { getFileIcon } from '../../common/file/icon'; import { strIsLink } from '../../common/string/tools'; @@ -44,3 +44,14 @@ export function getSourceNameIcon({ } return '/imgs/files/collection.svg'; } + +export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: string }) { + const { q = '', a, dataId } = props || {}; + const qaStr = `${q}\n${a}`.trim(); + return { + defaultIndex: true, + type: a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk, + text: a ? qaStr : q, + dataId + }; +} diff --git a/packages/global/core/module/api.d.ts b/packages/global/core/module/api.d.ts index e69de29bb..2af9ed7eb 100644 --- a/packages/global/core/module/api.d.ts +++ b/packages/global/core/module/api.d.ts @@ -0,0 +1,3 @@ +import { VectorModelItemType } from '../ai/model.d'; + +export type SelectedDatasetType = { datasetId: string; vectorModel: VectorModelItemType }[]; diff --git a/packages/global/package.json b/packages/global/package.json index 17ff0598c..a4acf2986 100644 --- a/packages/global/package.json +++ b/packages/global/package.json @@ -6,7 +6,8 @@ "timezones-list": "^3.0.2", "dayjs": "^1.11.7", "encoding": "^0.1.13", - "openai": "^4.16.1" + "openai": "^4.16.1", + "js-tiktoken": "^1.0.7" }, "devDependencies": { "@types/node": "^20.8.5" diff --git a/packages/global/support/user/team/constant.ts b/packages/global/support/user/team/constant.ts index 5768020b1..09b62b96d 100644 --- a/packages/global/support/user/team/constant.ts +++ b/packages/global/support/user/team/constant.ts @@ -24,7 +24,8 @@ export const TeamMemberRoleMap = { export enum TeamMemberStatusEnum { waiting = 'waiting', active = 'active', - reject = 'reject' + reject = 'reject', + leave = 'leave' } export const TeamMemberStatusMap = { [TeamMemberStatusEnum.waiting]: { @@ -38,5 +39,10 @@ export const TeamMemberStatusMap = { [TeamMemberStatusEnum.reject]: { label: 'user.team.member.reject', color: 'red.600' + }, + [TeamMemberStatusEnum.leave]: { + label: 'user.team.member.leave', + color: 'red.600' } }; +export const leaveStatus = { $ne: TeamMemberStatusEnum.leave }; diff --git a/packages/global/support/user/team/controller.d.ts b/packages/global/support/user/team/controller.d.ts index cf1f668be..663d9afab 100644 --- a/packages/global/support/user/team/controller.d.ts +++ b/packages/global/support/user/team/controller.d.ts @@ -37,4 +37,7 @@ export type UpdateInviteProps = { tmbId: string; status: TeamMemberSchema['status']; }; -export type InviteMemberResponse = Record<'invite' | 'inValid' | 'inTeam', string[]>; +export type InviteMemberResponse = Record< + 'invite' | 'inValid' | 'inTeam', + { username: string; userId: string }[] +>; diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts index 5373e7a42..a711910d6 100644 --- a/packages/global/support/user/team/type.d.ts +++ b/packages/global/support/user/team/type.d.ts @@ -16,6 +16,7 @@ export type TeamMemberSchema = { teamId: string; userId: string; createTime: Date; + name: string; role: `${TeamMemberRoleEnum}`; status: `${TeamMemberStatusEnum}`; defaultTeam: boolean; @@ -25,6 +26,7 @@ export type TeamItemType = { userId: string; teamId: string; teamName: string; + memberName: string; avatar: string; balance: number; tmbId: string; @@ -39,7 +41,7 @@ export type TeamMemberItemType = { userId: string; tmbId: string; teamId: string; - memberUsername: string; + memberName: string; avatar: string; role: `${TeamMemberRoleEnum}`; status: `${TeamMemberStatusEnum}`; diff --git a/packages/global/support/wallet/bill/type.d.ts b/packages/global/support/wallet/bill/type.d.ts index bacac7646..a676f8d28 100644 --- a/packages/global/support/wallet/bill/type.d.ts +++ b/packages/global/support/wallet/bill/type.d.ts @@ -15,7 +15,7 @@ export type BillSchema = CreateBillProps & { export type BillItemType = { id: string; - username: string; + memberName: string; time: Date; appName: string; source: BillSchema['source']; diff --git a/packages/service/common/api/plusRequest.ts b/packages/service/common/api/plusRequest.ts index 0186ca7b7..6aba9fe4e 100644 --- a/packages/service/common/api/plusRequest.ts +++ b/packages/service/common/api/plusRequest.ts @@ -15,9 +15,6 @@ interface ResponseDataType { * 请求开始 */ function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig { - if (config.headers) { - config.headers.rootkey = process.env.ROOT_KEY; - } return config; } @@ -62,7 +59,8 @@ const instance = axios.create({ timeout: 60000, // 超时时间 headers: { 'content-type': 'application/json', - 'Cache-Control': 'no-cache' + 'Cache-Control': 'no-cache', + rootkey: process.env.ROOT_KEY } }); diff --git a/packages/service/common/pg/index.ts b/packages/service/common/pg/index.ts index cc9177ffb..dccf9834e 100644 --- a/packages/service/common/pg/index.ts +++ b/packages/service/common/pg/index.ts @@ -171,8 +171,7 @@ export async function initPg() { tmb_id VARCHAR(50) NOT NULL, dataset_id VARCHAR(50) NOT NULL, collection_id VARCHAR(50) NOT NULL, - q TEXT NOT NULL, - a TEXT + data_id VARCHAR(50) NOT NULL ); CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 24, ef_construction = 64); `); diff --git a/projects/app/src/service/common/tiktoken.ts b/packages/service/core/chat/utils.ts similarity index 74% rename from projects/app/src/service/common/tiktoken.ts rename to packages/service/core/chat/utils.ts index 1136faf0b..6272b7c56 100644 --- a/projects/app/src/service/common/tiktoken.ts +++ b/packages/service/core/chat/utils.ts @@ -1,22 +1,7 @@ import type { ChatItemType } from '@fastgpt/global/core/chat/type.d'; import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; -import type { NextApiResponse } from 'next'; -import { countMessagesTokens, countPromptTokens } from '@/global/common/tiktoken'; -import { adaptRole_Chat2Message } from '@/utils/common/adapt/message'; - -export type ChatCompletionResponseType = { - streamResponse: any; - responseMessages: ChatItemType[]; - responseText: string; - totalTokens: number; -}; -export type StreamResponseType = { - chatResponse: any; - messages: ChatItemType[]; - res: NextApiResponse; - model: string; - [key: string]: any; -}; +import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; +import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt'; /* slice chat context by tokens */ export function ChatContextFilter({ diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 52800de12..575cf851e 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -56,8 +56,7 @@ const DatasetCollectionSchema = new Schema({ ref: 'dataset.files' }, rawLink: { - type: String, - default: '' + type: String }, // 451 初始化 pgCollectionId: { diff --git a/packages/service/core/dataset/controller.ts b/packages/service/core/dataset/controller.ts index f6f9ed5b1..baf81aa4a 100644 --- a/packages/service/core/dataset/controller.ts +++ b/packages/service/core/dataset/controller.ts @@ -1,5 +1,25 @@ import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type'; import { MongoDatasetCollection } from './collection/schema'; +import { MongoDataset } from './schema'; + +/* ============= dataset ========== */ +/* find all datasetId by top datasetId */ +export async function findDatasetIdTreeByTopDatasetId( + id: string, + result: string[] = [] +): Promise { + let allChildrenIds = [...result]; + + // find children + const children = await MongoDataset.find({ parentId: id }); + + for (const child of children) { + const grandChildrenIds = await findDatasetIdTreeByTopDatasetId(child._id, result); + allChildrenIds = allChildrenIds.concat(grandChildrenIds); + } + + return [String(id), ...allChildrenIds]; +} export async function getCollectionWithDataset(collectionId: string) { const data = ( diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts new file mode 100644 index 000000000..17bfe01ae --- /dev/null +++ b/packages/service/core/dataset/data/schema.ts @@ -0,0 +1,78 @@ +import { connectionMongo, type Model } from '../../../common/mongo'; +const { Schema, model, models } = connectionMongo; +import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d'; +import { + TeamCollectionName, + TeamMemberCollectionName +} from '@fastgpt/global/support/user/team/constant'; +import { DatasetCollectionName } from '../schema'; +import { DatasetColCollectionName } from '../collection/schema'; +import { DatasetDataIndexTypeMap } from '@fastgpt/global/core/dataset/constant'; + +export const DatasetDataCollectionName = 'dataset.datas'; + +const DatasetDataSchema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + tmbId: { + type: Schema.Types.ObjectId, + ref: TeamMemberCollectionName, + required: true + }, + datasetId: { + type: Schema.Types.ObjectId, + ref: DatasetCollectionName, + required: true + }, + collectionId: { + type: Schema.Types.ObjectId, + ref: DatasetColCollectionName, + required: true + }, + q: { + type: String, + required: true + }, + a: { + type: String, + default: '' + }, + indexes: { + type: [ + { + defaultIndex: { + type: Boolean, + default: false + }, + type: { + type: String, + enum: Object.keys(DatasetDataIndexTypeMap), + required: true + }, + dataId: { + type: String, + required: true + }, + text: { + type: String, + required: true + } + } + ], + default: [] + } +}); + +try { + DatasetDataSchema.index({ userId: 1 }); + DatasetDataSchema.index({ datasetId: 1 }); + DatasetDataSchema.index({ collectionId: 1 }); +} catch (error) { + console.log(error); +} + +export const MongoDatasetData: Model = + models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema); diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index 249601fdc..d521f1dc8 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type'; -import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; +import { DatasetDataIndexTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; import { DatasetColCollectionName } from '../collection/schema'; import { DatasetCollectionName } from '../schema'; import { @@ -33,12 +33,13 @@ const TrainingDataSchema = new Schema({ ref: DatasetCollectionName, required: true }, - datasetCollectionId: { + collectionId: { type: Schema.Types.ObjectId, ref: DatasetColCollectionName, required: true }, billId: { + // concat bill type: String, default: '' }, @@ -48,6 +49,7 @@ const TrainingDataSchema = new Schema({ required: true }, expireAt: { + // It will be deleted after 7 days type: Date, default: () => new Date() }, @@ -56,6 +58,7 @@ const TrainingDataSchema = new Schema({ default: () => new Date('2000/1/1') }, model: { + // ai model type: String, required: true }, @@ -71,13 +74,29 @@ const TrainingDataSchema = new Schema({ a: { type: String, default: '' + }, + indexes: { + type: [ + { + type: { + type: String, + enum: Object.keys(DatasetDataIndexTypeMap), + required: true + }, + text: { + type: String, + required: true + } + } + ], + default: [] } }); try { TrainingDataSchema.index({ lockTime: 1 }); TrainingDataSchema.index({ userId: 1 }); - TrainingDataSchema.index({ datasetCollectionId: 1 }); + TrainingDataSchema.index({ collectionId: 1 }); TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 }); } catch (error) { console.log(error); diff --git a/packages/service/support/activity/promotion/schema.ts b/packages/service/support/activity/promotion/schema.ts index 960a1f30b..5f9a52a5b 100644 --- a/packages/service/support/activity/promotion/schema.ts +++ b/packages/service/support/activity/promotion/schema.ts @@ -23,6 +23,7 @@ const PromotionRecordSchema = new Schema({ enum: ['pay', 'register'] }, amount: { + // 1 * PRICE_SCALE type: Number, required: true } diff --git a/packages/service/support/outLink/tools.ts b/packages/service/support/outLink/tools.ts index 8c4e7b2ca..71e2457c8 100644 --- a/packages/service/support/outLink/tools.ts +++ b/packages/service/support/outLink/tools.ts @@ -30,7 +30,7 @@ export const pushResult2Remote = async ({ shareId?: string; responseData?: any[]; }) => { - if (!shareId || !authToken) return; + if (!shareId || !authToken || !global.systemEnv.pluginBaseUrl) return; try { const outLink = await MongoOutLink.findOne({ shareId diff --git a/packages/service/support/permission/auth/common.ts b/packages/service/support/permission/auth/common.ts index 26f9091b5..7527a161c 100644 --- a/packages/service/support/permission/auth/common.ts +++ b/packages/service/support/permission/auth/common.ts @@ -1,5 +1,7 @@ +import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant'; import { parseHeaderCert } from '../controller'; import { AuthModeType } from '../type'; +import { authOutLinkValid } from './outLink'; export const authCert = async (props: AuthModeType) => { const result = await parseHeaderCert(props); @@ -10,3 +12,22 @@ export const authCert = async (props: AuthModeType) => { canWrite: true }; }; +export async function authCertAndShareId({ + shareId, + ...props +}: AuthModeType & { shareId?: string }) { + if (!shareId) { + return authCert(props); + } + + const { app } = await authOutLinkValid({ shareId }); + + return { + teamId: String(app.teamId), + tmbId: String(app.tmbId), + authType: AuthUserTypeEnum.outLink, + apikey: '', + isOwner: false, + canWrite: false + }; +} diff --git a/packages/service/support/permission/auth/dataset.ts b/packages/service/support/permission/auth/dataset.ts index e4c32fb5e..86ac29b60 100644 --- a/packages/service/support/permission/auth/dataset.ts +++ b/packages/service/support/permission/auth/dataset.ts @@ -27,11 +27,11 @@ export async function authDataset({ } > { const result = await parseHeaderCert(props); - const { userId, teamId, tmbId } = result; + const { teamId, tmbId } = result; const { role } = await getTeamInfoByTmbId({ tmbId }); const { dataset, isOwner, canWrite } = await (async () => { - const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toJSON(); + const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toObject(); if (!dataset) { return Promise.reject(DatasetErrEnum.unAuthDataset); diff --git a/packages/service/support/permission/auth/openapi.ts b/packages/service/support/permission/auth/openapi.ts index 8dc42418b..4fd444ef9 100644 --- a/packages/service/support/permission/auth/openapi.ts +++ b/packages/service/support/permission/auth/openapi.ts @@ -6,7 +6,6 @@ import { getTeamInfoByTmbId } from '../../user/team/controller'; import { MongoOpenApi } from '../../openapi/schema'; import { OpenApiErrEnum } from '@fastgpt/global/common/error/code/openapi'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; -import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; export async function authOpenApiKeyCrud({ id, diff --git a/packages/service/support/permission/auth/user.ts b/packages/service/support/permission/auth/user.ts index 171088dd6..bbcb2a1d1 100644 --- a/packages/service/support/permission/auth/user.ts +++ b/packages/service/support/permission/auth/user.ts @@ -37,13 +37,11 @@ export async function authUserRole(props: AuthModeType): Promise< teamOwner: boolean; } > { - const { userId, teamId, tmbId } = await parseHeaderCert(props); - const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId }); + const result = await parseHeaderCert(props); + const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId: result.tmbId }); return { - userId, - teamId, - tmbId, + ...result, isOwner: true, role: userRole, teamOwner: userRole === TeamMemberRoleEnum.owner, diff --git a/packages/service/support/user/team/controller.ts b/packages/service/support/user/team/controller.ts index 6ae909eaf..66eca315a 100644 --- a/packages/service/support/user/team/controller.ts +++ b/packages/service/support/user/team/controller.ts @@ -4,57 +4,42 @@ import { TeamMemberRoleEnum, TeamMemberStatusEnum, TeamCollectionName, - TeamMemberCollectionName + TeamMemberCollectionName, + leaveStatus } from '@fastgpt/global/support/user/team/constant'; -export async function getTeamInfoByTmbId({ - tmbId, - userId -}: { - tmbId?: string; - userId?: string; -}): Promise { - if (!tmbId && !userId) { - return Promise.reject('tmbId or userId is required'); - } - +async function getTeam(match: Record): Promise { const db = connectionMongo?.connection?.db; const TeamMember = db.collection(TeamMemberCollectionName); const results = await TeamMember.aggregate([ { - $match: tmbId - ? { - _id: new Types.ObjectId(tmbId) - } - : { - userId: new Types.ObjectId(userId), - defaultTeam: true - } + $match: match }, { $lookup: { - from: TeamCollectionName, // 关联的集合名 - localField: 'teamId', // TeamMember 集合中用于关联的字段 - foreignField: '_id', // Team 集合中用于关联的字段 - as: 'team' // 查询结果中的字段名，存放关联查询的结果 + from: TeamCollectionName, + localField: 'teamId', + foreignField: '_id', + as: 'team' } }, { - $unwind: '$team' // 将查询结果中的 team 字段展开，变成一个对象 + $unwind: '$team' } ]).toArray(); const tmb = results[0]; if (!tmb) { - return Promise.reject('team not exist'); + return Promise.reject('member not exist'); } return { userId: String(tmb.userId), teamId: String(tmb.teamId), teamName: tmb.team.name, + memberName: tmb.name, avatar: tmb.team.avatar, balance: tmb.team.balance, tmbId: String(tmb._id), @@ -65,11 +50,31 @@ export async function getTeamInfoByTmbId({ maxSize: tmb.team.maxSize }; } + +export async function getTeamInfoByTmbId({ tmbId }: { tmbId: string }) { + if (!tmbId) { + return Promise.reject('tmbId or userId is required'); + } + return getTeam({ + _id: new Types.ObjectId(tmbId), + status: leaveStatus + }); +} + +export async function getUserDefaultTeam({ userId }: { userId: string }) { + if (!userId) { + return Promise.reject('tmbId or userId is required'); + } + return getTeam({ + userId: new Types.ObjectId(userId), + defaultTeam: true + }); +} export async function createDefaultTeam({ userId, teamName = 'My Team', avatar = '/icon/logo.svg', - balance = 0, + balance, maxSize = 5 }: { userId: string; @@ -103,6 +108,7 @@ export async function createDefaultTeam({ await TeamMember.insertOne({ teamId: insertedId, userId, + name: 'Owner', role: TeamMemberRoleEnum.owner, status: TeamMemberStatusEnum.active, createTime: new Date(), @@ -116,7 +122,7 @@ export async function createDefaultTeam({ }, { $set: { - balance, + ...(balance !== undefined && { balance }), maxSize } } diff --git a/packages/service/support/wallet/bill/schema.ts b/packages/service/support/wallet/bill/schema.ts index 8673af155..275577624 100644 --- a/packages/service/support/wallet/bill/schema.ts +++ b/packages/service/support/wallet/bill/schema.ts @@ -36,6 +36,7 @@ const BillSchema = new Schema({ default: () => new Date() }, total: { + // 1 * PRICE_SCALE type: Number, required: true }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f2225007..ec2c0db8f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -51,6 +51,9 @@ importers: encoding: specifier: ^0.1.13 version: registry.npmmirror.com/encoding@0.1.13 + js-tiktoken: + specifier: ^1.0.7 + version: registry.npmmirror.com/js-tiktoken@1.0.7 openai: specifier: ^4.16.1 version: registry.npmmirror.com/openai@4.16.1(encoding@0.1.13) @@ -200,12 +203,6 @@ importers: immer: specifier: ^9.0.19 version: registry.npmmirror.com/immer@9.0.21 - js-cookie: - specifier: ^3.0.5 - version: registry.npmmirror.com/js-cookie@3.0.5 - js-tiktoken: - specifier: ^1.0.7 - version: registry.npmmirror.com/js-tiktoken@1.0.7 jschardet: specifier: ^3.0.0 version: registry.npmmirror.com/jschardet@3.0.0 @@ -242,9 +239,6 @@ importers: papaparse: specifier: ^5.4.1 version: registry.npmmirror.com/papaparse@5.4.1 - pg-query-stream: - specifier: ^4.5.3 - version: registry.npmmirror.com/pg-query-stream@4.5.3(pg@8.11.3) react: specifier: 18.2.0 version: registry.npmmirror.com/react@18.2.0 @@ -8721,13 +8715,6 @@ packages: set-function-name: registry.npmmirror.com/set-function-name@2.0.1 dev: true - registry.npmmirror.com/js-cookie@3.0.5: - resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-cookie/-/js-cookie-3.0.5.tgz} - name: js-cookie - version: 3.0.5 - engines: {node: '>=14'} - dev: false - registry.npmmirror.com/js-sdsl@4.4.2: resolution: {integrity: sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/js-sdsl/-/js-sdsl-4.4.2.tgz} name: js-sdsl @@ -10629,17 +10616,6 @@ packages: version: 2.6.2 dev: false - registry.npmmirror.com/pg-cursor@2.10.3(pg@8.11.3): - resolution: {integrity: sha512-rDyBVoqPVnx/PTmnwQAYgusSeAKlTL++gmpf5klVK+mYMFEqsOc6VHHZnPKc/4lOvr4r6fiMuoxSFuBF1dx4FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-cursor/-/pg-cursor-2.10.3.tgz} - id: registry.npmmirror.com/pg-cursor/2.10.3 - name: pg-cursor - version: 2.10.3 - peerDependencies: - pg: ^8 - dependencies: - pg: registry.npmmirror.com/pg@8.11.3 - dev: false - registry.npmmirror.com/pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-int8/-/pg-int8-1.0.1.tgz} name: pg-int8 @@ -10669,18 +10645,6 @@ packages: name: pg-protocol version: 1.6.0 - registry.npmmirror.com/pg-query-stream@4.5.3(pg@8.11.3): - resolution: {integrity: sha512-ufa94r/lHJdjAm3+zPZEO0gXAmCb4tZPaOt7O76mjcxdL/HxwTuryy76km+u0odBBgtfdKFYq/9XGfiYeQF0yA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-query-stream/-/pg-query-stream-4.5.3.tgz} - id: registry.npmmirror.com/pg-query-stream/4.5.3 - name: pg-query-stream - version: 4.5.3 - peerDependencies: - pg: ^8 - dependencies: - pg: registry.npmmirror.com/pg@8.11.3 - pg-cursor: registry.npmmirror.com/pg-cursor@2.10.3(pg@8.11.3) - dev: false - registry.npmmirror.com/pg-types@2.2.0: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/pg-types/-/pg-types-2.2.0.tgz} name: pg-types diff --git a/projects/app/next-i18next.config.js b/projects/app/next-i18next.config.js index 1f8c9fa69..65e1e918e 100644 --- a/projects/app/next-i18next.config.js +++ b/projects/app/next-i18next.config.js @@ -8,5 +8,8 @@ module.exports = { defaultLocale: 'zh', locales: ['en', 'zh', 'zh-Hans', 'zh-CN'], localeDetection: false - } + }, + localePath: + typeof window === 'undefined' ? require('path').resolve('./public/locales') : '/locales', + reloadOnPrerender: process.env.NODE_ENV === 'development' }; diff --git a/projects/app/package.json b/projects/app/package.json index 5e1565a79..041439161 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -33,8 +33,6 @@ "hyperdown": "^2.4.29", "i18next": "^22.5.1", "immer": "^9.0.19", - "js-cookie": "^3.0.5", - "js-tiktoken": "^1.0.7", "jschardet": "^3.0.0", "jsdom": "^22.1.0", "jsonwebtoken": "^9.0.2", @@ -47,7 +45,6 @@ "next-i18next": "^13.3.0", "nprogress": "^0.2.0", "papaparse": "^5.4.1", - "pg-query-stream": "^4.5.3", "react": "18.2.0", "react-day-picker": "^8.7.1", "react-dom": "18.2.0", diff --git a/projects/app/public/docs/versionIntro.md b/projects/app/public/docs/versionIntro.md index e4427ffbc..47e3c9d1f 100644 --- a/projects/app/public/docs/versionIntro.md +++ b/projects/app/public/docs/versionIntro.md @@ -1,9 +1,12 @@ ### Fast GPT V4.6 -1. 新增 - 团队空间试用版。 -2. 新增 - OpenAI语音播报。 -3. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/) -4. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词) -5. [使用文档](https://doc.fastgpt.in/docs/intro/) -6. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow) -7. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/) +1. 新增 - 团队空间 +2. 新增 - 多路向量（多个向量映射一组数据） +3. 新增 - tts语音 +4. 线上环境新增 - ReRank向量召回，提高召回精度 +5. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈 +6. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/) +7. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词) +8. [使用文档](https://doc.fastgpt.in/docs/intro/) +9. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow) +10. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/) diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 265c59a7d..4808b7cbb 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -292,17 +292,32 @@ "Select One Collection To Store": "Select the collection to store" }, "data": { + "Add Index": "Add Index", "Can not delete tip": "No modification permission", "Can not edit": "No edit permission", + "Custom Index Number": "Custom index{{number}}", + "Default Index": "Default Index", + "Delete Success Tip": "", "Delete Tip": "Confirm to delete the data?", "File import": "File Import", + "Index Edit": "Data Index", + "Index Placeholder": "Enter the index text content", "Input Data": "Import Data", "Input Success Tip": "Succeeded in importing data", "Update Data": "Update Data", - "Update Success Tip": "Update data successfully" + "Update Success Tip": "Update data successfully", + "edit": { + "Content": "Content", + "Course": "Document", + "Delete": "Delete", + "Index": "Index({{amount}})" + } }, "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!", - "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!" + "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!", + "test": { + "noResult": "Search results are empty" + } }, "error": { "team": { @@ -461,9 +476,11 @@ "Bill Detail": "Bill Detail", "Change": "Change", "Copy invite url": "Copy invitation link", + "Edit name": "Click to modify nickname", "Invite Url": "Invite Url", "Invite url tip": "Friends who register through this link will be permanently bound to you, and you will get a certain balance reward when they recharge. In addition, when friends register with their mobile phone number, you will get 5 yuan reward immediately.", "Language": "Language", + "Member Name": "Name", "Notice": "Notice", "Old password is error": "Old password is error", "OpenAI Account Setting": "OpenAI Account Setting", @@ -513,12 +530,15 @@ "Leave Team Failed": "Leave Team Failed", "Manage": "Team Manage", "Member": "Member", + "Member Name": "Member", "Over Max Member Tip": "Team max {{max}} people", "Personal Team": "Personal", "Processing invitations": "Processing invitations", "Processing invitations Tips": "You have {{amount}} of team invitations that need to be processed", "Reinvite": "Reinvite", - "Remove Member Confirm Tip": "Confirm to move {{username}} off the team", + "Remove Member Confirm Tip": "Are you sure you want to move {{username}} off the team? All its resources are transferred to the team creator's account.", + "Remove Member Failed": "Removing a team member failed", + "Remove Member Success": "Removing a team member succeeded", "Remove Member Tip": "Move out team", "Role": "Role", "Select Team": "Select Team", diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 07adf070e..8053b1cc8 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -292,17 +292,32 @@ "Select One Collection To Store": "选择一个文件进行存储" }, "data": { + "Add Index": "新增自定义索引", "Can not delete tip": "无修改权限", "Can not edit": "无编辑权限", + "Custom Index Number": "自定义索引{{number}}", + "Default Index": "默认索引", + "Delete Success Tip": "删除成功", "Delete Tip": "确认删除该条数据？", "File import": "文件导入", + "Index Edit": "数据索引", + "Index Placeholder": "输入索引文本内容", "Input Data": "导入新数据", "Input Success Tip": "导入数据成功", "Update Data": "更新数据", - "Update Success Tip": "更新数据成功" + "Update Success Tip": "更新数据成功", + "edit": { + "Content": "数据内容", + "Course": "说明文档", + "Delete": "删除数据", + "Index": "数据索引({{amount}})" + } }, "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！", - "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！" + "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！", + "test": { + "noResult": "搜索结果为空" + } }, "error": { "team": { @@ -461,9 +476,11 @@ "Bill Detail": "账单详情", "Change": "变更", "Copy invite url": "复制邀请链接", + "Edit name": "点击修改昵称", "Invite Url": "邀请链接", "Invite url tip": "通过该链接注册的好友将永久与你绑定，其充值时你会获得一定余额奖励。\n此外，好友使用手机号注册时，你将立即获得 5 元奖励。\n奖励会发送到您的默认团队中。", "Language": "语言", + "Member Name": "昵称", "Notice": "通知", "Old password is error": "旧密码错误", "OpenAI Account Setting": "OpenAI 账号配置", @@ -513,12 +530,15 @@ "Leave Team Failed": "离开团队异常", "Manage": "团队管理", "Member": "成员", + "Member Name": "成员名", "Over Max Member Tip": "团队最多{{max}}人", "Personal Team": "个人团队", "Processing invitations": "处理邀请", "Processing invitations Tips": "你有{{amount}}个需要处理的团队邀请", "Reinvite": "重新邀请", - "Remove Member Confirm Tip": "确认将 {{username}} 移出团队？", + "Remove Member Confirm Tip": "确认将 {{username}} 移出团队？其所有资源将转让到团队创建者的账户内。", + "Remove Member Failed": "移除团队成员异常", + "Remove Member Success": "移除团队成员成功", "Remove Member Tip": "移出团队", "Role": "身份", "Select Team": "团队选择", diff --git a/projects/app/src/components/ChatBox/QuoteModal.tsx b/projects/app/src/components/ChatBox/QuoteModal.tsx index 0c2c88ab9..7c4b3ce7e 100644 --- a/projects/app/src/components/ChatBox/QuoteModal.tsx +++ b/projects/app/src/components/ChatBox/QuoteModal.tsx @@ -10,13 +10,12 @@ import InputDataModal, { type InputDataType } from '@/pages/dataset/detail/components/InputDataModal'; import MyModal from '../MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useRouter } from 'next/router'; import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; import MyTooltip from '../MyTooltip'; import NextLink from 'next/link'; import { useSystemStore } from '@/web/common/system/useSystemStore'; -import { useUserStore } from '@/web/support/user/useUserStore'; const QuoteModal = ({ rawSearch = [], @@ -29,10 +28,9 @@ const QuoteModal = ({ const { isPc } = useSystemStore(); const theme = useTheme(); const router = useRouter(); - const { userInfo } = useUserStore(); const { toast } = useToast(); const { setIsLoading, Loading } = useLoading(); - const [editInputData, setEditInputData] = useState(); + const [editInputData, setEditInputData] = useState(); const isShare = useMemo(() => router.pathname === '/chat/share', [router.pathname]); @@ -99,7 +97,7 @@ const QuoteModal = ({ color={'black'} sourceName={item.sourceName} sourceId={item.sourceId} - addr={!isShare} + canView={!isShare} /> {!isShare && ( @@ -131,7 +129,7 @@ const QuoteModal = ({ - {item.q.length + item.a.length} + {item.q.length + (item.a?.length || 0)} {!isShare && item.score && ( @@ -183,7 +181,6 @@ const QuoteModal = ({ {editInputData && editInputData.id && ( setEditInputData(undefined)} onSuccess={() => { console.log('更新引用成功'); @@ -191,8 +188,8 @@ const QuoteModal = ({ onDelete={() => { console.log('删除引用成功'); }} - datasetId={editInputData.datasetId} - defaultValues={editInputData} + defaultValue={editInputData} + collectionId={editInputData.collectionId} /> )} diff --git a/projects/app/src/components/ChatBox/ResponseTags.tsx b/projects/app/src/components/ChatBox/ResponseTags.tsx index d695fe842..937735633 100644 --- a/projects/app/src/components/ChatBox/ResponseTags.tsx +++ b/projects/app/src/components/ChatBox/ResponseTags.tsx @@ -2,7 +2,7 @@ import React, { useMemo, useState } from 'react'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/api.d'; import type { ChatItemType } from '@fastgpt/global/core/chat/type'; import { Flex, BoxProps, useDisclosure, Image, useTheme } from '@chakra-ui/react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; import dynamic from 'next/dynamic'; diff --git a/projects/app/src/components/ChatBox/SelectMarkCollection.tsx b/projects/app/src/components/ChatBox/SelectMarkCollection.tsx index fca962490..6a11d22ee 100644 --- a/projects/app/src/components/ChatBox/SelectMarkCollection.tsx +++ b/projects/app/src/components/ChatBox/SelectMarkCollection.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useMemo, useState } from 'react'; import { ModalBody, useTheme, ModalFooter, Button, Box, Card, Flex, Grid } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; import Avatar from '../Avatar'; @@ -8,6 +8,7 @@ import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/ import dynamic from 'next/dynamic'; import { AdminFbkType } from '@fastgpt/global/core/chat/type.d'; import SelectCollections from '@/web/core/dataset/components/SelectCollections'; +import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; const InputDataModal = dynamic(() => import('@/pages/dataset/detail/components/InputDataModal')); @@ -163,15 +164,13 @@ const SelectMarkCollection = ({ {adminMarkData.datasetId && adminMarkData.collectionId && ( { if (!data.q || !adminMarkData.datasetId || !adminMarkData.collectionId || !data.id) { return onClose(); diff --git a/projects/app/src/components/ChatBox/WholeResponseModal.tsx b/projects/app/src/components/ChatBox/WholeResponseModal.tsx index c3f1ff8c7..bcfe513ca 100644 --- a/projects/app/src/components/ChatBox/WholeResponseModal.tsx +++ b/projects/app/src/components/ChatBox/WholeResponseModal.tsx @@ -1,7 +1,7 @@ import React, { useMemo, useState } from 'react'; import { Box, useTheme, Flex, Image } from '@chakra-ui/react'; import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/api.d'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { ModuleTemplatesFlat } from '@/constants/flow/ModuleTemplate'; import Tabs from '../Tabs'; diff --git a/projects/app/src/components/ChatBox/index.tsx b/projects/app/src/components/ChatBox/index.tsx index fb326f55a..133ea7b1a 100644 --- a/projects/app/src/components/ChatBox/index.tsx +++ b/projects/app/src/components/ChatBox/index.tsx @@ -30,7 +30,7 @@ import { } from '@chakra-ui/react'; import { feConfigs } from '@/web/common/system/staticData'; import { eventBus } from '@/web/common/utils/eventbus'; -import { adaptChat2GptMessages } from '@/utils/common/adapt/message'; +import { adaptChat2GptMessages } from '@fastgpt/global/core/chat/adapt'; import { useMarkdown } from '@/web/common/hooks/useMarkdown'; import { ModuleItemType } from '@fastgpt/global/core/module/type.d'; import { VariableInputEnum } from '@/constants/app'; @@ -41,7 +41,7 @@ import { htmlTemplate } from '@/constants/common'; import { useRouter } from 'next/router'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { TaskResponseKeyEnum } from '@fastgpt/global/core/chat/constants'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { customAlphabet } from 'nanoid'; import { adminUpdateChatFeedback, userUpdateChatFeedback } from '@/web/core/chat/api'; import type { AdminMarkType } from './SelectMarkCollection'; diff --git a/projects/app/src/components/CommunityModal/index.tsx b/projects/app/src/components/CommunityModal/index.tsx index 167c1bb3b..167f0fe9e 100644 --- a/projects/app/src/components/CommunityModal/index.tsx +++ b/projects/app/src/components/CommunityModal/index.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { Button, ModalFooter, ModalBody } from '@chakra-ui/react'; import MyModal from '../MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import Markdown from '../Markdown'; const md = ` diff --git a/projects/app/src/components/Language/index.tsx b/projects/app/src/components/Language/index.tsx deleted file mode 100644 index 0fbbd6335..000000000 --- a/projects/app/src/components/Language/index.tsx +++ /dev/null @@ -1,48 +0,0 @@ -import React, { useState } from 'react'; -import { Menu, MenuButton, MenuItem, MenuList, MenuButtonProps } from '@chakra-ui/react'; -import { getLangStore, LangEnum, setLangStore, langMap } from '@/web/common/utils/i18n'; -import MyIcon from '@/components/Icon'; -import { useTranslation } from 'react-i18next'; -import { useRouter } from 'next/router'; - -const Language = (props: MenuButtonProps) => { - const router = useRouter(); - const { i18n } = useTranslation(); - - const [language, setLanguage] = useState<`${LangEnum}`>(getLangStore()); - - return ( - - - - - - {Object.entries(langMap).map(([key, lang]) => ( - { - const lang = key as `${LangEnum}`; - setLangStore(lang); - setLanguage(lang); - i18n?.changeLanguage?.(lang); - router.reload(); - }} - > - {lang.label} - - ))} - - - ); -}; - -export default React.memo(Language); diff --git a/projects/app/src/components/Layout/navbarPhone.tsx b/projects/app/src/components/Layout/navbarPhone.tsx index 6153335c8..3462102c2 100644 --- a/projects/app/src/components/Layout/navbarPhone.tsx +++ b/projects/app/src/components/Layout/navbarPhone.tsx @@ -2,7 +2,7 @@ import React, { useMemo } from 'react'; import { useRouter } from 'next/router'; import { Flex, Box } from '@chakra-ui/react'; import { useChatStore } from '@/web/core/chat/storeChat'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import Badge from '../Badge'; import MyIcon from '../Icon'; diff --git a/projects/app/src/components/Tabs/index.tsx b/projects/app/src/components/Tabs/index.tsx index bba605a00..5e4a5e4cd 100644 --- a/projects/app/src/components/Tabs/index.tsx +++ b/projects/app/src/components/Tabs/index.tsx @@ -1,7 +1,7 @@ import React, { useMemo } from 'react'; import { Box, Grid } from '@chakra-ui/react'; import type { GridProps } from '@chakra-ui/react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; // @ts-ignore interface Props extends GridProps { diff --git a/projects/app/src/components/common/ParentPaths/index.tsx b/projects/app/src/components/common/ParentPaths/index.tsx index b3cd6ae69..72a4939a0 100644 --- a/projects/app/src/components/common/ParentPaths/index.tsx +++ b/projects/app/src/components/common/ParentPaths/index.tsx @@ -2,7 +2,7 @@ import MyIcon from '@/components/Icon'; import { Box, Flex } from '@chakra-ui/react'; import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type'; import React, { useMemo } from 'react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; const ParentPaths = (props: { paths?: ParentTreePathItemType[]; diff --git a/projects/app/src/components/common/Textarea/TagTextarea.tsx b/projects/app/src/components/common/Textarea/TagTextarea.tsx index 5b26a8189..10d2f6ff4 100644 --- a/projects/app/src/components/common/Textarea/TagTextarea.tsx +++ b/projects/app/src/components/common/Textarea/TagTextarea.tsx @@ -10,7 +10,7 @@ import { useTheme } from '@chakra-ui/react'; import { useToast } from '@/web/common/hooks/useToast'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; type Props = BoxProps & { defaultValues: string[]; onUpdate: (e: string[]) => void }; diff --git a/projects/app/src/components/core/dataset/SelectModal.tsx b/projects/app/src/components/core/dataset/SelectModal.tsx index e6891a928..b6eb3075e 100644 --- a/projects/app/src/components/core/dataset/SelectModal.tsx +++ b/projects/app/src/components/core/dataset/SelectModal.tsx @@ -2,7 +2,7 @@ import { getDatasets, getDatasetPaths } from '@/web/core/dataset/api'; import MyModal from '@/components/MyModal'; import { useQuery } from '@tanstack/react-query'; import React, { Dispatch, useMemo, useState } from 'react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { Box, Flex, ModalHeader } from '@chakra-ui/react'; import MyIcon from '@/components/Icon'; diff --git a/projects/app/src/components/core/module/AIChatSettingsModal.tsx b/projects/app/src/components/core/module/AIChatSettingsModal.tsx index 195547f78..ff2e990c2 100644 --- a/projects/app/src/components/core/module/AIChatSettingsModal.tsx +++ b/projects/app/src/components/core/module/AIChatSettingsModal.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useMemo, useState } from 'react'; import MyModal from '@/components/MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { EditFormType } from '@/web/core/app/basicSettings'; import { useForm } from 'react-hook-form'; import { diff --git a/projects/app/src/components/core/module/DatasetSelectModal.tsx b/projects/app/src/components/core/module/DatasetSelectModal.tsx index 4443910c4..015a63a68 100644 --- a/projects/app/src/components/core/module/DatasetSelectModal.tsx +++ b/projects/app/src/components/core/module/DatasetSelectModal.tsx @@ -14,14 +14,14 @@ import { import Avatar from '@/components/Avatar'; import { useForm } from 'react-hook-form'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; -import type { SelectedDatasetType } from '@/types/core/dataset'; +import type { SelectedDatasetType } from '@fastgpt/global/core/module/api.d'; import { useToast } from '@/web/common/hooks/useToast'; import MySlider from '@/components/Slider'; import MyTooltip from '@/components/MyTooltip'; import MyModal from '@/components/MyModal'; import MyIcon from '@/components/Icon'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { feConfigs } from '@/web/common/system/staticData'; import DatasetSelectContainer, { useDatasetSelect } from '@/components/core/dataset/SelectModal'; diff --git a/projects/app/src/components/core/module/Flow/ImportSettings.tsx b/projects/app/src/components/core/module/Flow/ImportSettings.tsx index 0883bb382..d1c38fbfa 100644 --- a/projects/app/src/components/core/module/Flow/ImportSettings.tsx +++ b/projects/app/src/components/core/module/Flow/ImportSettings.tsx @@ -1,7 +1,7 @@ import React, { useState } from 'react'; import { Textarea, Button, ModalBody, ModalFooter } from '@chakra-ui/react'; import MyModal from '@/components/MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useToast } from '@/web/common/hooks/useToast'; import { useFlowProviderStore } from './FlowProvider'; diff --git a/projects/app/src/components/core/module/Flow/SelectAppModal.tsx b/projects/app/src/components/core/module/Flow/SelectAppModal.tsx index 597f15ee6..e6fcb0c04 100644 --- a/projects/app/src/components/core/module/Flow/SelectAppModal.tsx +++ b/projects/app/src/components/core/module/Flow/SelectAppModal.tsx @@ -4,7 +4,7 @@ import MyModal from '@/components/MyModal'; import { useQuery } from '@tanstack/react-query'; import type { SelectAppItemType } from '@fastgpt/global/core/module/type'; import Avatar from '@/components/Avatar'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useLoading } from '@/web/common/hooks/useLoading'; import { useUserStore } from '@/web/support/user/useUserStore'; import { useAppStore } from '@/web/core/app/store/useAppStore'; diff --git a/projects/app/src/components/core/module/Flow/TemplateList.tsx b/projects/app/src/components/core/module/Flow/TemplateList.tsx index 5fd6c43a5..35b8c380e 100644 --- a/projects/app/src/components/core/module/Flow/TemplateList.tsx +++ b/projects/app/src/components/core/module/Flow/TemplateList.tsx @@ -10,7 +10,7 @@ import Avatar from '@/components/Avatar'; import { useFlowProviderStore } from './FlowProvider'; import { customAlphabet } from 'nanoid'; import { appModule2FlowNode } from '@/utils/adapt'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useRouter } from 'next/router'; const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6); import MyIcon from '@/components/Icon'; diff --git a/projects/app/src/components/core/module/Flow/components/modules/FieldEditModal.tsx b/projects/app/src/components/core/module/Flow/components/modules/FieldEditModal.tsx index 0ffa3c2dc..2af10d86a 100644 --- a/projects/app/src/components/core/module/Flow/components/modules/FieldEditModal.tsx +++ b/projects/app/src/components/core/module/Flow/components/modules/FieldEditModal.tsx @@ -13,7 +13,7 @@ import { useForm } from 'react-hook-form'; import MyModal from '@/components/MyModal'; import Avatar from '@/components/Avatar'; import { FlowNodeValTypeEnum } from '@fastgpt/global/core/module/node/constant'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import MySelect from '@/components/Select'; const typeSelectList = [ diff --git a/projects/app/src/components/core/module/Flow/components/modules/NodeCard.tsx b/projects/app/src/components/core/module/Flow/components/modules/NodeCard.tsx index 5e8b561b0..86a21451b 100644 --- a/projects/app/src/components/core/module/Flow/components/modules/NodeCard.tsx +++ b/projects/app/src/components/core/module/Flow/components/modules/NodeCard.tsx @@ -5,7 +5,7 @@ import Avatar from '@/components/Avatar'; import type { FlowModuleItemType } from '@fastgpt/global/core/module/type.d'; import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useEditTitle } from '@/web/common/hooks/useEditTitle'; import { useToast } from '@/web/common/hooks/useToast'; import { useFlowProviderStore, onChangeNode } from '../../FlowProvider'; diff --git a/projects/app/src/components/core/module/Flow/components/nodes/NodeCQNode.tsx b/projects/app/src/components/core/module/Flow/components/nodes/NodeCQNode.tsx index bf217d322..630c61c9c 100644 --- a/projects/app/src/components/core/module/Flow/components/nodes/NodeCQNode.tsx +++ b/projects/app/src/components/core/module/Flow/components/nodes/NodeCQNode.tsx @@ -15,7 +15,7 @@ import { FlowNodeValTypeEnum, FlowNodeSpecialInputKeyEnum } from '@fastgpt/global/core/module/node/constant'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import SourceHandle from '../render/SourceHandle'; import MyTooltip from '@/components/MyTooltip'; import { onChangeNode } from '../../FlowProvider'; diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx index 5f92576ab..2031b2a0e 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx @@ -26,12 +26,12 @@ import MySlider from '@/components/Slider'; import MyTooltip from '@/components/MyTooltip'; import TargetHandle from './TargetHandle'; import MyIcon from '@/components/Icon'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { AIChatProps } from '@/types/core/aiChat'; import { chatModelList } from '@/web/common/system/staticData'; import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; -import { SelectedDatasetType } from '@/types/core/dataset'; +import type { SelectedDatasetType } from '@fastgpt/global/core/module/api.d'; import { useQuery } from '@tanstack/react-query'; import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d'; import type { EditFieldModeType, EditFieldType } from '../modules/FieldEditModal'; diff --git a/projects/app/src/components/support/apikey/Table.tsx b/projects/app/src/components/support/apikey/Table.tsx index ae35119fe..a687d4b96 100644 --- a/projects/app/src/components/support/apikey/Table.tsx +++ b/projects/app/src/components/support/apikey/Table.tsx @@ -33,7 +33,7 @@ import dayjs from 'dayjs'; import { AddIcon, QuestionOutlineIcon } from '@chakra-ui/icons'; import { useCopyData } from '@/web/common/hooks/useCopyData'; import { feConfigs } from '@/web/common/system/staticData'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import MyIcon from '@/components/Icon'; import MyModal from '@/components/MyModal'; import { useForm } from 'react-hook-form'; diff --git a/projects/app/src/components/support/permission/IconText/index.tsx b/projects/app/src/components/support/permission/IconText/index.tsx index 515fba31e..45f1293b0 100644 --- a/projects/app/src/components/support/permission/IconText/index.tsx +++ b/projects/app/src/components/support/permission/IconText/index.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant'; import { Box, Flex, FlexProps } from '@chakra-ui/react'; import MyIcon from '@/components/Icon'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; const PermissionIconText = ({ permission, diff --git a/projects/app/src/components/support/permission/Radio/index.tsx b/projects/app/src/components/support/permission/Radio/index.tsx index 2c86e5973..6f29bb53a 100644 --- a/projects/app/src/components/support/permission/Radio/index.tsx +++ b/projects/app/src/components/support/permission/Radio/index.tsx @@ -1,7 +1,7 @@ import React from 'react'; import MyRadio from '@/components/Radio'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; const PermissionRadio = ({ value, diff --git a/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx b/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx index 0ba2f7ea8..33d9d03d0 100644 --- a/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx +++ b/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx @@ -1,6 +1,6 @@ import React, { useCallback, useState } from 'react'; import { useForm } from 'react-hook-form'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { compressImgAndUpload } from '@/web/common/file/controller'; import { useToast } from '@/web/common/hooks/useToast'; diff --git a/projects/app/src/components/support/user/team/TeamManageModal/InviteModal.tsx b/projects/app/src/components/support/user/team/TeamManageModal/InviteModal.tsx index a0085dfcf..141b197d8 100644 --- a/projects/app/src/components/support/user/team/TeamManageModal/InviteModal.tsx +++ b/projects/app/src/components/support/user/team/TeamManageModal/InviteModal.tsx @@ -1,6 +1,6 @@ import React, { useMemo, useState } from 'react'; import MyModal from '@/components/MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { ModalCloseButton, ModalBody, Box, ModalFooter, Button } from '@chakra-ui/react'; import TagTextarea from '@/components/common/Textarea/TagTextarea'; import MySelect from '@/components/Select'; @@ -57,8 +57,8 @@ const InviteModal = ({ undefined, t('user.team.Invite Member Success Tip', { success: res.invite.length, - inValid: res.inValid.join(', '), - inTeam: res.inTeam.join(', ') + inValid: res.inValid.map((item) => item.username).join(', '), + inTeam: res.inTeam.map((item) => item.username).join(', ') }) )(); }, diff --git a/projects/app/src/components/support/user/team/TeamManageModal/index.tsx b/projects/app/src/components/support/user/team/TeamManageModal/index.tsx index 35d3e85dc..faf3bf6de 100644 --- a/projects/app/src/components/support/user/team/TeamManageModal/index.tsx +++ b/projects/app/src/components/support/user/team/TeamManageModal/index.tsx @@ -1,6 +1,6 @@ import React, { useMemo, useState } from 'react'; import MyModal from '@/components/MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useQuery } from '@tanstack/react-query'; import { getTeamList, @@ -100,7 +100,9 @@ const TeamManageModal = ({ onClose }: { onClose: () => void }) => { mutationFn: delRemoveMember, onSuccess() { refetchMembers(); - } + }, + successToast: t('user.team.Remove Member Success'), + errorToast: t('user.team.Remove Member Failed') }); const { mutate: onLeaveTeam, isLoading: isLoadingLeaveTeam } = useRequest({ mutationFn: async (teamId?: string) => { @@ -306,7 +308,7 @@ const TeamManageModal = ({ onClose }: { onClose: () => void }) => { - {item.memberUsername} + {item.memberName} {t(TeamMemberRoleMap[item.role]?.label || '')} @@ -383,7 +385,7 @@ const TeamManageModal = ({ onClose }: { onClose: () => void }) => { }), undefined, t('user.team.Remove Member Confirm Tip', { - username: item.memberUsername + username: item.memberName }) )() } diff --git a/projects/app/src/components/support/user/team/TeamMenu/index.tsx b/projects/app/src/components/support/user/team/TeamMenu/index.tsx index e49934bcc..9b70de731 100644 --- a/projects/app/src/components/support/user/team/TeamMenu/index.tsx +++ b/projects/app/src/components/support/user/team/TeamMenu/index.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { Box, Button, Flex, Image, useDisclosure, useTheme } from '@chakra-ui/react'; import { useUserStore } from '@/web/support/user/useUserStore'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import MyTooltip from '@/components/MyTooltip'; import dynamic from 'next/dynamic'; import { feConfigs } from '@/web/common/system/staticData'; diff --git a/projects/app/src/components/support/user/team/UpdateInviteModal/index.tsx b/projects/app/src/components/support/user/team/UpdateInviteModal/index.tsx index f90496081..682b1a5f9 100644 --- a/projects/app/src/components/support/user/team/UpdateInviteModal/index.tsx +++ b/projects/app/src/components/support/user/team/UpdateInviteModal/index.tsx @@ -1,5 +1,5 @@ import React, { useEffect } from 'react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import MyModal from '@/components/MyModal'; import { Button, diff --git a/projects/app/src/constants/dataset.ts b/projects/app/src/constants/dataset.ts index 2497c5b9e..aff029935 100644 --- a/projects/app/src/constants/dataset.ts +++ b/projects/app/src/constants/dataset.ts @@ -1,4 +1,7 @@ -import type { DatasetItemType } from '@/types/core/dataset'; +import type { + DatasetCollectionItemType, + DatasetItemType +} from '@fastgpt/global/core/dataset/type.d'; export const defaultDatasetDetail: DatasetItemType = { _id: '', @@ -10,7 +13,7 @@ export const defaultDatasetDetail: DatasetItemType = { type: 'dataset', avatar: '/icon/logo.svg', name: '', - tags: '', + tags: [], permission: 'private', isOwner: false, canWrite: false, @@ -22,3 +25,32 @@ export const defaultDatasetDetail: DatasetItemType = { maxToken: 3000 } }; + +export const defaultCollectionDetail: DatasetCollectionItemType = { + _id: '', + userId: '', + teamId: '', + tmbId: '', + datasetId: { + _id: '', + parentId: '', + userId: '', + teamId: '', + tmbId: '', + updateTime: new Date(), + type: 'dataset', + avatar: '/icon/logo.svg', + name: '', + tags: [], + permission: 'private', + vectorModel: 'text-embedding-ada-002' + }, + parentId: '', + name: '', + type: 'file', + updateTime: new Date(), + metadata: {}, + canWrite: false, + sourceName: '', + sourceId: '' +}; diff --git a/projects/app/src/global/common/string/tools.ts b/projects/app/src/global/common/string/tools.ts deleted file mode 100644 index 5711360af..000000000 --- a/projects/app/src/global/common/string/tools.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { getErrText } from '@fastgpt/global/common/error/utils'; -import { countPromptTokens } from '@/global/common/tiktoken'; - -/* - replace {{variable}} to value -*/ -export function replaceVariable(text: string, obj: Record) { - for (const key in obj) { - const val = obj[key]; - if (!['string', 'number'].includes(typeof val)) continue; - - text = text.replace(new RegExp(`{{(${key})}}`, 'g'), String(val)); - } - return text || ''; -} - -/** - * text split into chunks - * maxLen - one chunk len. max: 3500 - * overlapLen - The size of the before and after Text - * maxLen > overlapLen - */ -export const splitText2Chunks = ({ text = '', maxLen }: { text: string; maxLen: number }) => { - const overlapLen = Math.floor(maxLen * 0.15); // Overlap length - const tempMarker = 'SPLIT_HERE_SPLIT_HERE'; - - const stepReg: Record = { - 0: /(\n\n)/g, - 1: /([\n])/g, - 2: /[。]|(?!<[^a-zA-Z])\.\s/g, - 3: /([！？]|!\s|\?\s)/g, - 4: /([；]|;\s)/g, - 5: /([，]|,\s)/g - }; - - const splitTextRecursively = ({ text = '', step }: { text: string; step: number }) => { - if (text.length <= maxLen) { - return [text]; - } - const reg = stepReg[step]; - - if (!reg) { - // use slice-maxLen to split text - const chunks: string[] = []; - let chunk = ''; - for (let i = 0; i < text.length; i += maxLen - overlapLen) { - chunk = text.slice(i, i + maxLen); - chunks.push(chunk); - } - return chunks; - } - - // split text by delimiters - const splitTexts = text - .replace(reg, `$1${tempMarker}`) - .split(`${tempMarker}`) - .filter((part) => part); - - let chunks: string[] = []; - let preChunk = ''; - let chunk = ''; - for (let i = 0; i < splitTexts.length; i++) { - let text = splitTexts[i]; - // chunk over size - if (text.length > maxLen) { - const innerChunks = splitTextRecursively({ text, step: step + 1 }); - if (innerChunks.length === 0) continue; - // If the last chunk is too small, it is merged into the next chunk - if (innerChunks[innerChunks.length - 1].length <= maxLen * 0.5) { - text = innerChunks.pop() || ''; - chunks = chunks.concat(innerChunks); - } else { - chunks = chunks.concat(innerChunks); - continue; - } - } - - chunk += text; - // size over lapLen, push it to next chunk - if (chunk.length > maxLen - overlapLen) { - preChunk += text; - } - if (chunk.length >= maxLen) { - chunks.push(chunk); - chunk = preChunk; - preChunk = ''; - } - } - - if (chunk && !chunks[chunks.length - 1].endsWith(chunk)) { - chunks.push(chunk); - } - return chunks; - }; - - try { - const chunks = splitTextRecursively({ text, step: 0 }); - - const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0); - - return { - chunks, - tokens - }; - } catch (err) { - throw new Error(getErrText(err)); - } -}; diff --git a/projects/app/src/global/core/api/datasetReq.d.ts b/projects/app/src/global/core/api/datasetReq.d.ts index ef9427449..7a03d5f61 100644 --- a/projects/app/src/global/core/api/datasetReq.d.ts +++ b/projects/app/src/global/core/api/datasetReq.d.ts @@ -2,7 +2,7 @@ import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core import type { RequestPaging } from '@/types'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; import type { SearchTestItemType } from '@/types/core/dataset'; -import { DatasetChunkItemType, UploadChunkItemType } from '@fastgpt/global/core/dataset/type'; +import { UploadChunkItemType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; @@ -10,19 +10,11 @@ import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant' export type DatasetUpdateParams = { id: string; parentId?: string; - tags?: string; + tags?: string[]; name?: string; avatar?: string; permission?: `${PermissionTypeEnum}`; }; -export type CreateDatasetParams = { - parentId?: string; - name: string; - tags: string; - avatar: string; - vectorModel?: string; - type: `${DatasetTypeEnum}`; -}; export type SearchTestProps = { datasetId: string; @@ -54,20 +46,6 @@ export type UpdateDatasetCollectionParams = { }; /* ==== data ===== */ -export type SetOneDatasetDataProps = { - id?: string; - collectionId: string; - q?: string; // embedding content - a?: string; // bonus content -}; -export type PushDataProps = { - collectionId: string; - data: DatasetChunkItemType[]; - mode: `${TrainingModeEnum}`; - prompt?: string; - billId?: string; -}; - export type GetDatasetDataListProps = RequestPaging & { searchText?: string; collectionId: string; diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts new file mode 100644 index 000000000..dae6f72f9 --- /dev/null +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -0,0 +1,35 @@ +import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type'; + +/* ================= dataset ===================== */ +export type CreateDatasetParams = { + parentId?: string; + name: string; + tags: string; + avatar: string; + vectorModel?: string; + type: `${DatasetTypeEnum}`; +}; + +/* ================= collection ===================== */ + +/* ================= data ===================== */ +export type InsertOneDatasetDataProps = PushDatasetDataChunkProps & { + collectionId: string; +}; +export type PushDatasetDataProps = { + collectionId: string; + data: PushDatasetDataChunkProps[]; + mode: `${TrainingModeEnum}`; + prompt?: string; + billId?: string; +}; +export type UpdateDatasetDataProps = { + id: string; + q?: string; // embedding content + a?: string; // bonus content + indexes: (Omit & { + dataId?: string; // pg data id + })[]; +}; diff --git a/projects/app/src/global/core/dataset/response.d.ts b/projects/app/src/global/core/dataset/type.d.ts similarity index 72% rename from projects/app/src/global/core/dataset/response.d.ts rename to projects/app/src/global/core/dataset/type.d.ts index 0a216ec74..bb8a03073 100644 --- a/projects/app/src/global/core/dataset/response.d.ts +++ b/projects/app/src/global/core/dataset/type.d.ts @@ -1,5 +1,8 @@ import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type'; -import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; +import { + DatasetCollectionSchemaType, + DatasetDataSchemaType +} from '@fastgpt/global/core/dataset/type.d'; /* ================= dataset ===================== */ @@ -11,7 +14,7 @@ export type DatasetCollectionsListItemType = { name: string; type: DatasetCollectionSchemaType['type']; updateTime: Date; - dataAmount?: number; + dataAmount: number; trainingAmount: number; metadata: DatasetCollectionSchemaType['metadata']; canWrite: boolean; @@ -19,7 +22,10 @@ export type DatasetCollectionsListItemType = { /* ================= data ===================== */ export type DatasetDataListItemType = { - id: string; + _id: string; + datasetId: string; + collectionId: string; q: string; // embedding content a: string; // bonus content + indexes: DatasetDataSchemaType['indexes']; }; diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx index e893cd34d..b80ad7bbd 100644 --- a/projects/app/src/pages/_app.tsx +++ b/projects/app/src/pages/_app.tsx @@ -10,10 +10,10 @@ import NProgress from 'nprogress'; //nprogress module import Router from 'next/router'; import { clientInitData, feConfigs } from '@/web/common/system/staticData'; import { appWithTranslation, useTranslation } from 'next-i18next'; -import { getLangStore, setLangStore } from '@/web/common/utils/i18n'; import { useRouter } from 'next/router'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d'; +import { change2DefaultLng, setLngStore } from '@/web/common/utils/i18n'; import 'nprogress/nprogress.css'; import '@/web/styles/reset.scss'; @@ -57,6 +57,7 @@ function App({ Component, pageProps }: AppProps) { ); setScripts(scripts || []); })(); + // add window error track window.onerror = function (msg, url) { window.umami?.track('windowError', { @@ -76,23 +77,22 @@ function App({ Component, pageProps }: AppProps) { }, []); useEffect(() => { - hiId && localStorage.setItem('inviterId', hiId); - }, [hiId]); + // get default language + const targetLng = change2DefaultLng(i18n.language); + if (targetLng) { + setLngStore(targetLng); + router.replace(router.asPath, undefined, { locale: targetLng }); + } + }, []); useEffect(() => { - const lang = getLangStore() || 'zh'; - i18n?.changeLanguage?.(lang); - setLangStore(lang); - - return () => { - setLastRoute(router.asPath); - }; - }, [router.asPath]); + hiId && localStorage.setItem('inviterId', hiId); + }, [hiId]); return ( <> - {feConfigs?.systemTitle || 'AI'} + {feConfigs?.systemTitle || process.env.SYSTEM_NAME || 'GPT'} + + {process.env.SYSTEM_NAME || 'FastGPT'}
diff --git a/projects/app/src/pages/_error.tsx b/projects/app/src/pages/_error.tsx index b943e467c..1c119ad0a 100644 --- a/projects/app/src/pages/_error.tsx +++ b/projects/app/src/pages/_error.tsx @@ -25,8 +25,9 @@ function Error() { return (

- 部分系统不兼容，导致页面崩溃。如果可以，请联系作者，反馈下具体操作和页面。大部分是 苹果 的 - safari 浏览器导致，可以尝试更换 chrome 浏览器。 + 部分系统不兼容，导致页面崩溃。如果可以，请联系作者，反馈下具体操作和页面。 大部分是 苹果 的 + safari 浏览器导致，可以尝试更换 chrome + 浏览器。或者是因为开了中文翻译导致，请检查并关闭中文翻译。

); } diff --git a/projects/app/src/pages/account/components/ApiKeyTable.tsx b/projects/app/src/pages/account/components/ApiKeyTable.tsx index 4ca5bada8..2e8276279 100644 --- a/projects/app/src/pages/account/components/ApiKeyTable.tsx +++ b/projects/app/src/pages/account/components/ApiKeyTable.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import ApiKeyTable from '@/components/support/apikey/Table'; const ApiKey = () => { diff --git a/projects/app/src/pages/account/components/BillDetail.tsx b/projects/app/src/pages/account/components/BillDetail.tsx index 140060a52..d24476fd3 100644 --- a/projects/app/src/pages/account/components/BillDetail.tsx +++ b/projects/app/src/pages/account/components/BillDetail.tsx @@ -16,7 +16,7 @@ import dayjs from 'dayjs'; import { BillSourceMap } from '@fastgpt/global/support/wallet/bill/constants'; import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; import MyModal from '@/components/MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void }) => { const { t } = useTranslation(); @@ -30,7 +30,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void 用户: - {bill.username} + {bill.memberName} 订单号: diff --git a/projects/app/src/pages/account/components/BillTable.tsx b/projects/app/src/pages/account/components/BillTable.tsx index 83cf34bb0..c799d9984 100644 --- a/projects/app/src/pages/account/components/BillTable.tsx +++ b/projects/app/src/pages/account/components/BillTable.tsx @@ -55,7 +55,7 @@ const BillTable = () => { - + @@ -66,7 +66,7 @@ const BillTable = () => { {bills.map((item) => ( - + diff --git a/projects/app/src/pages/account/components/Info.tsx b/projects/app/src/pages/account/components/Info.tsx index b7f5acbd3..0cbab6af8 100644 --- a/projects/app/src/pages/account/components/Info.tsx +++ b/projects/app/src/pages/account/components/Info.tsx @@ -1,5 +1,14 @@ -import React, { useCallback, useRef, useState } from 'react'; -import { Box, Flex, Button, useDisclosure, useTheme, Divider, Select } from '@chakra-ui/react'; +import React, { useCallback, useRef } from 'react'; +import { + Box, + Flex, + Button, + useDisclosure, + useTheme, + Divider, + Select, + Input +} from '@chakra-ui/react'; import { useForm } from 'react-hook-form'; import { UserUpdateParams } from '@/types/user'; import { useToast } from '@/web/common/hooks/useToast'; @@ -16,10 +25,11 @@ import Loading from '@/components/Loading'; import Avatar from '@/components/Avatar'; import MyIcon from '@/components/Icon'; import MyTooltip from '@/components/MyTooltip'; -import { getLangStore, LangEnum, langMap, setLangStore } from '@/web/common/utils/i18n'; +import { langMap, setLngStore } from '@/web/common/utils/i18n'; import { useRouter } from 'next/router'; import MySelect from '@/components/Select'; import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; +import { putUpdateMemberName } from '@/web/support/user/team/api'; const TeamMenu = dynamic(() => import('@/components/support/user/team/TeamMenu')); const PayModal = dynamic(() => import('./PayModal'), { @@ -63,8 +73,6 @@ const UserInfo = () => { multiple: false }); - const [language, setLanguage] = useState<`${LangEnum}`>(getLangStore()); - const onclickSave = useCallback( async (data: UserType) => { await updateUserInfo({ @@ -153,6 +161,27 @@ const UserInfo = () => { ml={[0, 10]} mt={[6, 0]} > + {feConfigs.isPlus && ( + + {t('user.Member Name')}:  + { + const val = e.target.value; + if (val === userInfo?.team?.memberName) return; + try { + putUpdateMemberName(val); + } catch (error) {} + }} + /> + + )} {t('user.Account')}:  {userInfo?.username} @@ -167,17 +196,15 @@ const UserInfo = () => { {t('user.Language')}:  ({ label: lang.label, value: key }))} onchange={(val: any) => { const lang = val; - setLangStore(lang); - setLanguage(lang); - i18n?.changeLanguage?.(lang); - router.reload(); + setLngStore(lang); + router.replace(router.basePath, router.asPath, { locale: lang }); }} /> diff --git a/projects/app/src/pages/account/components/OpenAIAccountModal.tsx b/projects/app/src/pages/account/components/OpenAIAccountModal.tsx index e7df7a487..7e82962ea 100644 --- a/projects/app/src/pages/account/components/OpenAIAccountModal.tsx +++ b/projects/app/src/pages/account/components/OpenAIAccountModal.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { ModalBody, Box, Flex, Input, ModalFooter, Button } from '@chakra-ui/react'; import MyModal from '@/components/MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useForm } from 'react-hook-form'; import { useRequest } from '@/web/common/hooks/useRequest'; import type { UserType } from '@fastgpt/global/support/user/type.d'; diff --git a/projects/app/src/pages/account/components/PayModal.tsx b/projects/app/src/pages/account/components/PayModal.tsx index 3023e6e41..b254663a1 100644 --- a/projects/app/src/pages/account/components/PayModal.tsx +++ b/projects/app/src/pages/account/components/PayModal.tsx @@ -5,7 +5,7 @@ import { useToast } from '@/web/common/hooks/useToast'; import { useQuery } from '@tanstack/react-query'; import { useRouter } from 'next/router'; import { getErrText } from '@fastgpt/global/common/error/utils'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import Markdown from '@/components/Markdown'; import MyModal from '@/components/MyModal'; import { priceMd } from '@/web/common/system/staticData'; diff --git a/projects/app/src/pages/account/components/UpdatePswModal.tsx b/projects/app/src/pages/account/components/UpdatePswModal.tsx index 023ed2310..34aa89ad2 100644 --- a/projects/app/src/pages/account/components/UpdatePswModal.tsx +++ b/projects/app/src/pages/account/components/UpdatePswModal.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { ModalBody, Box, Flex, Input, ModalFooter, Button } from '@chakra-ui/react'; import MyModal from '@/components/MyModal'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useForm } from 'react-hook-form'; import { useRequest } from '@/web/common/hooks/useRequest'; import { updatePasswordByOld } from '@/web/support/user/api'; diff --git a/projects/app/src/pages/account/index.tsx b/projects/app/src/pages/account/index.tsx index d293b6be1..8a233651d 100644 --- a/projects/app/src/pages/account/index.tsx +++ b/projects/app/src/pages/account/index.tsx @@ -12,7 +12,7 @@ import Tabs from '@/components/Tabs'; import UserInfo from './components/Info'; import { serviceSideProps } from '@/web/common/utils/i18n'; import { feConfigs } from '@/web/common/system/staticData'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import Script from 'next/script'; const Promotion = dynamic(() => import('./components/Promotion')); @@ -35,67 +35,64 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => { const { t } = useTranslation(); const { userInfo, setUserInfo } = useUserStore(); - const tabList = useMemo( - () => [ - { - icon: 'meLight', - label: t('user.Personal Information'), - id: TabEnum.info - }, - ...(feConfigs?.isPlus - ? [ - { - icon: 'billRecordLight', - label: t('user.Usage Record'), - id: TabEnum.bill - } - ] - : []), - ...(feConfigs?.show_promotion - ? [ - { - icon: 'promotionLight', - label: t('user.Promotion Record'), - id: TabEnum.promotion - } - ] - : []), - ...(feConfigs?.show_pay && userInfo?.team.canWrite - ? [ - { - icon: 'payRecordLight', - label: t('user.Recharge Record'), - id: TabEnum.pay - } - ] - : []), - ...(userInfo?.team.canWrite - ? [ - { - icon: 'apikey', - label: t('user.apikey.key'), - id: TabEnum.apikey - } - ] - : []), - ...(feConfigs.isPlus - ? [ - { - icon: 'informLight', - label: t('user.Notice'), - id: TabEnum.inform - } - ] - : []), + const tabList = [ + { + icon: 'meLight', + label: t('user.Personal Information'), + id: TabEnum.info + }, + ...(feConfigs?.isPlus + ? [ + { + icon: 'billRecordLight', + label: t('user.Usage Record'), + id: TabEnum.bill + } + ] + : []), + ...(feConfigs?.show_promotion + ? [ + { + icon: 'promotionLight', + label: t('user.Promotion Record'), + id: TabEnum.promotion + } + ] + : []), + ...(feConfigs?.show_pay && userInfo?.team.canWrite + ? [ + { + icon: 'payRecordLight', + label: t('user.Recharge Record'), + id: TabEnum.pay + } + ] + : []), + ...(userInfo?.team.canWrite + ? [ + { + icon: 'apikey', + label: t('user.apikey.key'), + id: TabEnum.apikey + } + ] + : []), + ...(feConfigs.isPlus + ? [ + { + icon: 'informLight', + label: t('user.Notice'), + id: TabEnum.inform + } + ] + : []), - { - icon: 'loginoutLight', - label: t('user.Sign Out'), - id: TabEnum.loginout - } - ], - [t, userInfo?.team.canWrite] - ); + { + icon: 'loginoutLight', + label: t('user.Sign Out'), + id: TabEnum.loginout + } + ]; const { openConfirm, ConfirmModal } = useConfirm({ content: '确认退出登录？' diff --git a/projects/app/src/pages/api/admin/initv46-2.ts b/projects/app/src/pages/api/admin/initv46-2.ts new file mode 100644 index 000000000..c7c7a7264 --- /dev/null +++ b/projects/app/src/pages/api/admin/initv46-2.ts @@ -0,0 +1,169 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { delay } from '@/utils/tools'; +import { PgClient } from '@fastgpt/service/common/pg'; +import { + DatasetDataIndexTypeEnum, + PgDatasetTableName +} from '@fastgpt/global/core/dataset/constant'; + +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller'; + +let success = 0; +/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */ +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { limit = 50 } = req.body as { limit: number }; + await authCert({ req, authRoot: true }); + await connectToDatabase(); + success = 0; + + try { + await Promise.allSettled([ + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN data_id VARCHAR(50);`), + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN q DROP NOT NULL;`), // q can null + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN a DROP NOT NULL;`), // a can null + PgClient.query( + `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id TYPE VARCHAR(50) USING team_id::VARCHAR(50);` + ), // team_id varchar + PgClient.query( + `ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id TYPE VARCHAR(50) USING tmb_id::VARCHAR(50);` + ), // tmb_id varchar + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id SET NOT NULL;`), // team_id not null + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id SET NOT NULL;`), // tmb_id not null + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN dataset_id SET NOT NULL;`), // dataset_id not null + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN collection_id SET NOT NULL;`) // collection_id not null + ]); + } catch (error) {} + + await initPgData(); + + jsonRes(res, { + data: await init(limit), + message: + '初始化任务已开始，请注意日志进度。可通过 select count(id) from modeldata where data_id is null; 检查是否完全初始化，如果结果为 0 ，则完全初始化。' + }); + } catch (error) { + console.log(error); + + jsonRes(res, { + code: 500, + error + }); + } +} + +type PgItemType = { + id: string; + q: string; + a: string; + dataset_id: string; + collection_id: string; + team_id: string; + tmb_id: string; +}; + +async function initPgData() { + const limit = 10; + const { rows } = await PgClient.query<{ user_id: string }>(` + SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id='null'; +`); + console.log('init pg', rows.length); + let success = 0; + for (let i = 0; i < limit; i++) { + init(i); + } + async function init(index: number): Promise { + const userId = rows[index]?.user_id; + if (!userId) return; + try { + const tmb = await getUserDefaultTeam({ userId }); + // update pg + await PgClient.query( + `Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id='null';` + ); + console.log(++success); + init(index + limit); + } catch (error) { + if (error === 'default team not exist') { + return; + } + console.log(error); + await delay(1000); + return init(index); + } + } +} + +async function init(limit: number): Promise { + const { rows: idList } = await PgClient.query<{ id: string }>( + `SELECT id FROM ${PgDatasetTableName} WHERE data_id IS NULL` + ); + + console.log('totalCount', idList.length); + if (idList.length === 0) return; + + for (let i = 0; i < limit; i++) { + initData(i); + } + + async function initData(index: number): Promise { + const dataId = idList[index]?.id; + if (!dataId) { + console.log('done'); + return; + } + // get limit data where data_id is null + const { rows } = await PgClient.query( + `SELECT id,q,a,dataset_id,collection_id,team_id,tmb_id FROM ${PgDatasetTableName} WHERE id=${dataId};` + ); + const data = rows[0]; + if (!data) { + console.log('done'); + return; + } + + let id = ''; + try { + // create mongo data and update data_id + const { _id } = await MongoDatasetData.create({ + teamId: data.team_id.trim(), + tmbId: data.tmb_id.trim(), + datasetId: data.dataset_id, + collectionId: data.collection_id, + q: data.q, + a: data.a, + indexes: [ + { + defaultIndex: !data.a, + type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk, + dataId: data.id, + text: data.q + } + ] + }); + id = _id; + // update pg data_id + await PgClient.query( + `UPDATE ${PgDatasetTableName} SET data_id='${String(_id)}' WHERE id=${dataId};` + ); + + console.log(++success); + return initData(index + limit); + } catch (error) { + console.log(error); + console.log(data); + + try { + if (id) { + await MongoDatasetData.findByIdAndDelete(id); + } + } catch (error) {} + await delay(500); + return initData(index); + } + } +} diff --git a/projects/app/src/pages/api/admin/initv46.ts b/projects/app/src/pages/api/admin/initv46.ts index 35ce38c9c..a8a55f9f1 100644 --- a/projects/app/src/pages/api/admin/initv46.ts +++ b/projects/app/src/pages/api/admin/initv46.ts @@ -4,21 +4,14 @@ import { connectToDatabase } from '@/service/mongo'; import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema'; import { createDefaultTeam, - getTeamInfoByTmbId + getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller'; import { MongoUser } from '@fastgpt/service/support/user/schema'; import { UserModelSchema } from '@fastgpt/global/support/user/type'; import { delay } from '@/utils/tools'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { - DatasetCollectionSchemaType, - DatasetSchemaType, - DatasetTrainingSchemaType -} from '@fastgpt/global/core/dataset/type'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { connectionMongo } from '@fastgpt/service/common/mongo'; -import { Types } from 'mongoose'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { PgClient } from '@fastgpt/service/common/pg'; import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant'; @@ -30,6 +23,7 @@ import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; import { MongoPlugin } from '@fastgpt/service/core/plugin/schema'; import { POST } from '@fastgpt/service/common/api/plusRequest'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -182,7 +176,7 @@ async function initMongoTeamId(limit: number) { async function init(userId: string): Promise { try { - const tmb = await getTeamInfoByTmbId({ userId }); + const tmb = await getUserDefaultTeam({ userId }); await schema.updateMany( { @@ -225,7 +219,7 @@ async function initDatasetAndApp() { } async function initCollectionFileTeam(limit: number) { /* init user default Team */ - const DatasetFile = connectionMongo.connection.db.collection(`dataset.files`); + const DatasetFile = getGFSCollection('dataset'); const matchWhere = { $or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }] }; @@ -264,7 +258,7 @@ async function initCollectionFileTeam(limit: number) { async function init(userId: string): Promise { try { - const tmb = await getTeamInfoByTmbId({ + const tmb = await getUserDefaultTeam({ userId }); @@ -295,8 +289,8 @@ async function initPgData() { // add column try { await Promise.all([ - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id CHAR(50);`), - PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id CHAR(50);`), + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id VARCHAR(50);`), + PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id VARCHAR(50);`), PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`) ]); } catch (error) { @@ -316,7 +310,7 @@ async function initPgData() { const userId = rows[index]?.user_id; if (!userId) return; try { - const tmb = await getTeamInfoByTmbId({ userId }); + const tmb = await getUserDefaultTeam({ userId }); // update pg await PgClient.query( `Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;` diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts index 48270e04a..675003cb0 100644 --- a/projects/app/src/pages/api/common/file/upload.ts +++ b/projects/app/src/pages/api/common/file/upload.ts @@ -80,8 +80,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< try { await connectToDatabase(); const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); - console.log(req.body); - const { files, bucketName, metadata } = await upload.doUpload(req, res); const upLoadResults = await Promise.all( diff --git a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts index de561a1a8..2036f14a9 100644 --- a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts +++ b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts @@ -4,15 +4,16 @@ import { connectToDatabase } from '@/service/mongo'; import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d'; import { pushQuestionGuideBill } from '@/service/support/wallet/bill/push'; import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { messages } = req.body as CreateQuestionGuideParams; - const { tmbId, teamId } = await authCert({ + const { messages, shareId } = req.body as CreateQuestionGuideParams; + const { tmbId, teamId } = await authCertAndShareId({ req, - authToken: true + authToken: true, + shareId }); const qgModel = global.qgModels[0]; diff --git a/projects/app/src/pages/api/core/dataset/allDataset.ts b/projects/app/src/pages/api/core/dataset/allDataset.ts index 4452adda8..ab3b5ed2f 100644 --- a/projects/app/src/pages/api/core/dataset/allDataset.ts +++ b/projects/app/src/pages/api/core/dataset/allDataset.ts @@ -3,10 +3,11 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { getVectorModel } from '@/service/core/ai/model'; -import type { DatasetItemType } from '@/types/core/dataset'; +import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; +/* get all dataset by teamId or tmbId */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); @@ -20,7 +21,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const data = datasets.map((item) => ({ ...item.toJSON(), - tags: item.tags.join(' '), vectorModel: getVectorModel(item.vectorModel), canWrite: String(item.tmbId) === tmbId, isOwner: teamOwner || String(item.tmbId) === tmbId diff --git a/projects/app/src/pages/api/core/dataset/collection/delById.ts b/projects/app/src/pages/api/core/dataset/collection/delById.ts index 7c971c9ab..a3be7461d 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delById.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delById.ts @@ -30,15 +30,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const collections = await findCollectionAndChild(collectionId, '_id metadata'); const delIdList = collections.map((item) => item._id); - // delete pg data - await delDataByCollectionId({ collectionIds: delIdList }); - // delete training data await MongoDatasetTraining.deleteMany({ - datasetCollectionId: { $in: delIdList }, + collectionId: { $in: delIdList }, teamId }); + // delete pg data + await delDataByCollectionId({ collectionIds: delIdList }); + // delete file await Promise.all( collections.map((collection) => { diff --git a/projects/app/src/pages/api/core/dataset/collection/detail.ts b/projects/app/src/pages/api/core/dataset/collection/detail.ts index 4705024d9..73ba162d4 100644 --- a/projects/app/src/pages/api/core/dataset/collection/detail.ts +++ b/projects/app/src/pages/api/core/dataset/collection/detail.ts @@ -27,8 +27,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< jsonRes(res, { data: { ...collection, - datasetId: collection.datasetId._id, - canWrite + canWrite, + sourceName: collection?.name, + sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink } }); } catch (err) { diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 48184259a..9b06ea4c8 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -3,15 +3,15 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema'; import { Types } from '@fastgpt/service/common/mongo'; -import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response'; +import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d'; import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; import { PagingData } from '@/types'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; -import { countCollectionData } from '@/service/core/dataset/data/utils'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; import { startQueue } from '@/service/utils/tools'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; -import { getTeamInfoByTmbId } from '@fastgpt/service/support/user/team/controller'; +import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; +import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -30,7 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // auth dataset and get my role const { tmbId } = await authDataset({ req, authToken: true, datasetId, per: 'r' }); - const { canWrite } = await getTeamInfoByTmbId({ tmbId }); + const { canWrite } = await authUserRole({ req, authToken: true }); const match = { datasetId: new Types.ObjectId(datasetId), @@ -59,7 +59,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...item, dataAmount: 0, trainingAmount: 0, - canWrite // admin or owner can write + canWrite // admin or team owner can write })) ), total: await MongoDatasetCollection.countDocuments(match) @@ -67,51 +67,75 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); } - const collections: DatasetCollectionsListItemType[] = await MongoDatasetCollection.aggregate([ - { - $match: match - }, - { - $lookup: { - from: DatasetTrainingCollectionName, - localField: '_id', - foreignField: 'datasetCollectionId', - as: 'trainings_amount' + const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([ + MongoDatasetCollection.aggregate([ + { + $match: match + }, + { + $lookup: { + from: DatasetTrainingCollectionName, + let: { id: '$_id' }, + pipeline: [ + { + $match: { + $expr: { + $eq: ['$collectionId', '$$id'] + } + } + }, + { $project: { _id: 1 } } + ], + as: 'trainings' + } + }, + { + $lookup: { + from: DatasetDataCollectionName, + let: { id: '$_id' }, + pipeline: [ + { + $match: { + $expr: { + $eq: ['$collectionId', '$$id'] + } + } + }, + { $project: { _id: 1 } } + ], + as: 'datas' + } + }, + // 统计子集合的数量和子训练的数量 + { + $project: { + _id: 1, + parentId: 1, + tmbId: 1, + name: 1, + type: 1, + updateTime: 1, + trainingAmount: { $size: '$trainings' }, + dataAmount: { $size: '$datas' }, + metadata: 1 + } + }, + { + $sort: { updateTime: -1 } + }, + { + $skip: (pageNum - 1) * pageSize + }, + { + $limit: pageSize } - }, - // 统计子集合的数量和子训练的数量 - { - $project: { - _id: 1, - parentId: 1, - tmbId: 1, - name: 1, - type: 1, - updateTime: 1, - trainingAmount: { $size: '$trainings_amount' }, - metadata: 1 - } - }, - { - $sort: { updateTime: -1 } - }, - { - $skip: (pageNum - 1) * pageSize - }, - { - $limit: pageSize - } + ]), + MongoDatasetCollection.countDocuments(match) ]); - const counts = await countCollectionData({ - collectionIds: collections.map((item) => item._id), - datasetId - }); - const data = await Promise.all( collections.map(async (item, i) => ({ ...item, - dataAmount: item.type === DatasetCollectionTypeEnum.folder ? undefined : counts[i], canWrite: String(item.tmbId) === tmbId || canWrite })) ); @@ -126,7 +150,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< pageNum, pageSize, data, - total: await MongoDatasetCollection.countDocuments(match) + total } }); } catch (err) { diff --git a/projects/app/src/pages/api/core/dataset/collection/paths.ts b/projects/app/src/pages/api/core/dataset/collection/paths.ts index 4e4662ff4..a47111adc 100644 --- a/projects/app/src/pages/api/core/dataset/collection/paths.ts +++ b/projects/app/src/pages/api/core/dataset/collection/paths.ts @@ -1,7 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; -import type { DatasetPathItemType } from '@/types/core/dataset'; +import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d'; import { getDatasetCollectionPaths } from '@fastgpt/service/core/dataset/collection/utils'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; @@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< parentId }); - jsonRes(res, { + jsonRes(res, { data: paths }); } catch (err) { diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index e3b0a4850..86ad28d06 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d'; +import type { CreateDatasetParams } from '@/global/core/dataset/api.d'; import { createDefaultCollection } from './collection/create'; import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user'; diff --git a/projects/app/src/pages/api/core/dataset/data/delDataById.ts b/projects/app/src/pages/api/core/dataset/data/delete.ts similarity index 78% rename from projects/app/src/pages/api/core/dataset/data/delDataById.ts rename to projects/app/src/pages/api/core/dataset/data/delete.ts index a32e9ba8d..8b84afc81 100644 --- a/projects/app/src/pages/api/core/dataset/data/delDataById.ts +++ b/projects/app/src/pages/api/core/dataset/data/delete.ts @@ -1,10 +1,9 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; -import { PgClient } from '@fastgpt/service/common/pg'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; -import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant'; import { connectToDatabase } from '@/service/mongo'; import { authDatasetData } from '@/service/support/permission/auth/dataset'; +import { deleteDataByDataId } from '@/service/core/dataset/data/controller'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -20,11 +19,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // 凭证校验 await authDatasetData({ req, authToken: true, dataId, per: 'w' }); - await PgClient.delete(PgDatasetTableName, { - where: [['id', dataId]] - }); + await deleteDataByDataId(dataId); - jsonRes(res); + jsonRes(res, { + data: 'success' + }); } catch (err) { console.log(err); jsonRes(res, { diff --git a/projects/app/src/pages/api/core/dataset/data/getDataById.ts b/projects/app/src/pages/api/core/dataset/data/detail.ts similarity index 88% rename from projects/app/src/pages/api/core/dataset/data/getDataById.ts rename to projects/app/src/pages/api/core/dataset/data/detail.ts index dcdb41cf0..f734eb3c5 100644 --- a/projects/app/src/pages/api/core/dataset/data/getDataById.ts +++ b/projects/app/src/pages/api/core/dataset/data/detail.ts @@ -13,12 +13,9 @@ export type Response = { export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - let { dataId } = req.query as { + const { dataId } = req.query as { dataId: string; }; - if (!dataId) { - throw new Error('缺少参数'); - } // 凭证校验 const { datasetData } = await authDatasetData({ req, authToken: true, dataId, per: 'r' }); diff --git a/projects/app/src/pages/api/core/dataset/data/exportAll.ts b/projects/app/src/pages/api/core/dataset/data/exportAll.ts deleted file mode 100644 index f8d8ce464..000000000 --- a/projects/app/src/pages/api/core/dataset/data/exportAll.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { MongoUser } from '@fastgpt/service/support/user/schema'; -import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant'; -import { findAllChildrenIds } from '../delete'; -import QueryStream from 'pg-query-stream'; -import { PgClient } from '@fastgpt/service/common/pg'; -import { addLog } from '@fastgpt/service/common/mongo/controller'; -import { responseWriteController } from '@fastgpt/service/common/response'; -import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await connectToDatabase(); - let { datasetId } = req.query as { - datasetId: string; - }; - - if (!datasetId || !global.pgClient) { - throw new Error('缺少参数'); - } - - // 凭证校验 - const { userId } = await authDataset({ req, authToken: true, datasetId, per: 'w' }); - - const exportIds = [datasetId, ...(await findAllChildrenIds(datasetId))]; - - const limitMinutesAgo = new Date( - Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000 - ); - - // auth export times - const authTimes = await MongoUser.findOne( - { - _id: userId, - $or: [ - { 'limit.exportKbTime': { $exists: false } }, - { 'limit.exportKbTime': { $lte: limitMinutesAgo } } - ] - }, - '_id limit' - ); - - if (!authTimes) { - const minutes = `${global.feConfigs?.limit?.exportLimitMinutes || 0} 分钟`; - throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`); - } - - const { rows } = await PgClient.query( - `SELECT count(id) FROM ${PgDatasetTableName} where dataset_id IN (${exportIds - .map((id) => `'${id}'`) - .join(',')})` - ); - const total = rows?.[0]?.count || 0; - - addLog.info(`export datasets: ${userId}`, { total }); - - if (total > 100000) { - throw new Error('数据量超出 10 万，无法导出'); - } - - // connect pg - global.pgClient.connect((err, client, done) => { - if (err) { - console.error(err); - res.end('Error connecting to database'); - return; - } - if (!client) return; - - // create pg select stream - const query = new QueryStream( - `SELECT q, a FROM ${PgDatasetTableName} where dataset_id IN (${exportIds - .map((id) => `'${id}'`) - .join(',')})` - ); - const stream = client.query(query); - - res.setHeader('Content-Type', 'text/csv; charset=utf-8'); - res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; '); - - const write = responseWriteController({ - res, - readStream: stream - }); - - write('index,content'); - - // parse data every row - stream.on('data', ({ q, a }: { q: string; a: string }) => { - if (res.closed) { - return stream.destroy(); - } - q = q.replace(/"/g, '""'); - a = a.replace(/"/g, '""'); - // source = source?.replace(/"/g, '""'); - - write(`\n"${q}","${a || ''}"`); - }); - // finish - stream.on('end', async () => { - try { - // update export time - await MongoUser.findByIdAndUpdate(userId, { - 'limit.exportKbTime': new Date() - }); - } catch (error) {} - - // close response - done(); - res.end(); - }); - stream.on('error', (err) => { - done(err); - res.end('Error exporting data'); - }); - }); - } catch (err) { - res.status(500); - jsonRes(res, { - code: 500, - error: err - }); - } -} - -export const config = { - api: { - responseLimit: '100mb' - } -}; diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts index 204c9ea4f..d69fb4669 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertData.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts @@ -6,8 +6,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; -import { SetOneDatasetDataProps } from '@/global/core/api/datasetReq'; -import { countPromptTokens } from '@/global/common/tiktoken'; +import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { getVectorModel } from '@/service/core/ai/model'; import { hasSameValue } from '@/service/core/dataset/data/utils'; import { insertData2Dataset } from '@/service/core/dataset/data/controller'; @@ -15,11 +14,12 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/ import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; import { authTeamBalance } from '@/service/support/permission/auth/bill'; import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push'; +import { InsertOneDatasetDataProps } from '@/global/core/dataset/api'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { collectionId, q, a } = req.body as SetOneDatasetDataProps; + const { collectionId, q, a, indexes } = req.body as InsertOneDatasetDataProps; if (!q) { return Promise.reject('q is required'); @@ -38,8 +38,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex per: 'w' }); - await authTeamBalance(teamId); - // auth collection and get dataset const [ { @@ -68,11 +66,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex const { insertId, tokenLen } = await insertData2Dataset({ teamId, tmbId, + datasetId, + collectionId, q: formatQ, a: formatA, - collectionId, - datasetId, - model: vectorModel + model: vectorModel, + indexes }); pushGenerateVectorBill({ diff --git a/projects/app/src/pages/api/core/dataset/data/getDataList.ts b/projects/app/src/pages/api/core/dataset/data/list.ts similarity index 57% rename from projects/app/src/pages/api/core/dataset/data/getDataList.ts rename to projects/app/src/pages/api/core/dataset/data/list.ts index 50b963202..d4707b0a4 100644 --- a/projects/app/src/pages/api/core/dataset/data/getDataList.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -1,11 +1,11 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; -import { PgClient } from '@fastgpt/service/common/pg'; -import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant'; -import type { DatasetDataListItemType } from '@/global/core/dataset/response.d'; +import type { DatasetDataListItemType } from '@/global/core/dataset/type.d'; import type { GetDatasetDataListProps } from '@/global/core/api/datasetReq'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { PagingData } from '@/types'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -22,30 +22,29 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< searchText = searchText.replace(/'/g, ''); - const where: any = [ - ['collection_id', collectionId], - searchText ? `AND (q ILIKE '%${searchText}%' OR a ILIKE '%${searchText}%')` : '' - ]; + const match = { + collectionId, + ...(searchText + ? { + $or: [{ q: new RegExp(searchText, 'i') }, { a: new RegExp(searchText, 'i') }] + } + : {}) + }; - const [searchRes, total] = await Promise.all([ - PgClient.select(PgDatasetTableName, { - fields: ['id', 'q', 'a'], - where, - order: [{ field: 'id', mode: 'DESC' }], - limit: pageSize, - offset: pageSize * (pageNum - 1) - }), - PgClient.count(PgDatasetTableName, { - fields: ['id'], - where - }) + const [data, total] = await Promise.all([ + MongoDatasetData.find(match, '_id datasetId collectionId q a indexes') + .sort({ _id: -1 }) + .skip((pageNum - 1) * pageSize) + .limit(pageSize) + .lean(), + MongoDatasetData.countDocuments(match) ]); - jsonRes(res, { + jsonRes>(res, { data: { pageNum, pageSize, - data: searchRes.rows, + data, total } }); diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index 8f02e5f50..996d0c2ac 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -4,32 +4,27 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; -import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; import { startQueue } from '@/service/utils/tools'; -import { DatasetChunkItemType } from '@fastgpt/global/core/dataset/type'; -import { countPromptTokens } from '@/global/common/tiktoken'; +import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import type { PushDataResponse } from '@/global/core/api/datasetRes.d'; -import type { PushDataProps } from '@/global/core/api/datasetReq.d'; +import type { PushDatasetDataProps } from '@/global/core/dataset/api.d'; +import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; import { getVectorModel } from '@/service/core/ai/model'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; -const modeMap = { - [TrainingModeEnum.index]: true, - [TrainingModeEnum.qa]: true -}; - export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { collectionId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps; + const { collectionId, data, mode = TrainingModeEnum.chunk } = req.body as PushDatasetDataProps; if (!collectionId || !Array.isArray(data)) { throw new Error('collectionId or data is empty'); } - if (modeMap[mode] === undefined) { - throw new Error('Mode is not index or qa'); + if (!TrainingTypeMap[mode]) { + throw new Error(`Mode is not ${Object.keys(TrainingTypeMap).join(', ')}`); } if (data.length > 200) { @@ -68,8 +63,8 @@ export async function pushDataToDatasetCollection({ mode, prompt, billId -}: { teamId: string; tmbId: string } & PushDataProps): Promise { - // get vector model +}: { teamId: string; tmbId: string } & PushDatasetDataProps): Promise { + // get dataset vector model const { datasetId: { _id: datasetId, vectorModel } } = await getCollectionWithDataset(collectionId); @@ -77,7 +72,7 @@ export async function pushDataToDatasetCollection({ const vectorModelData = getVectorModel(vectorModel); const modeMap = { - [TrainingModeEnum.index]: { + [TrainingModeEnum.chunk]: { maxToken: vectorModelData.maxToken * 1.5, model: vectorModelData.model }, @@ -89,13 +84,12 @@ export async function pushDataToDatasetCollection({ // filter repeat or equal content const set = new Set(); - const filterResult: Record = { + const filterResult: Record = { success: [], overToken: [], repeat: [], error: [] }; - await Promise.all( data.map(async (item) => { if (!item.q) { @@ -128,13 +122,14 @@ export async function pushDataToDatasetCollection({ teamId, tmbId, datasetId, - datasetCollectionId: collectionId, + collectionId, billId, mode, prompt, model: modeMap[mode].model, q: item.q, - a: item.a + a: item.a, + indexes: item.indexes })) ); diff --git a/projects/app/src/pages/api/core/dataset/data/updateData.ts b/projects/app/src/pages/api/core/dataset/data/update.ts similarity index 60% rename from projects/app/src/pages/api/core/dataset/data/updateData.ts rename to projects/app/src/pages/api/core/dataset/data/update.ts index 0ddea06bd..6ca9b4314 100644 --- a/projects/app/src/pages/api/core/dataset/data/updateData.ts +++ b/projects/app/src/pages/api/core/dataset/data/update.ts @@ -2,52 +2,50 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; import { connectToDatabase } from '@/service/mongo'; -import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq.d'; import { updateData2Dataset } from '@/service/core/dataset/data/controller'; import { authDatasetData } from '@/service/support/permission/auth/dataset'; import { authTeamBalance } from '@/service/support/permission/auth/bill'; import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push'; +import { UpdateDatasetDataProps } from '@/global/core/dataset/api'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { id, collectionId, q = '', a } = req.body as SetOneDatasetDataProps; - - if (!id || !collectionId) { - throw new Error('缺少参数'); - } + const { id, q = '', a, indexes } = req.body as UpdateDatasetDataProps; // auth data permission - const { datasetData, teamId, tmbId } = await authDatasetData({ + const { + collection: { + datasetId: { vectorModel } + }, + teamId, + tmbId + } = await authDatasetData({ req, authToken: true, dataId: id, per: 'w' }); + // auth team balance await authTeamBalance(teamId); - // auth user and get kb - const dataset = await MongoDataset.findById(datasetData.datasetId, 'vectorModel'); - - if (!dataset) { - throw new Error("Can't find database"); - } - const { tokenLen } = await updateData2Dataset({ dataId: id, q, a, - model: dataset.vectorModel + indexes, + model: vectorModel }); - pushGenerateVectorBill({ - teamId, - tmbId, - tokenLen: tokenLen, - model: dataset.vectorModel - }); + if (tokenLen) { + pushGenerateVectorBill({ + teamId, + tmbId, + tokenLen: tokenLen, + model: vectorModel + }); + } jsonRes(res); } catch (err) { diff --git a/projects/app/src/pages/api/core/dataset/delete.ts b/projects/app/src/pages/api/core/dataset/delete.ts index 28f31a383..741bf2c9e 100644 --- a/projects/app/src/pages/api/core/dataset/delete.ts +++ b/projects/app/src/pages/api/core/dataset/delete.ts @@ -3,12 +3,12 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import { PgClient } from '@fastgpt/service/common/pg'; -import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant'; import { delDatasetFiles } from '@fastgpt/service/core/dataset/file/controller'; import { Types } from '@fastgpt/service/common/mongo'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { delDataByDatasetId } from '@/service/core/dataset/data/controller'; +import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -24,17 +24,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // auth owner await authDataset({ req, authToken: true, datasetId: id, per: 'owner' }); - const deletedIds = [id, ...(await findAllChildrenIds(id))]; + const deletedIds = await findDatasetIdTreeByTopDatasetId(id); - // delete training data + // delete training data(There could be a training mission) await MongoDatasetTraining.deleteMany({ - datasetId: { $in: deletedIds.map((id) => new Types.ObjectId(id)) } + datasetId: { $in: deletedIds } }); - // delete all pg data - await PgClient.delete(PgDatasetTableName, { - where: [`dataset_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`] - }); + // delete all dataset.data and pg data + await delDataByDatasetId({ datasetIds: deletedIds }); // delete related files await delDatasetFiles({ datasetId: id }); @@ -57,17 +55,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); } } - -export async function findAllChildrenIds(id: string) { - // find children - const children = await MongoDataset.find({ parentId: id }); - - let allChildrenIds = children.map((child) => String(child._id)); - - for (const child of children) { - const grandChildrenIds = await findAllChildrenIds(child._id); - allChildrenIds = allChildrenIds.concat(grandChildrenIds); - } - - return allChildrenIds; -} diff --git a/projects/app/src/pages/api/core/dataset/detail.ts b/projects/app/src/pages/api/core/dataset/detail.ts index 968d65a9e..a9a65b143 100644 --- a/projects/app/src/pages/api/core/dataset/detail.ts +++ b/projects/app/src/pages/api/core/dataset/detail.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { getVectorModel } from '@/service/core/ai/model'; -import { DatasetItemType } from '@/types/core/dataset'; +import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -27,7 +27,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< jsonRes(res, { data: { ...dataset, - tags: dataset.tags.join(' '), vectorModel: getVectorModel(dataset.vectorModel), canWrite, isOwner diff --git a/projects/app/src/pages/api/core/dataset/exportAll.ts b/projects/app/src/pages/api/core/dataset/exportAll.ts new file mode 100644 index 000000000..ebdd2ae2c --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/exportAll.ts @@ -0,0 +1,117 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { MongoUser } from '@fastgpt/service/support/user/schema'; +import { addLog } from '@fastgpt/service/common/mongo/controller'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller'; +import { Readable } from 'stream'; +import type { Cursor } from '@fastgpt/service/common/mongo'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + let { datasetId } = req.query as { + datasetId: string; + }; + + if (!datasetId || !global.pgClient) { + throw new Error('缺少参数'); + } + + // 凭证校验 + const { userId } = await authDataset({ req, authToken: true, datasetId, per: 'w' }); + + const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId); + + const limitMinutesAgo = new Date( + Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000 + ); + + // auth export times + const authTimes = await MongoUser.findOne( + { + _id: userId, + $or: [ + { 'limit.exportKbTime': { $exists: false } }, + { 'limit.exportKbTime': { $lte: limitMinutesAgo } } + ] + }, + '_id limit' + ); + + if (!authTimes) { + const minutes = `${global.feConfigs?.limit?.exportLimitMinutes || 0} 分钟`; + throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`); + } + + // auth max data + const total = await MongoDatasetData.countDocuments({ + datasetId: { $in: exportIds } + }); + + addLog.info(`export datasets: ${datasetId}`, { total }); + + if (total > 100000) { + throw new Error('数据量超出 10 万，无法导出'); + } + + res.setHeader('Content-Type', 'text/csv; charset=utf-8;'); + res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; '); + + const cursor = MongoDatasetData.find<{ + _id: string; + collectionId: { name: string }; + q: string; + a: string; + }>( + { + datasetId: { $in: exportIds } + }, + 'q a' + ).cursor(); + + function cursorToReadableStream(cursor: Cursor) { + const readable = new Readable({ + objectMode: true, + read() {} + }); + + readable.push(`\uFEFFindex,content`); + + cursor.on('data', (doc) => { + const q = doc.q.replace(/"/g, '""') || ''; + const a = doc.a.replace(/"/g, '""') || ''; + + readable.push(`\n"${q}","${a}"`); + }); + + cursor.on('end', async () => { + readable.push(null); + cursor.close(); + await MongoUser.findByIdAndUpdate(userId, { + 'limit.exportKbTime': new Date() + }); + }); + + return readable; + } + + // @ts-ignore + const stream = cursorToReadableStream(cursor); + stream.pipe(res); + } catch (err) { + res.status(500); + jsonRes(res, { + code: 500, + error: err + }); + } +} + +export const config = { + api: { + responseLimit: '100mb' + } +}; diff --git a/projects/app/src/pages/api/core/dataset/list.ts b/projects/app/src/pages/api/core/dataset/list.ts index 271f53abe..f8dd47dfe 100644 --- a/projects/app/src/pages/api/core/dataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/list.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { getVectorModel } from '@/service/core/ai/model'; -import type { DatasetItemType } from '@/types/core/dataset'; +import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; @@ -27,7 +27,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const data = await Promise.all( datasets.map(async (item) => ({ ...item.toJSON(), - tags: item.tags.join(' '), vectorModel: getVectorModel(item.vectorModel), canWrite, isOwner: teamOwner || String(item.tmbId) === tmbId diff --git a/projects/app/src/pages/api/core/dataset/paths.ts b/projects/app/src/pages/api/core/dataset/paths.ts index ec91f360f..1ebe71bd0 100644 --- a/projects/app/src/pages/api/core/dataset/paths.ts +++ b/projects/app/src/pages/api/core/dataset/paths.ts @@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; -import type { DatasetPathItemType } from '@/types/core/dataset'; +import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -19,7 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await authDataset({ req, authToken: true, datasetId: parentId, per: 'r' }); - jsonRes(res, { + jsonRes(res, { data: await getParents(parentId) }); } catch (err) { @@ -30,7 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } } -async function getParents(parentId?: string): Promise { +async function getParents(parentId?: string): Promise { if (!parentId) { return []; } diff --git a/projects/app/src/pages/api/core/dataset/searchTest.ts b/projects/app/src/pages/api/core/dataset/searchTest.ts index b55716b43..5d93f594e 100644 --- a/projects/app/src/pages/api/core/dataset/searchTest.ts +++ b/projects/app/src/pages/api/core/dataset/searchTest.ts @@ -8,7 +8,7 @@ import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { authTeamBalance } from '@/service/support/permission/auth/bill'; import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push'; import { countModelPrice } from '@/service/support/wallet/bill/utils'; -import { searchDatasetData } from '@/service/core/dataset/data/utils'; +import { searchDatasetData } from '@/service/core/dataset/data/pg'; import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools'; import { ModelTypeEnum } from '@/service/core/ai/model'; import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants'; diff --git a/projects/app/src/pages/api/core/dataset/data/getQueueLen.ts b/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts similarity index 100% rename from projects/app/src/pages/api/core/dataset/data/getQueueLen.ts rename to projects/app/src/pages/api/core/dataset/training/getQueueLen.ts diff --git a/projects/app/src/pages/api/plusApi/[...path].ts b/projects/app/src/pages/api/plusApi/[...path].ts index 455426084..868daaf06 100644 --- a/projects/app/src/pages/api/plusApi/[...path].ts +++ b/projects/app/src/pages/api/plusApi/[...path].ts @@ -27,8 +27,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) url, data, { - // @ts-ignore - headers: req.headers + headers: { + ...req.headers, + // @ts-ignore + rootkey: undefined + } }, method ); diff --git a/projects/app/src/pages/api/system/getInitData.ts b/projects/app/src/pages/api/system/getInitData.ts index 5333e2bc3..e345f6618 100644 --- a/projects/app/src/pages/api/system/getInitData.ts +++ b/projects/app/src/pages/api/system/getInitData.ts @@ -4,7 +4,7 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { readFileSync } from 'fs'; import type { InitDateResponse } from '@/global/common/api/systemRes'; import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; -import { getTikTokenEnc } from '@/global/common/tiktoken'; +import { getTikTokenEnc } from '@fastgpt/global/common/string/tiktoken'; import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent'; import { defaultChatModels, @@ -98,7 +98,9 @@ export function getInitConfig() { global.systemEnv = res.SystemParams ? { ...defaultSystemEnv, ...res.SystemParams } : defaultSystemEnv; - global.feConfigs = res.FeConfig ? { ...defaultFeConfigs, ...res.FeConfig } : defaultFeConfigs; + global.feConfigs = res.FeConfig + ? { ...defaultFeConfigs, ...res.FeConfig, isPlus: !!res.SystemParams?.pluginBaseUrl } + : defaultFeConfigs; global.chatModels = res.ChatModels || defaultChatModels; global.qaModels = res.QAModels || defaultQAModels; diff --git a/projects/app/src/pages/api/user/account/tokenLogin.ts b/projects/app/src/pages/api/user/account/tokenLogin.ts index 164bcb1de..222d9765d 100644 --- a/projects/app/src/pages/api/user/account/tokenLogin.ts +++ b/projects/app/src/pages/api/user/account/tokenLogin.ts @@ -8,10 +8,10 @@ import { getUserDetail } from '@/service/support/user/controller'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { userId, tmbId } = await authCert({ req, authToken: true }); + const { tmbId } = await authCert({ req, authToken: true }); jsonRes(res, { - data: await getUserDetail({ tmbId, userId }) + data: await getUserDetail({ tmbId }) }); } catch (err) { jsonRes(res, { diff --git a/projects/app/src/pages/api/v1/chat/completions.ts b/projects/app/src/pages/api/v1/chat/completions.ts index dc0effb2a..5ae2ba315 100644 --- a/projects/app/src/pages/api/v1/chat/completions.ts +++ b/projects/app/src/pages/api/v1/chat/completions.ts @@ -170,7 +170,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex // auth app, get history const { history } = await getChatHistory({ chatId, tmbId: user.team.tmbId }); - const isOwner = !shareId && String(user.team.tmbId) === String(app.tmbId); + const isAppOwner = !shareId && String(user.team.tmbId) === String(app.tmbId); /* format prompts */ const prompts = history.concat(gptMessage2ChatType(messages)); @@ -208,7 +208,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex teamId: user.team.teamId, tmbId: user.team.tmbId, variables, - isOwner, // owner update use time + updateUseTime: isAppOwner, // owner update use time shareId, source: (() => { if (shareId) { diff --git a/projects/app/src/pages/app/detail/components/BasicEdit/index.tsx b/projects/app/src/pages/app/detail/components/BasicEdit/index.tsx index 4f7bebf0b..f8d7a3ae5 100644 --- a/projects/app/src/pages/app/detail/components/BasicEdit/index.tsx +++ b/projects/app/src/pages/app/detail/components/BasicEdit/index.tsx @@ -41,7 +41,7 @@ import { useRouter } from 'next/router'; import { useToast } from '@/web/common/hooks/useToast'; import { AppSchema } from '@fastgpt/global/core/app/type.d'; import { delModelById } from '@/web/core/app/api'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { getGuideModule } from '@/global/core/app/modules/utils'; import dynamic from 'next/dynamic'; @@ -481,8 +481,9 @@ const Settings = ({ appId }: { appId: string }) => { gridGap={[2, 4]} > {selectDatasets.map((item) => ( - + { diff --git a/projects/app/src/pages/app/detail/components/OutLink/Share.tsx b/projects/app/src/pages/app/detail/components/OutLink/Share.tsx index 610b3bd2f..7a54a8265 100644 --- a/projects/app/src/pages/app/detail/components/OutLink/Share.tsx +++ b/projects/app/src/pages/app/detail/components/OutLink/Share.tsx @@ -38,7 +38,7 @@ import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d'; import { useRequest } from '@/web/common/hooks/useRequest'; import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; import { OutLinkTypeEnum } from '@fastgpt/global/support/outLink/constant'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { useToast } from '@/web/common/hooks/useToast'; import { feConfigs } from '@/web/common/system/staticData'; import MyTooltip from '@/components/MyTooltip'; diff --git a/projects/app/src/pages/app/detail/components/QGSwitch.tsx b/projects/app/src/pages/app/detail/components/QGSwitch.tsx index 94a433492..9f6c72d36 100644 --- a/projects/app/src/pages/app/detail/components/QGSwitch.tsx +++ b/projects/app/src/pages/app/detail/components/QGSwitch.tsx @@ -3,7 +3,7 @@ import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { Box, Flex, Switch, type SwitchProps } from '@chakra-ui/react'; import React from 'react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; const QGSwitch = (props: SwitchProps) => { const { t } = useTranslation(); diff --git a/projects/app/src/pages/app/detail/components/TTSSelect.tsx b/projects/app/src/pages/app/detail/components/TTSSelect.tsx index 35ee1f5f3..487c63132 100644 --- a/projects/app/src/pages/app/detail/components/TTSSelect.tsx +++ b/projects/app/src/pages/app/detail/components/TTSSelect.tsx @@ -3,7 +3,7 @@ import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { Box, Flex } from '@chakra-ui/react'; import React, { useCallback, useMemo } from 'react'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import MySelect from '@/components/Select'; import { TTSTypeEnum } from '@/constants/app'; import { Text2SpeechVoiceEnum, openaiTTSModel } from '@fastgpt/global/core/ai/speech/constant'; diff --git a/projects/app/src/pages/app/list/index.tsx b/projects/app/src/pages/app/list/index.tsx index b1fe2f3e3..4c3c57f90 100644 --- a/projects/app/src/pages/app/list/index.tsx +++ b/projects/app/src/pages/app/list/index.tsx @@ -94,6 +94,7 @@ const MyApps = () => { label={userInfo?.team.canWrite ? t('app.To Settings') : t('app.To Chat')} > { - +
{t('wallet.bill.bill username')}{t('user.team.Member Name')} {t('user.Time')} {t('user.Source')} {t('user.Application Name')}
{item.username}{item.memberName} {dayjs(item.time).format('YYYY/MM/DD HH:mm:ss')} {BillSourceMap[item.source]} {t(item.appName) || '-'} {collection.dataAmount ?? '-'} + {collection.type === DatasetCollectionTypeEnum.folder + ? '-' + : collection.dataAmount} + {dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')} { const BoxRef = useRef(null); @@ -127,9 +128,8 @@ const DataCard = () => { onClick={() => { if (!collection) return; setEditInputData({ - collectionId: collection._id, - sourceId: collection.metadata?.fileId || collection.metadata?.rawLink, - sourceName: collection.name + q: '', + indexes: [getDefaultIndex({ dataId: `${Date.now()}` })] }); }} > @@ -175,7 +175,7 @@ const DataCard = () => { > {datasetDataList.map((item) => ( { onClick={() => { if (!collection) return; setEditInputData({ - id: item.id, - collectionId: collection._id, + id: item._id, q: item.q, a: item.a, - sourceId: collection.metadata?.fileId || collection.metadata?.rawLink, - sourceName: collection.name + indexes: item.indexes }); }} > @@ -210,7 +208,7 @@ const DataCard = () => { - ID:{item.id} + ID:{item._id} {canWrite && ( { openConfirm(async () => { try { setIsLoading(true); - await delOneDatasetDataById(item.id); + await delOneDatasetDataById(item._id); getData(pageNum); } catch (error) { toast({ @@ -262,11 +260,11 @@ const DataCard = () => { {editInputData !== undefined && collection && ( setEditInputData(undefined)} onSuccess={() => getData(pageNum)} - canWrite={canWrite} + onDelete={() => getData(pageNum)} /> )} diff --git a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx index dcde2019a..f514fe649 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx @@ -2,7 +2,7 @@ import MyIcon from '@/components/Icon'; import { useLoading } from '@/web/common/hooks/useLoading'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useToast } from '@/web/common/hooks/useToast'; -import { splitText2Chunks } from '@/global/common/string/tools'; +import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { simpleText } from '@fastgpt/global/common/string/tools'; import { fileDownload, @@ -19,15 +19,13 @@ import { customAlphabet } from 'nanoid'; import dynamic from 'next/dynamic'; import MyTooltip from '@/components/MyTooltip'; import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d'; -import type { - DatasetChunkItemType, - DatasetCollectionSchemaType -} from '@fastgpt/global/core/dataset/type'; +import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { getFileIcon } from '@fastgpt/global/common/file/icon'; -import { countPromptTokens } from '@/global/common/tiktoken'; +import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d'; const UrlFetchModal = dynamic(() => import('./UrlFetchModal')); const CreateFileModal = dynamic(() => import('./CreateFileModal')); @@ -37,7 +35,7 @@ const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); export type FileItemType = { id: string; // fileId / raw Link filename: string; - chunks: DatasetChunkItemType[]; + chunks: PushDatasetDataChunkProps[]; text: string; // raw text icon: string; tokens: number; // total tokens @@ -152,7 +150,6 @@ const FileSelect = ({ fileId } }; - console.log(fileItem); onPushFiles([fileItem]); continue; diff --git a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx index 4513ecd70..2c7b33fb2 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx @@ -3,7 +3,7 @@ import { Box, type BoxProps, Flex, useTheme, ModalCloseButton } from '@chakra-ui import MyRadio from '@/components/Radio/index'; import dynamic from 'next/dynamic'; import ChunkImport from './Chunk'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; const QAImport = dynamic(() => import('./QA'), {}); const CsvImport = dynamic(() => import('./Csv'), {}); @@ -14,7 +14,7 @@ import { qaModelList } from '@/web/common/system/staticData'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; export enum ImportTypeEnum { - index = 'index', + chunk = 'chunk', qa = 'qa', csv = 'csv' } @@ -33,16 +33,16 @@ const ImportData = ({ const { t } = useTranslation(); const theme = useTheme(); const { datasetDetail } = useDatasetStore(); - const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.index); + const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.chunk); const typeMap = useMemo(() => { const vectorModel = datasetDetail.vectorModel; const qaModel = qaModelList[0]; const map = { - [ImportTypeEnum.index]: { + [ImportTypeEnum.chunk]: { defaultChunkLen: vectorModel?.defaultToken || 500, unitPrice: vectorModel?.price || 0.2, - mode: TrainingModeEnum.index + mode: TrainingModeEnum.chunk }, [ImportTypeEnum.qa]: { defaultChunkLen: qaModel?.maxContext * 0.5 || 8000, @@ -52,7 +52,7 @@ const ImportData = ({ [ImportTypeEnum.csv]: { defaultChunkLen: vectorModel?.defaultToken || 500, unitPrice: vectorModel?.price || 0.2, - mode: TrainingModeEnum.index + mode: TrainingModeEnum.chunk } }; return map[importType]; @@ -82,7 +82,7 @@ const ImportData = ({ icon: 'indexImport', title: '直接分段', desc: '选择文本文件，直接将其按分段进行处理', - value: ImportTypeEnum.index + value: ImportTypeEnum.chunk }, { icon: 'qaImport', @@ -110,7 +110,7 @@ const ImportData = ({ onUploadSuccess={uploadSuccess} > - {importType === ImportTypeEnum.index && } + {importType === ImportTypeEnum.chunk && } {importType === ImportTypeEnum.qa && } {importType === ImportTypeEnum.csv && } diff --git a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx index b3386eb1b..0046a25ee 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx @@ -12,7 +12,7 @@ import FileSelect, { FileItemType, Props as FileSelectProps } from './FileSelect import { useRequest } from '@/web/common/hooks/useRequest'; import { postDatasetCollection } from '@/web/core/dataset/api'; import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; -import { splitText2Chunks } from '@/global/common/string/tools'; +import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { useToast } from '@/web/common/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; @@ -22,7 +22,7 @@ import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete'; import MyIcon from '@/components/Icon'; import { chunksUpload } from '@/web/core/dataset/utils'; import { postCreateTrainingBill } from '@/web/support/wallet/bill/api'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import { ImportTypeEnum } from './ImportModal'; const filenameStyles = { @@ -39,7 +39,7 @@ type useImportStoreType = { setSuccessChunks: Dispatch>; isUnselectedFile: boolean; totalChunks: number; - onclickUpload: (e: { files: FileItemType[] }) => void; + onclickUpload: (e: { prompt?: string }) => void; onReSplitChunks: () => void; price: number; uploading: boolean; @@ -49,7 +49,7 @@ type useImportStoreType = { setReShowRePreview: Dispatch>; }; const StateContext = createContext({ - onclickUpload: function (e: { files: FileItemType[] }): void { + onclickUpload: function (e: { prompt?: string }): void { throw new Error('Function not implemented.'); }, uploading: false, @@ -125,7 +125,8 @@ const Provider = ({ /* start upload data */ const { mutate: onclickUpload, isLoading: uploading } = useRequest({ - mutationFn: async () => { + mutationFn: async (props?: { prompt?: string }) => { + const { prompt } = props || {}; let totalInsertion = 0; for await (const file of files) { const chunks = file.chunks; @@ -150,7 +151,8 @@ const Provider = ({ mode, onUploading: (insertLen) => { setSuccessChunks((state) => state + insertLen); - } + }, + prompt }); totalInsertion += insertLen; } diff --git a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx index 23c2cda8f..2d0f2b06c 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx @@ -5,7 +5,7 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons'; import { Prompt_AgentQA } from '@/global/core/prompt/agent'; -import { replaceVariable } from '@/global/common/string/tools'; +import { replaceVariable } from '@fastgpt/global/common/string/tools'; import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; @@ -81,7 +81,10 @@ const QAImport = () => { 重新生成预览 )} - diff --git a/projects/app/src/pages/dataset/detail/components/Info.tsx b/projects/app/src/pages/dataset/detail/components/Info.tsx index 5df20f8a9..de798df78 100644 --- a/projects/app/src/pages/dataset/detail/components/Info.tsx +++ b/projects/app/src/pages/dataset/detail/components/Info.tsx @@ -16,11 +16,11 @@ import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { useConfirm } from '@/web/common/hooks/useConfirm'; import { UseFormReturn } from 'react-hook-form'; import { compressImgAndUpload } from '@/web/common/file/controller'; -import type { DatasetItemType } from '@/types/core/dataset'; +import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d'; import Avatar from '@/components/Avatar'; import Tag from '@/components/Tag'; import MyTooltip from '@/components/MyTooltip'; -import { useTranslation } from 'react-i18next'; +import { useTranslation } from 'next-i18next'; import PermissionRadio from '@/components/support/permission/Radio'; export interface ComponentRef { @@ -208,13 +208,15 @@ const Info = ( placeholder={'标签,使用空格分割。'} maxLength={30} onChange={(e) => { - setValue('tags', e.target.value); + setValue( + 'tags', + e.target.value.split(' ').filter((item) => item) + ); setRefresh(!refresh); }} /> {getValues('tags') - .split(' ') .filter((item) => item) .map((item, i) => ( diff --git a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx index a01b56a6d..5dbb6bed1 100644 --- a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx @@ -1,10 +1,11 @@ -import React, { useMemo } from 'react'; -import { Box, Flex, Button, Textarea, IconButton, BoxProps, Image, Link } from '@chakra-ui/react'; -import { useForm } from 'react-hook-form'; +import React, { useMemo, useState } from 'react'; +import { Box, Flex, Button, Textarea, BoxProps, Image, useTheme, Grid } from '@chakra-ui/react'; +import { useFieldArray, useForm } from 'react-hook-form'; import { - postData2Dataset, + postInsertData2Dataset, putDatasetDataById, - delOneDatasetDataById + delOneDatasetDataById, + getDatasetCollectionById } from '@/web/core/dataset/api'; import { useToast } from '@/web/common/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; @@ -13,62 +14,106 @@ import MyModal from '@/components/MyModal'; import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { useQuery } from '@tanstack/react-query'; -import { useTranslation } from 'react-i18next'; -import { useDatasetStore } from '@/web/core/dataset/store/dataset'; +import { useTranslation } from 'next-i18next'; import { getFileAndOpen } from '@/web/core/dataset/utils'; -import { strIsLink } from '@fastgpt/global/common/string/tools'; import { useSystemStore } from '@/web/common/system/useSystemStore'; -import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq'; import { useRequest } from '@/web/common/hooks/useRequest'; -import { countPromptTokens } from '@/global/common/tiktoken'; +import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { useConfirm } from '@/web/common/hooks/useConfirm'; -import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; -import { feConfigs } from '@/web/common/system/staticData'; +import { getDefaultIndex, getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; +import { feConfigs, vectorModelList } from '@/web/common/system/staticData'; +import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type'; +import SideTabs from '@/components/SideTabs'; +import { useLoading } from '@/web/common/hooks/useLoading'; +import DeleteIcon from '@/components/Icon/delete'; +import { defaultCollectionDetail } from '@/constants/dataset'; -export type RawSourceType = { +export type RawSourceTextProps = BoxProps & { sourceName?: string; sourceId?: string; - addr?: boolean; + canView?: boolean; }; -export type RawSourceTextProps = BoxProps & RawSourceType; -export type InputDataType = SetOneDatasetDataProps & RawSourceType; +export type InputDataType = { + id?: string; + q: string; + a?: string; + indexes: (Omit & { + dataId?: string; // pg data id + })[]; +}; + +enum TabEnum { + content = 'content', + index = 'index', + delete = 'delete', + doc = 'doc' +} const InputDataModal = ({ + collectionId, + defaultValue, onClose, onSuccess, - onDelete, - datasetId, - defaultValues = { - collectionId: '', - sourceId: '', - sourceName: '' - }, - canWrite + onDelete }: { + collectionId: string; + defaultValue: InputDataType; onClose: () => void; - onSuccess: (data: SetOneDatasetDataProps) => void; + onSuccess: (data: InputDataType) => void; onDelete?: () => void; - datasetId: string; - defaultValues: InputDataType; - canWrite: boolean; }) => { const { t } = useTranslation(); + const theme = useTheme(); const { toast } = useToast(); - const { datasetDetail, loadDatasetDetail } = useDatasetStore(); + const { Loading } = useLoading(); + const [currentTab, setCurrentTab] = useState(TabEnum.content); - const { register, handleSubmit, reset } = useForm({ - defaultValues + const { register, handleSubmit, reset, control } = useForm({ + defaultValues: defaultValue }); + const { + fields: indexes, + append: appendIndexes, + remove: removeIndexes + } = useFieldArray({ + control, + name: 'indexes' + }); + + const tabList = [ + { label: t('dataset.data.edit.Content'), id: TabEnum.content, icon: 'overviewLight' }, + { + label: t('dataset.data.edit.Index', { amount: indexes.length }), + id: TabEnum.index, + icon: 'kbTest' + }, + ...(defaultValue.id + ? [{ label: t('dataset.data.edit.Delete'), id: TabEnum.delete, icon: 'delete' }] + : []), + { label: t('dataset.data.edit.Course'), id: TabEnum.doc, icon: 'courseLight' } + ]; const { ConfirmModal, openConfirm } = useConfirm({ content: t('dataset.data.Delete Tip') }); - const maxToken = datasetDetail.vectorModel?.maxToken || 2000; + const { data: collection = defaultCollectionDetail } = useQuery( + ['loadCollectionId', collectionId], + () => { + return getDatasetCollectionById(collectionId); + } + ); - /** - * 确认导入新数据 - */ + const maxToken = useMemo(() => { + const vectorModel = + vectorModelList.find((item) => item.model === collection.datasetId.vectorModel) || + vectorModelList[0]; + + return vectorModel?.maxToken || 3000; + }, [collection.datasetId.vectorModel]); + + // import new data const { mutate: sureImportData, isLoading: isImporting } = useRequest({ mutationFn: async (e: InputDataType) => { if (!e.q) { @@ -85,10 +130,16 @@ const InputDataModal = ({ } const data = { ...e }; - delete data.sourceName; - delete data.sourceId; - data.id = await postData2Dataset(data); + data.id = await postInsertData2Dataset({ + collectionId: collection._id, + q: e.q, + a: e.a, + // remove dataId + indexes: e.indexes.map((index) => + index.defaultIndex ? getDefaultIndex({ q: e.q, a: e.a }) : index + ) + }); return data; }, @@ -97,26 +148,26 @@ const InputDataModal = ({ reset({ ...e, q: '', - a: '' + a: '', + indexes: [getDefaultIndex({ q: e.q, a: e.a, dataId: `${Date.now()}` })] }); onSuccess(e); }, errorToast: t('common.error.unKnow') }); - + // update const { mutate: onUpdateData, isLoading: isUpdating } = useRequest({ - mutationFn: async (e: SetOneDatasetDataProps) => { + mutationFn: async (e: InputDataType) => { if (!e.id) return e; // not exactly same - if (e.q !== defaultValues.q || e.a !== defaultValues.a) { - await putDatasetDataById({ - ...e, - q: e.q === defaultValues.q ? '' : e.q - }); - return e; - } + await putDatasetDataById({ + id: e.id, + q: e.q, + a: e.a, + indexes: e.indexes + }); return e; }, @@ -127,153 +178,207 @@ const InputDataModal = ({ onClose(); } }); + // delete + const { mutate: onDeleteData, isLoading: isDeleting } = useRequest({ + mutationFn: () => { + if (!onDelete || !defaultValue.id) return Promise.resolve(null); + return delOneDatasetDataById(defaultValue.id); + }, + onSuccess() { + if (!onDelete) return; + onDelete(); + onClose(); + }, + successToast: t('common.Delete Success'), + errorToast: t('common.error.unKnow') + }); const loading = useMemo(() => isImporting || isUpdating, [isImporting, isUpdating]); - useQuery(['loadDatasetDetail'], () => { - if (datasetDetail._id === datasetId) return null; - return loadDatasetDetail(datasetId); - }); - return ( - - - {defaultValues.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')} - - - 结构详解 - - - } - w={'90vw'} - maxW={'90vw'} - h={'90vh'} - > - - - - - {'被搜索的内容'} - - - - -