Optimize the structure and naming of projects (#335)

client/src/api/core/dataset/data.d.ts

@@ -0,0 +1,26 @@
+import { KbTypeEnum } from '@/constants/dataset';
+import type { RequestPaging } from '@/types';
+import { TrainingModeEnum } from '@/constants/plugin';
+
+export type PushDataProps = {
+  kbId: string;
+  data: DatasetItemType[];
+  mode: `${TrainingModeEnum}`;
+  prompt?: string;
+};
+export type PushDataResponse = {
+  insertLen: number;
+};
+
+export type UpdateDataPrams = {
+  dataId: string;
+  kbId: string;
+  a?: string;
+  q?: string;
+};
+
+export type GetDatasetDataListProps = RequestPaging & {
+  kbId: string;
+  searchText: string;
+  fileId: string;
+};

client/src/api/core/dataset/data.ts

@@ -0,0 +1,72 @@
+import { GET, POST, PUT, DELETE } from '@/api/request';
+import type { DatasetDataItemType } from '@/types/core/dataset/data';
+import type {
+  PushDataProps,
+  PushDataResponse,
+  UpdateDataPrams,
+  GetDatasetDataListProps
+} from './data.d';
+import { QuoteItemType } from '@/types/chat';
+import { getToken } from '@/utils/user';
+import download from 'downloadjs';
+
+/* kb data */
+export const getDatasetDataList = (data: GetDatasetDataListProps) =>
+  POST(`/core/dataset/data/getDataList`, data);
+
+/**
+ * export and download data
+ */
+export const exportDatasetData = (data: { kbId: string }) =>
+  fetch(`/api/core/dataset/data/exportAll?kbId=${data.kbId}`, {
+    method: 'GET',
+    headers: {
+      token: getToken()
+    }
+  })
+    .then(async (res) => {
+      if (!res.ok) {
+        const data = await res.json();
+        throw new Error(data?.message || 'Export failed');
+      }
+      return res.blob();
+    })
+    .then((blob) => download(blob, 'dataset.csv', 'text/csv'));
+
+/**
+ * 获取模型正在拆分数据的数量
+ */
+export const getTrainingData = (data: { kbId: string; init: boolean }) =>
+  POST<{
+    qaListLen: number;
+    vectorListLen: number;
+  }>(`/core/dataset/data/getTrainingData`, data);
+
+/* get length of system training queue */
+export const getTrainingQueueLen = () => GET<number>(`/core/dataset/data/getQueueLen`);
+
+export const getDatasetDataItemById = (dataId: string) =>
+  GET<QuoteItemType>(`/core/dataset/data/getDataById`, { dataId });
+
+/**
+ * push data to training queue
+ */
+export const postChunks2Dataset = (data: PushDataProps) =>
+  POST<PushDataResponse>(`/core/dataset/data/pushData`, data);
+
+/**
+ * insert one data to dataset (immediately insert)
+ */
+export const postData2Dataset = (data: { kbId: string; data: DatasetDataItemType }) =>
+  POST<string>(`/core/dataset/data/insertData`, data);
+
+/**
+ * 更新一条数据
+ */
+export const putDatasetDataById = (data: UpdateDataPrams) =>
+  PUT('/core/dataset/data/updateData', data);
+/**
+ * 删除一条知识库数据
+ */
+export const delOneDatasetDataById = (dataId: string) =>
+  DELETE(`/core/dataset/data/delDataById?dataId=${dataId}`);

client/src/api/core/dataset/file.ts

@@ -1,14 +1,15 @@
 import { GET, POST, PUT, DELETE } from '@/api/request';
-import type { FileInfo, KbFileItemType } from '@/types/plugin';
+import type { DatasetFileItemType } from '@/types/core/dataset/file';
+import type { GSFileInfoType } from '@/types/common/file';
 
 import type { GetFileListProps, UpdateFileProps } from './file.d';
 
 export const getDatasetFiles = (data: GetFileListProps) =>
-  POST<KbFileItemType[]>(`/core/dataset/file/list`, data);
+  POST<DatasetFileItemType[]>(`/core/dataset/file/list`, data);
 export const delDatasetFileById = (params: { fileId: string; kbId: string }) =>
   DELETE(`/core/dataset/file/delById`, params);
 export const getFileInfoById = (fileId: string) =>
-  GET<FileInfo>(`/core/dataset/file/detail`, { fileId });
+  GET<GSFileInfoType>(`/core/dataset/file/detail`, { fileId });
 export const delDatasetEmptyFiles = (kbId: string) =>
   DELETE(`/core/dataset/file/delEmptyFiles`, { kbId });
 

client/src/api/core/dataset/index.d.ts

@@ -0,0 +1,34 @@
+import { KbTypeEnum } from '@/constants/dataset';
+import type { RequestPaging } from '@/types';
+import { TrainingModeEnum } from '@/constants/plugin';
+import type { SearchTestItemType } from '@/types/core/dataset';
+
+export type DatasetUpdateParams = {
+  id: string;
+  parentId?: string;
+  tags?: string;
+  name?: string;
+  avatar?: string;
+};
+export type CreateDatasetParams = {
+  parentId?: string;
+  name: string;
+  tags: string[];
+  avatar: string;
+  vectorModel?: string;
+  type: `${KbTypeEnum}`;
+};
+
+export type DatasetUpdateParams = {
+  id: string;
+  parentId?: string;
+  tags?: string;
+  name?: string;
+  avatar?: string;
+};
+
+export type SearchTestProps = {
+  kbId: string;
+  text: string;
+};
+export type SearchTestResponseType = SearchTestItemType['results'];

client/src/api/core/dataset/index.ts

@@ -0,0 +1,32 @@
+import { GET, POST, PUT, DELETE } from '@/api/request';
+import type { DatasetItemType, DatasetsItemType, DatasetPathItemType } from '@/types/core/dataset';
+import type {
+  DatasetUpdateParams,
+  CreateDatasetParams,
+  SearchTestProps,
+  SearchTestResponseType
+} from './index.d';
+import { KbTypeEnum } from '@/constants/dataset';
+
+export const getDatasets = (data: { parentId?: string; type?: `${KbTypeEnum}` }) =>
+  GET<DatasetsItemType[]>(`/core/dataset/list`, data);
+
+/**
+ * get type=dataset list
+ */
+export const getAllDataset = () => GET<DatasetsItemType[]>(`/core/dataset/allDataset`);
+
+export const getDatasetPaths = (parentId?: string) =>
+  GET<DatasetPathItemType[]>('/core/dataset/paths', { parentId });
+
+export const getDatasetById = (id: string) => GET<DatasetItemType>(`/core/dataset/detail?id=${id}`);
+
+export const postCreateDataset = (data: CreateDatasetParams) =>
+  POST<string>(`/core/dataset/create`, data);
+
+export const putDatasetById = (data: DatasetUpdateParams) => PUT(`/core/dataset/update`, data);
+
+export const delDatasetById = (id: string) => DELETE(`/core/dataset/delete?id=${id}`);
+
+export const postSearchText = (data: SearchTestProps) =>
+  POST<SearchTestResponseType>(`/core/dataset/searchTest`, data);

client/src/api/plugins/kb.ts

@@ -1,106 +0,0 @@
-import { GET, POST, PUT, DELETE } from '../request';
-import type { DatasetItemType, KbItemType, KbListItemType, KbPathItemType } from '@/types/plugin';
-import { TrainingModeEnum } from '@/constants/plugin';
-import {
-  Props as PushDataProps,
-  Response as PushDateResponse
-} from '@/pages/api/openapi/kb/pushData';
-import {
-  Props as SearchTestProps,
-  Response as SearchTestResponse
-} from '@/pages/api/openapi/kb/searchTest';
-import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
-import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb';
-import { QuoteItemType } from '@/types/chat';
-import { KbTypeEnum } from '@/constants/kb';
-import { getToken } from '@/utils/user';
-import download from 'downloadjs';
-
-/* knowledge base */
-export const getKbList = (data: { parentId?: string; type?: `${KbTypeEnum}` }) =>
-  GET<KbListItemType[]>(`/plugins/kb/list`, data);
-export const getAllDataset = () => GET<KbListItemType[]>(`/plugins/kb/allDataset`);
-
-export const getKbPaths = (parentId?: string) =>
-  GET<KbPathItemType[]>('/plugins/kb/paths', { parentId });
-
-export const getKbById = (id: string) => GET<KbItemType>(`/plugins/kb/detail?id=${id}`);
-
-export const postCreateKb = (data: CreateKbParams) => POST<string>(`/plugins/kb/create`, data);
-
-export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, data);
-
-export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`);
-
-/* kb data */
-export const getKbDataList = (data: GetKbDataListProps) =>
-  POST(`/plugins/kb/data/getDataList`, data);
-
-/**
- * export and download data
- */
-export const exportDataset = (data: { kbId: string }) =>
-  fetch(`/api/plugins/kb/data/exportAll?kbId=${data.kbId}`, {
-    method: 'GET',
-    headers: {
-      token: getToken()
-    }
-  })
-    .then(async (res) => {
-      if (!res.ok) {
-        const data = await res.json();
-        throw new Error(data?.message || 'Export failed');
-      }
-      return res.blob();
-    })
-    .then((blob) => download(blob, 'dataset.csv', 'text/csv'));
-
-/**
- * 获取模型正在拆分数据的数量
- */
-export const getTrainingData = (data: { kbId: string; init: boolean }) =>
-  POST<{
-    qaListLen: number;
-    vectorListLen: number;
-  }>(`/plugins/kb/data/getTrainingData`, data);
-
-/* get length of system training queue */
-export const getTrainingQueueLen = () => GET<number>(`/plugins/kb/data/getQueueLen`);
-
-export const getKbDataItemById = (dataId: string) =>
-  GET<QuoteItemType>(`/plugins/kb/data/getDataById`, { dataId });
-
-/**
- * 直接push数据
- */
-export const postKbDataFromList = (data: PushDataProps) =>
-  POST<PushDateResponse>(`/openapi/kb/pushData`, data);
-
-/**
- * insert one data to dataset
- */
-export const insertData2Kb = (data: { kbId: string; data: DatasetItemType }) =>
-  POST<string>(`/plugins/kb/data/insertData`, data);
-
-/**
- * 更新一条数据
- */
-export const putKbDataById = (data: UpdateDataProps) => PUT('/openapi/kb/updateData', data);
-/**
- * 删除一条知识库数据
- */
-export const delOneKbDataByDataId = (dataId: string) =>
-  DELETE(`/openapi/kb/delDataById?dataId=${dataId}`);
-
-/**
- * 拆分数据
- */
-export const postSplitData = (data: {
-  kbId: string;
-  chunks: string[];
-  prompt: string;
-  mode: `${TrainingModeEnum}`;
-}) => POST(`/openapi/text/pushData`, data);
-
-export const searchText = (data: SearchTestProps) =>
-  POST<SearchTestResponse>(`/openapi/kb/searchTest`, data);

client/src/api/request/kb.d.ts

@@ -1,24 +0,0 @@
-import { KbTypeEnum } from '@/constants/kb';
-import type { RequestPaging } from '@/types';
-
-export type KbUpdateParams = {
-  id: string;
-  parentId?: string;
-  tags?: string;
-  name?: string;
-  avatar?: string;
-};
-export type CreateKbParams = {
-  parentId?: string;
-  name: string;
-  tags: string[];
-  avatar: string;
-  vectorModel?: string;
-  type: `${KbTypeEnum}`;
-};
-
-export type GetKbDataListProps = RequestPaging & {
-  kbId: string;
-  searchText: string;
-  fileId: string;
-};

client/src/components/APIKeyModal/index.tsx

@@ -19,7 +19,8 @@ import { useQuery, useMutation } from '@tanstack/react-query';
 import { useLoading } from '@/hooks/useLoading';
 import dayjs from 'dayjs';
 import { AddIcon, DeleteIcon } from '@chakra-ui/icons';
-import { getErrText, useCopyData } from '@/utils/tools';
+import { getErrText } from '@/utils/tools';
+import { useCopyData } from '@/hooks/useCopyData';
 import { useToast } from '@/hooks/useToast';
 import MyIcon from '../Icon';
 import MyModal from '../MyModal';

client/src/components/ChatBox/QuoteModal.tsx

@@ -1,6 +1,6 @@
 import React, { useCallback, useMemo, useState } from 'react';
 import { ModalBody, Box, useTheme } from '@chakra-ui/react';
-import { getKbDataItemById } from '@/api/plugins/kb';
+import { getDatasetDataItemById } from '@/api/core/dataset/data';
 import { useLoading } from '@/hooks/useLoading';
 import { useToast } from '@/hooks/useToast';
 import { getErrText } from '@/utils/tools';
@@ -8,10 +8,10 @@ import { QuoteItemType } from '@/types/chat';
 import MyIcon from '@/components/Icon';
 import InputDataModal, { RawFileText } from '@/pages/kb/detail/components/InputDataModal';
 import MyModal from '../MyModal';
-import { KbDataItemType } from '@/types/plugin';
+import type { PgDataItemType } from '@/types/core/dataset/data';
 import { useRouter } from 'next/router';
 
-type SearchType = KbDataItemType & {
+type SearchType = PgDataItemType & {
   kb_id?: string;
 };
 
@@ -40,7 +40,7 @@ const QuoteModal = ({
       if (!item.id) return;
       try {
         setIsLoading(true);
-        const data = await getKbDataItemById(item.id);
+        const data = await getDatasetDataItemById(item.id);
 
         if (!data) {
           onUpdateQuote(item.id, '已删除');

client/src/components/ChatBox/SelectDataset.tsx

@@ -4,7 +4,7 @@ import { useTranslation } from 'next-i18next';
 import { useToast } from '@/hooks/useToast';
 import Avatar from '../Avatar';
 import MyIcon from '@/components/Icon';
-import { KbTypeEnum } from '@/constants/kb';
+import { KbTypeEnum } from '@/constants/dataset';
 import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
 
 const SelectDataset = ({

client/src/components/ChatBox/index.tsx

@@ -16,13 +16,9 @@ import {
   ExportChatType
 } from '@/types/chat';
 import { useToast } from '@/hooks/useToast';
-import {
-  useCopyData,
-  voiceBroadcast,
-  cancelBroadcast,
-  hasVoiceApi,
-  getErrText
-} from '@/utils/tools';
+import { voiceBroadcast, cancelBroadcast, hasVoiceApi } from '@/utils/web/voice';
+import { getErrText } from '@/utils/tools';
+import { useCopyData } from '@/hooks/useCopyData';
 import { Box, Card, Flex, Input, Textarea, Button, useTheme, BoxProps } from '@chakra-ui/react';
 import { feConfigs } from '@/store/static';
 import { event } from '@/utils/plugin/eventbus';
@@ -32,7 +28,7 @@ import { VariableItemType } from '@/types/app';
 import { VariableInputEnum } from '@/constants/app';
 import { useForm } from 'react-hook-form';
 import { MessageItemType } from '@/pages/api/openapi/v1/chat/completions';
-import { fileDownload } from '@/utils/file';
+import { fileDownload } from '@/utils/web/file';
 import { htmlTemplate } from '@/constants/common';
 import { useRouter } from 'next/router';
 import { useGlobalStore } from '@/store/global';

client/src/components/Language/index.tsx

@@ -1,6 +1,6 @@
 import React, { useState } from 'react';
 import { Menu, MenuButton, MenuItem, MenuList, MenuButtonProps } from '@chakra-ui/react';
-import { getLangStore, LangEnum, setLangStore, langMap } from '@/utils/i18n';
+import { getLangStore, LangEnum, setLangStore, langMap } from '@/utils/web/i18n';
 import MyIcon from '@/components/Icon';
 import { useTranslation } from 'react-i18next';
 import { useRouter } from 'next/router';

client/src/components/Markdown/CodeLight.tsx

@@ -2,7 +2,7 @@ import React from 'react';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { Box, Flex, useColorModeValue } from '@chakra-ui/react';
 import Icon from '@/components/Icon';
-import { useCopyData } from '@/utils/tools';
+import { useCopyData } from '@/hooks/useCopyData';
 
 const codeLight: { [key: string]: React.CSSProperties } = {
   'code[class*=language-]': {

client/src/components/Markdown/chat/Quote.tsx

@@ -1,6 +1,6 @@
 import React, { useMemo } from 'react';
 import { Box, useTheme } from '@chakra-ui/react';
-import { getFileAndOpen } from '@/utils/common/file';
+import { getFileAndOpen } from '@/utils/web/file';
 import { useToast } from '@/hooks/useToast';
 import { getErrText } from '@/utils/tools';
 

client/src/components/core/dataset/SelectModal.tsx

@@ -1,4 +1,4 @@
-import { getKbList, getKbPaths } from '@/api/plugins/kb';
+import { getDatasets, getDatasetPaths } from '@/api/core/dataset';
 import MyModal from '@/components/MyModal';
 import { useQuery } from '@tanstack/react-query';
 import React, { Dispatch, useMemo, useState } from 'react';
@@ -12,7 +12,7 @@ type PathItemType = {
   parentName: string;
 };
 
-const DatasetSelectModal = ({
+const DatasetSelectContainer = ({
   isOpen,
   parentId,
   setParentId,
@@ -97,7 +97,7 @@ export const useDatasetSelect = () => {
   const [parentId, setParentId] = useState<string>();
 
   const { data } = useQuery(['loadDatasetData', parentId], () =>
-    Promise.all([getKbList({ parentId }), getKbPaths(parentId)])
+    Promise.all([getDatasets({ parentId }), getDatasetPaths(parentId)])
   );
 
   const paths = useMemo(
@@ -119,4 +119,4 @@ export const useDatasetSelect = () => {
   };
 };
 
-export default DatasetSelectModal;
+export default DatasetSelectContainer;

client/src/constants/dataset.ts

@@ -1,6 +1,6 @@
-import type { KbItemType } from '@/types/plugin';
+import type { DatasetItemType } from '@/types/core/dataset';
 
-export const defaultKbDetail: KbItemType = {
+export const defaultKbDetail: DatasetItemType = {
   _id: '',
   userId: '',
   avatar: '/icon/logo.svg',

client/src/hooks/useCopyData.tsx

@@ -0,0 +1,39 @@
+import { useTranslation } from 'react-i18next';
+import { useToast } from './useToast';
+
+/**
+ * copy text data
+ */
+export const useCopyData = () => {
+  const { t } = useTranslation();
+  const { toast } = useToast();
+
+  return {
+    copyData: async (
+      data: string,
+      title: string | null = t('common.Copy Successful'),
+      duration = 1000
+    ) => {
+      try {
+        if (navigator.clipboard) {
+          await navigator.clipboard.writeText(data);
+        } else {
+          throw new Error('');
+        }
+      } catch (error) {
+        const textarea = document.createElement('textarea');
+        textarea.value = data;
+        document.body.appendChild(textarea);
+        textarea.select();
+        document.execCommand('copy');
+        document.body.removeChild(textarea);
+      }
+
+      toast({
+        title,
+        status: 'success',
+        duration
+      });
+    }
+  };
+};

client/src/pages/_app.tsx

@@ -10,7 +10,7 @@ import NProgress from 'nprogress'; //nprogress module
 import Router from 'next/router';
 import { clientInitData, feConfigs } from '@/store/static';
 import { appWithTranslation, useTranslation } from 'next-i18next';
-import { getLangStore, setLangStore } from '@/utils/i18n';
+import { getLangStore, setLangStore } from '@/utils/web/i18n';
 import { useRouter } from 'next/router';
 import { useGlobalStore } from '@/store/global';
 
@@ -85,7 +85,10 @@ function App({ Component, pageProps }: AppProps) {
     <>
       <Head>
         <title>{feConfigs?.systemTitle || 'AI'}</title>
-        <meta name="description" content="Embedding + LLM, Build AI knowledge base" />
+        <meta
+          name="description"
+          content="FastGPT is a knowledge-based question answering system built on the LLM. It offers out-of-the-box data processing and model invocation capabilities. Moreover, it allows for workflow orchestration through Flow visualization, thereby enabling complex question and answer scenarios!"
+        />
         <meta
           name="viewport"
           content="width=device-width,initial-scale=1.0,maximum-scale=1.0,minimum-scale=1.0,user-scalable=no, viewport-fit=cover"

client/src/pages/_error.tsx

@@ -1,6 +1,6 @@
 import { useEffect } from 'react';
 import { useRouter } from 'next/router';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { useGlobalStore } from '@/store/global';
 import { addLog } from '@/service/utils/tools';
 import { getErrText } from '@/utils/tools';

client/src/pages/account/components/Info.tsx

@@ -20,7 +20,7 @@ import { UserType } from '@/types/user';
 import { useQuery } from '@tanstack/react-query';
 import dynamic from 'next/dynamic';
 import { useSelectFile } from '@/hooks/useSelectFile';
-import { compressImg } from '@/utils/file';
+import { compressImg } from '@/utils/web/file';
 import { feConfigs, systemVersion } from '@/store/static';
 import { useTranslation } from 'next-i18next';
 import { timezoneList } from '@/utils/user';
@@ -28,7 +28,7 @@ import Loading from '@/components/Loading';
 import Avatar from '@/components/Avatar';
 import MyIcon from '@/components/Icon';
 import MyTooltip from '@/components/MyTooltip';
-import { getLangStore, LangEnum, langMap, setLangStore } from '@/utils/i18n';
+import { getLangStore, LangEnum, langMap, setLangStore } from '@/utils/web/i18n';
 import { useRouter } from 'next/router';
 import MyMenu from '@/components/MyMenu';
 import MySelect from '@/components/Select';

client/src/pages/account/components/Promotion.tsx

@@ -22,7 +22,7 @@ import { useLoading } from '@/hooks/useLoading';
 
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { useCopyData } from '@/utils/tools';
+import { useCopyData } from '@/hooks/useCopyData';
 import { usePagination } from '@/hooks/usePagination';
 import { PromotionRecordType } from '@/api/response/user';
 import MyIcon from '@/components/Icon';

client/src/pages/account/index.tsx

@@ -10,7 +10,7 @@ import PageContainer from '@/components/PageContainer';
 import SideTabs from '@/components/SideTabs';
 import Tabs from '@/components/Tabs';
 import UserInfo from './components/Info';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { feConfigs } from '@/store/static';
 import { useTranslation } from 'react-i18next';
 import Script from 'next/script';

client/src/pages/api/admin/initv44.ts

@@ -3,7 +3,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
 import { connectToDatabase, KB } from '@/service/mongo';
-import { KbTypeEnum } from '@/constants/kb';
+import { KbTypeEnum } from '@/constants/dataset';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 

client/src/pages/api/core/dataset/allDataset.ts

@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, KB } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { getVectorModel } from '@/service/utils/data';
-import { KbListItemType } from '@/types/plugin';
+import type { DatasetsItemType } from '@/types/core/dataset';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       vectorModel: getVectorModel(item.vectorModel)
     }));
 
-    jsonRes<KbListItemType[]>(res, {
+    jsonRes<DatasetsItemType[]>(res, {
       data
     });
   } catch (err) {

client/src/pages/api/core/dataset/create.ts

@@ -2,11 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, KB } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
-import type { CreateKbParams } from '@/api/request/kb';
+import type { CreateDatasetParams } from '@/api/core/dataset/index.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateKbParams;
+    const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateDatasetParams;
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });

client/src/pages/api/core/dataset/data/getDataById.ts

@@ -3,8 +3,8 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
-import type { KbDataItemType } from '@/types/plugin';
 import { PgDatasetTableName } from '@/constants/plugin';
+import type { PgDataItemType } from '@/types/core/dataset/data';
 
 export type Response = {
   id: string;
@@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const where: any = [['user_id', userId], 'AND', ['id', dataId]];
 
-    const searchRes = await PgClient.select<KbDataItemType>(PgDatasetTableName, {
+    const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
       fields: ['kb_id', 'id', 'q', 'a', 'source', 'file_id'],
       where,
       limit: 1

client/src/pages/api/core/dataset/data/getDataList.ts

@@ -3,9 +3,9 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
-import type { KbDataItemType } from '@/types/plugin';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { OtherFileId } from '@/constants/kb';
+import { OtherFileId } from '@/constants/dataset';
+import type { PgDataItemType } from '@/types/core/dataset/data';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -50,7 +50,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     ];
 
     const [searchRes, total] = await Promise.all([
-      PgClient.select<KbDataItemType>(PgDatasetTableName, {
+      PgClient.select<PgDataItemType>(PgDatasetTableName, {
         fields: ['id', 'q', 'a', 'source', 'file_id'],
         where,
         order: [{ field: 'id', mode: 'DESC' }],

client/src/pages/api/core/dataset/data/insertData.ts

@@ -1,18 +1,18 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, KB } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
 import { authKb, authUser } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { insertKbItem, PgClient } from '@/service/pg';
+import { insertData2Dataset, PgClient } from '@/service/pg';
 import { getVectorModel } from '@/service/utils/data';
 import { getVector } from '@/pages/api/openapi/plugin/vector';
-import { DatasetItemType } from '@/types/plugin';
+import { DatasetDataItemType } from '@/types/core/dataset/data';
 import { countPromptTokens } from '@/utils/common/tiktoken';
 
 export type Props = {
   kbId: string;
-  data: DatasetItemType;
+  data: DatasetDataItemType;
 };
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -58,7 +58,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       userId
     });
 
-    const response = await insertKbItem({
+    const response = await insertData2Dataset({
       userId,
       kbId,
       data: [

client/src/pages/api/core/dataset/data/pushData.ts

@@ -8,19 +8,9 @@ import { PgDatasetTableName, TrainingModeEnum } from '@/constants/plugin';
 import { startQueue } from '@/service/utils/tools';
 import { PgClient } from '@/service/pg';
 import { getVectorModel } from '@/service/utils/data';
-import { DatasetItemType } from '@/types/plugin';
+import { DatasetDataItemType } from '@/types/core/dataset/data';
 import { countPromptTokens } from '@/utils/common/tiktoken';
-
-export type Props = {
-  kbId: string;
-  data: DatasetItemType[];
-  mode: `${TrainingModeEnum}`;
-  prompt?: string;
-};
-
-export type Response = {
-  insertLen: number;
-};
+import type { PushDataProps, PushDataResponse } from '@/api/core/dataset/data.d';
 
 const modeMap = {
   [TrainingModeEnum.index]: true,
@@ -29,7 +19,7 @@ const modeMap = {
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    const { kbId, data, mode = TrainingModeEnum.index, prompt } = req.body as Props;
+    const { kbId, data, mode = TrainingModeEnum.index, prompt } = req.body as PushDataProps;
 
     if (!kbId || !Array.isArray(data)) {
       throw new Error('KbId or data is empty');
@@ -48,7 +38,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     // 凭证校验
     const { userId } = await authUser({ req });
 
-    jsonRes<Response>(res, {
+    jsonRes<PushDataResponse>(res, {
       data: await pushDataToKb({
         kbId,
         data,
@@ -71,7 +61,7 @@ export async function pushDataToKb({
   data,
   mode,
   prompt
-}: { userId: string } & Props): Promise<Response> {
+}: { userId: string } & PushDataProps): Promise<PushDataResponse> {
   const [kb, vectorModel] = await Promise.all([
     authKb({
       userId,
@@ -94,7 +84,7 @@ export async function pushDataToKb({
 
   // 过滤重复的 qa 内容
   const set = new Set();
-  const filterData: DatasetItemType[] = [];
+  const filterData: DatasetDataItemType[] = [];
 
   data.forEach((item) => {
     if (!item.q) return;
@@ -150,7 +140,7 @@ export async function pushDataToKb({
     )
   )
     .filter((item) => item.status === 'fulfilled')
-    .map<DatasetItemType>((item: any) => item.value);
+    .map<DatasetDataItemType>((item: any) => item.value);
 
   // 插入记录
   const insertRes = await TrainingData.insertMany(

client/src/pages/api/core/dataset/data/updateData.ts

@@ -4,19 +4,13 @@ import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
 import { KB, connectToDatabase } from '@/service/mongo';
-import { getVector } from '../plugin/vector';
+import { getVector } from '@/pages/api/openapi/plugin/vector';
 import { PgDatasetTableName } from '@/constants/plugin';
-
-export type Props = {
-  dataId: string;
-  kbId: string;
-  a?: string;
-  q?: string;
-};
+import type { UpdateDataPrams } from '@/api/core/dataset/data.d';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    const { dataId, a = '', q = '', kbId } = req.body as Props;
+    const { dataId, a = '', q = '', kbId } = req.body as UpdateDataPrams;
 
     if (!dataId) {
       throw new Error('缺少参数');

client/src/pages/api/core/dataset/file/delById.ts

@@ -6,7 +6,7 @@ import { GridFSStorage } from '@/service/lib/gridfs';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { Types } from 'mongoose';
-import { OtherFileId } from '@/constants/kb';
+import { OtherFileId } from '@/constants/dataset';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {

client/src/pages/api/core/dataset/file/detail.ts

@@ -3,8 +3,8 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
-import { OtherFileId } from '@/constants/kb';
-import type { FileInfo } from '@/types/plugin';
+import { OtherFileId } from '@/constants/dataset';
+import type { GSFileInfoType } from '@/types/common/file';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const { userId } = await authUser({ req, authToken: true });
 
     if (fileId === OtherFileId) {
-      return jsonRes<FileInfo>(res, {
+      return jsonRes<GSFileInfoType>(res, {
         data: {
           id: OtherFileId,
           size: 0,
@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const file = await gridFs.findAndAuthFile(fileId);
 
-    jsonRes<FileInfo>(res, {
+    jsonRes<GSFileInfoType>(res, {
       data: file
     });
   } catch (err) {

client/src/pages/api/core/dataset/file/list.ts

@@ -5,7 +5,7 @@ import { authUser } from '@/service/utils/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { FileStatusEnum, OtherFileId } from '@/constants/kb';
+import { FileStatusEnum, OtherFileId } from '@/constants/dataset';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {

client/src/pages/api/core/dataset/list.ts

@@ -3,8 +3,8 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, KB } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { getVectorModel } from '@/service/utils/data';
-import { KbListItemType } from '@/types/plugin';
-import { KbTypeEnum } from '@/constants/kb';
+import type { DatasetsItemType } from '@/types/core/dataset';
+import { KbTypeEnum } from '@/constants/dataset';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -27,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       }))
     );
 
-    jsonRes<KbListItemType[]>(res, {
+    jsonRes<DatasetsItemType[]>(res, {
       data
     });
   } catch (err) {

client/src/pages/api/core/dataset/paths.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, KB } from '@/service/mongo';
-import { KbPathItemType } from '@/types/plugin';
+import type { DatasetPathItemType } from '@/types/core/dataset';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -9,7 +9,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     const { parentId } = req.query as { parentId: string };
 
-    jsonRes<KbPathItemType[]>(res, {
+    jsonRes<DatasetPathItemType[]>(res, {
       data: await getParents(parentId)
     });
   } catch (err) {
@@ -20,7 +20,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
   }
 }
 
-async function getParents(parentId?: string): Promise<KbPathItemType[]> {
+async function getParents(parentId?: string): Promise<DatasetPathItemType[]> {
   if (!parentId) {
     return [];
   }

client/src/pages/api/core/dataset/searchTest.ts

@@ -3,20 +3,14 @@ import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
-import { getVector } from '../plugin/vector';
-import type { KbTestItemType } from '@/types/plugin';
+import { getVector } from '../../openapi/plugin/vector';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { KB } from '@/service/mongo';
-
-export type Props = {
-  kbId: string;
-  text: string;
-};
-export type Response = KbTestItemType['results'];
+import type { SearchTestProps, SearchTestResponseType } from '@/api/core/dataset/index.d';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    const { kbId, text } = req.body as Props;
+    const { kbId, text } = req.body as SearchTestProps;
 
     if (!kbId || !text) {
       throw new Error('缺少参数');
@@ -49,7 +43,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
         COMMIT;`
     );
 
-    jsonRes<Response>(res, {
+    jsonRes<SearchTestResponseType>(res, {
       data: response?.[2]?.rows || []
     });
   } catch (err) {

client/src/pages/api/core/dataset/update.ts

@@ -2,11 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, KB } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
-import type { KbUpdateParams } from '@/api/request/kb';
+import type { DatasetUpdateParams } from '@/api/core/dataset/index.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    const { id, parentId, name, avatar, tags } = req.body as KbUpdateParams;
+    const { id, parentId, name, avatar, tags } = req.body as DatasetUpdateParams;
 
     if (!id) {
       throw new Error('缺少参数');

client/src/pages/app/detail/components/API.tsx

@@ -1,6 +1,6 @@
 import React, { useEffect, useState } from 'react';
 import { Box, Divider, Flex, useTheme, Button, Skeleton, useDisclosure } from '@chakra-ui/react';
-import { useCopyData } from '@/utils/tools';
+import { useCopyData } from '@/hooks/useCopyData';
 import dynamic from 'next/dynamic';
 import MyIcon from '@/components/Icon';
 import { useGlobalStore } from '@/store/global';

client/src/pages/app/detail/components/AdEdit/components/Nodes/NodeKbSearch.tsx

@@ -9,16 +9,16 @@ import Divider from '../modules/Divider';
 import Container from '../modules/Container';
 import RenderInput from '../render/RenderInput';
 import RenderOutput from '../render/RenderOutput';
-import { KBSelectModal } from '../../../KBSelectModal';
-import type { SelectedKbType } from '@/types/plugin';
+import { DatasetSelectModal } from '../../../DatasetSelectModal';
+import type { SelectedDatasetType } from '@/types/core/dataset';
 import Avatar from '@/components/Avatar';
 
 const KBSelect = ({
   activeKbs = [],
   onChange
 }: {
-  activeKbs: SelectedKbType;
-  onChange: (e: SelectedKbType) => void;
+  activeKbs: SelectedDatasetType;
+  onChange: (e: SelectedDatasetType) => void;
 }) => {
   const theme = useTheme();
   const { allDatasets, loadAllDatasets } = useDatasetStore();
@@ -57,7 +57,7 @@ const KBSelect = ({
           </Flex>
         ))}
       </Grid>
-      <KBSelectModal
+      <DatasetSelectModal
         isOpen={isOpenKbSelect}
         activeKbs={activeKbs}
         onChange={onChange}

client/src/pages/app/detail/components/AdEdit/index.tsx

@@ -33,7 +33,7 @@ import type { AppSchema } from '@/types/mongoSchema';
 import { useUserStore } from '@/store/user';
 import { useToast } from '@/hooks/useToast';
 import { useTranslation } from 'next-i18next';
-import { useCopyData } from '@/utils/tools';
+import { useCopyData } from '@/hooks/useCopyData';
 import dynamic from 'next/dynamic';
 
 import MyIcon from '@/components/Icon';

client/src/pages/app/detail/components/BasicEdit/index.tsx

@@ -56,13 +56,13 @@ import MyIcon from '@/components/Icon';
 import ChatBox, { type ComponentRef, type StartChatFnProps } from '@/components/ChatBox';
 
 import { addVariable } from '../VariableEditModal';
-import { KbParamsModal } from '../KBSelectModal';
+import { KbParamsModal } from '../DatasetSelectModal';
 import { AppTypeEnum } from '@/constants/app';
 import { useDatasetStore } from '@/store/dataset';
 
 const VariableEditModal = dynamic(() => import('../VariableEditModal'));
 const InfoModal = dynamic(() => import('../InfoModal'));
-const KBSelectModal = dynamic(() => import('../KBSelectModal'));
+const DatasetSelectModal = dynamic(() => import('../DatasetSelectModal'));
 const AIChatSettingsModal = dynamic(() => import('../AIChatSettingsModal'));
 
 const Settings = ({ appId }: { appId: string }) => {
@@ -565,7 +565,7 @@ const Settings = ({ appId }: { appId: string }) => {
         />
       )}
       {isOpenKbSelect && (
-        <KBSelectModal
+        <DatasetSelectModal
           isOpen={isOpenKbSelect}
           activeKbs={selectedKbList.map((item) => ({
             kbId: item._id,

client/src/pages/app/detail/components/DatasetSelectModal.tsx

@@ -14,18 +14,18 @@ import {
 import Avatar from '@/components/Avatar';
 import { useForm } from 'react-hook-form';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import type { SelectedKbType } from '@/types/plugin';
+import type { SelectedDatasetType } from '@/types/core/dataset';
 import { useToast } from '@/hooks/useToast';
 import MySlider from '@/components/Slider';
 import MyTooltip from '@/components/MyTooltip';
 import MyModal from '@/components/MyModal';
 import MyIcon from '@/components/Icon';
-import { KbTypeEnum } from '@/constants/kb';
+import { KbTypeEnum } from '@/constants/dataset';
 import { useTranslation } from 'react-i18next';
 import { useQuery } from '@tanstack/react-query';
 import { useDatasetStore } from '@/store/dataset';
 import { feConfigs } from '@/store/static';
-import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
+import DatasetSelectContainer, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
 
 export type KbParamsType = {
   searchSimilarity: number;
@@ -33,20 +33,20 @@ export type KbParamsType = {
   searchEmptyText: string;
 };
 
-export const KBSelectModal = ({
+export const DatasetSelectModal = ({
   isOpen,
   activeKbs = [],
   onChange,
   onClose
 }: {
   isOpen: boolean;
-  activeKbs: SelectedKbType;
-  onChange: (e: SelectedKbType) => void;
+  activeKbs: SelectedDatasetType;
+  onChange: (e: SelectedDatasetType) => void;
   onClose: () => void;
 }) => {
   const { t } = useTranslation();
   const theme = useTheme();
-  const [selectedKbList, setSelectedKbList] = useState<SelectedKbType>(activeKbs);
+  const [selectedKbList, setSelectedKbList] = useState<SelectedDatasetType>(activeKbs);
   const { toast } = useToast();
   const { paths, parentId, setParentId, datasets } = useDatasetSelect();
   const { allDatasets, loadAllDatasets } = useDatasetStore();
@@ -61,7 +61,7 @@ export const KBSelectModal = ({
   }, [datasets, allDatasets, selectedKbList]);
 
   return (
-    <DatasetSelectModal
+    <DatasetSelectContainer
       isOpen={isOpen}
       paths={paths}
       parentId={parentId}
@@ -186,7 +186,7 @@ export const KBSelectModal = ({
           onClick={() => {
             // filter out the kb that is not in the kList
             const filterKbList = selectedKbList.filter((kb) => {
-              return datasets.find((item) => item._id === kb.kbId);
+              return allDatasets.find((item) => item._id === kb.kbId);
             });
 
             onClose();
@@ -196,7 +196,7 @@ export const KBSelectModal = ({
           完成
         </Button>
       </ModalFooter>
-    </DatasetSelectModal>
+    </DatasetSelectContainer>
   );
 };
 
@@ -297,4 +297,4 @@ export const KbParamsModal = ({
   );
 };
 
-export default KBSelectModal;
+export default DatasetSelectModal;

client/src/pages/app/detail/components/InfoModal.tsx

@@ -13,7 +13,7 @@ import { useForm } from 'react-hook-form';
 import { AppSchema } from '@/types/mongoSchema';
 import { useToast } from '@/hooks/useToast';
 import { useSelectFile } from '@/hooks/useSelectFile';
-import { compressImg } from '@/utils/file';
+import { compressImg } from '@/utils/web/file';
 import { getErrText } from '@/utils/tools';
 import { useUserStore } from '@/store/user';
 import { useRequest } from '@/hooks/useRequest';

client/src/pages/app/detail/components/OutLink.tsx

@@ -30,7 +30,8 @@ import {
   createShareChat,
   putShareChat
 } from '@/api/support/outLink';
-import { formatTimeToChatTime, useCopyData } from '@/utils/tools';
+import { formatTimeToChatTime } from '@/utils/tools';
+import { useCopyData } from '@/hooks/useCopyData';
 import { useForm } from 'react-hook-form';
 import { defaultOutLinkForm } from '@/constants/model';
 import type { OutLinkEditType } from '@/types/support/outLink';

client/src/pages/app/detail/index.tsx

@@ -14,7 +14,7 @@ import MyIcon from '@/components/Icon';
 import PageContainer from '@/components/PageContainer';
 import Loading from '@/components/Loading';
 import BasicEdit from './components/BasicEdit';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 
 const AdEdit = dynamic(() => import('./components/AdEdit'), {
   ssr: false,

client/src/pages/app/list/component/CreateModal.tsx

@@ -13,7 +13,7 @@ import {
 } from '@chakra-ui/react';
 import { useSelectFile } from '@/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImg } from '@/utils/file';
+import { compressImg } from '@/utils/web/file';
 import { getErrText } from '@/utils/tools';
 import { useToast } from '@/hooks/useToast';
 import { postCreateApp } from '@/api/app';

client/src/pages/app/list/index.tsx

@@ -16,7 +16,7 @@ import { AddIcon } from '@chakra-ui/icons';
 import { delModelById } from '@/api/app';
 import { useToast } from '@/hooks/useToast';
 import { useConfirm } from '@/hooks/useConfirm';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { useTranslation } from 'next-i18next';
 
 import MyIcon from '@/components/Icon';

client/src/pages/appStore/index.tsx

@@ -7,7 +7,7 @@ import type { ShareAppItem } from '@/types/app';
 import { useUserStore } from '@/store/user';
 import ShareModelList from './components/list';
 import styles from './index.module.scss';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 
 const modelList = () => {
   const { Loading } = useLoading();

client/src/pages/chat/index.tsx

@@ -30,7 +30,7 @@ import SliderApps from './components/SliderApps';
 import ChatHeader from './components/ChatHeader';
 import { getErrText } from '@/utils/tools';
 import { useUserStore } from '@/store/user';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 
 const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => {
   const router = useRouter();

client/src/pages/chat/share.tsx

@@ -19,7 +19,7 @@ import ChatBox, { type ComponentRef, type StartChatFnProps } from '@/components/
 import PageContainer from '@/components/PageContainer';
 import ChatHeader from './components/ChatHeader';
 import ChatHistorySlider from './components/ChatHistorySlider';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 
 const OutLink = ({ shareId, chatId }: { shareId: string; chatId: string }) => {
   const router = useRouter();

client/src/pages/index.tsx

@@ -1,7 +1,7 @@
 import React, { useEffect } from 'react';
 import { Box } from '@chakra-ui/react';
 import { feConfigs } from '@/store/static';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { useRouter } from 'next/router';
 
 import Navbar from './components/Navbar';
@@ -10,6 +10,7 @@ import Ability from './components/Ability';
 import Choice from './components/Choice';
 import Footer from './components/Footer';
 import Loading from '@/components/Loading';
+import Head from 'next/head';
 
 const Home = ({ homeUrl = '/' }: { homeUrl: string }) => {
   const router = useRouter();
@@ -23,26 +24,33 @@ const Home = ({ homeUrl = '/' }: { homeUrl: string }) => {
     router.prefetch('/login');
   }, []);
 
-  return homeUrl === '/' ? (
-    <Box id="home" bg={'myWhite.600'} h={'100vh'} overflowY={'auto'} overflowX={'hidden'}>
-      <Box position={'fixed'} zIndex={10} top={0} left={0} right={0}>
-        <Navbar />
-      </Box>
-      <Box maxW={'1200px'} pt={'70px'} m={'auto'}>
-        <Hero />
-        <Ability />
-        <Box my={[4, 6]}>
-          <Choice />
-        </Box>
-      </Box>
-      {feConfigs?.show_git && (
-        <Box bg={'white'}>
-          <Footer />
+  return (
+    <>
+      <Head>
+        <title>{feConfigs?.systemTitle || 'FastGPT'}</title>
+      </Head>
+      {homeUrl === '/' ? (
+        <Box id="home" bg={'myWhite.600'} h={'100vh'} overflowY={'auto'} overflowX={'hidden'}>
+          <Box position={'fixed'} zIndex={10} top={0} left={0} right={0}>
+            <Navbar />
+          </Box>
+          <Box maxW={'1200px'} pt={'70px'} m={'auto'}>
+            <Hero />
+            <Ability />
+            <Box my={[4, 6]}>
+              <Choice />
+            </Box>
+          </Box>
+          {feConfigs?.show_git && (
+            <Box bg={'white'}>
+              <Footer />
+            </Box>
+          )}
         </Box>
+      ) : (
+        <Loading />
       )}
-    </Box>
-  ) : (
-    <Loading />
+    </>
   );
 };
 

client/src/pages/kb/detail/components/DataCard.tsx

@@ -1,8 +1,12 @@
 import React, { useCallback, useState, useRef, useMemo } from 'react';
 import { Box, Card, IconButton, Flex, Grid, Image } from '@chakra-ui/react';
-import type { KbDataItemType } from '@/types/plugin';
+import type { PgDataItemType } from '@/types/core/dataset/data';
 import { usePagination } from '@/hooks/usePagination';
-import { getKbDataList, delOneKbDataByDataId, getTrainingData } from '@/api/plugins/kb';
+import {
+  getDatasetDataList,
+  delOneDatasetDataById,
+  getTrainingData
+} from '@/api/core/dataset/data';
 import { getFileInfoById } from '@/api/core/dataset/file';
 import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
 import { useQuery } from '@tanstack/react-query';
@@ -39,8 +43,8 @@ const DataCard = ({ kbId }: { kbId: string }) => {
     getData,
     pageNum,
     pageSize
-  } = usePagination<KbDataItemType>({
-    api: getKbDataList,
+  } = usePagination<PgDataItemType>({
+    api: getDatasetDataList,
     pageSize: 24,
     params: {
       kbId,
@@ -205,7 +209,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
                   openConfirm(async () => {
                     try {
                       setIsDeleting(true);
-                      await delOneKbDataByDataId(item.id);
+                      await delOneDatasetDataById(item.id);
                       getData(pageNum);
                     } catch (error) {
                       toast({

client/src/pages/kb/detail/components/FileCard.tsx

@@ -10,12 +10,9 @@ import {
   Td,
   Tbody,
   Image,
-  MenuButton,
-  Menu,
-  MenuList,
-  MenuItem
+  MenuButton
 } from '@chakra-ui/react';
-import { getTrainingData } from '@/api/plugins/kb';
+import { getTrainingData } from '@/api/core/dataset/data';
 import { getDatasetFiles, delDatasetFileById, updateDatasetFile } from '@/api/core/dataset/file';
 import { useQuery } from '@tanstack/react-query';
 import { debounce } from 'lodash';
@@ -28,10 +25,10 @@ import dayjs from 'dayjs';
 import { fileImgs } from '@/constants/common';
 import { useRequest } from '@/hooks/useRequest';
 import { useLoading } from '@/hooks/useLoading';
-import { FileStatusEnum, OtherFileId } from '@/constants/kb';
+import { FileStatusEnum, OtherFileId } from '@/constants/dataset';
 import { useRouter } from 'next/router';
 import { usePagination } from '@/hooks/usePagination';
-import { KbFileItemType } from '@/types/plugin';
+import type { DatasetFileItemType } from '@/types/core/dataset/file';
 import { useGlobalStore } from '@/store/global';
 import MyMenu from '@/components/MyMenu';
 import { useEditTitle } from '@/hooks/useEditTitle';
@@ -56,7 +53,7 @@ const FileCard = ({ kbId }: { kbId: string }) => {
     isLoading,
     pageNum,
     pageSize
-  } = usePagination<KbFileItemType>({
+  } = usePagination<DatasetFileItemType>({
     api: getDatasetFiles,
     pageSize: 20,
     params: {

client/src/pages/kb/detail/components/Import/Chunk.tsx

@@ -15,7 +15,6 @@ import { useToast } from '@/hooks/useToast';
 import { useConfirm } from '@/hooks/useConfirm';
 import { useRouter } from 'next/router';
 import { useMutation } from '@tanstack/react-query';
-import { postKbDataFromList } from '@/api/plugins/kb';
 import { splitText2Chunks } from '@/utils/file';
 import { getErrText } from '@/utils/tools';
 import { formatPrice } from '@/utils/user';
@@ -28,6 +27,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
 import FileSelect, { type FileItemType } from './FileSelect';
 import { useDatasetStore } from '@/store/dataset';
 import { updateDatasetFile } from '@/api/core/dataset/file';
+import { chunksUpload } from '@/utils/web/core/dataset';
 
 const fileExtension = '.txt, .doc, .docx, .pdf, .md';
 
@@ -75,22 +75,18 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
         )
       );
 
-      // subsection import
-      let success = 0;
-      const step = 300;
-      for (let i = 0; i < chunks.length; i += step) {
-        const { insertLen } = await postKbDataFromList({
-          kbId,
-          data: chunks.slice(i, i + step),
-          mode: TrainingModeEnum.index
-        });
-
-        success += insertLen;
-        setSuccessChunks(success);
-      }
+      // upload data
+      const { insertLen } = await chunksUpload({
+        kbId,
+        chunks,
+        mode: TrainingModeEnum.index,
+        onUploading: (insertLen) => {
+          setSuccessChunks(insertLen);
+        }
+      });
 
       toast({
-        title: `去重后共导入 ${success} 条数据，请耐心等待训练.`,
+        title: `去重后共导入 ${insertLen} 条数据，请耐心等待训练.`,
         status: 'success'
       });
 

client/src/pages/kb/detail/components/Import/Csv.tsx

@@ -3,7 +3,6 @@ import { Box, Flex, Button, useTheme, Image } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
 import { useConfirm } from '@/hooks/useConfirm';
 import { useMutation } from '@tanstack/react-query';
-import { postKbDataFromList } from '@/api/plugins/kb';
 import { getErrText } from '@/utils/tools';
 import MyIcon from '@/components/Icon';
 import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
@@ -12,6 +11,7 @@ import FileSelect, { type FileItemType } from './FileSelect';
 import { useRouter } from 'next/router';
 import { useDatasetStore } from '@/store/dataset';
 import { updateDatasetFile } from '@/api/core/dataset/file';
+import { chunksUpload } from '@/utils/web/core/dataset';
 
 const fileExtension = '.csv';
 
@@ -62,22 +62,18 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
         });
       }
 
-      // subsection import
-      let success = 0;
-      const step = 300;
-      for (let i = 0; i < filterChunks.length; i += step) {
-        const { insertLen } = await postKbDataFromList({
-          kbId,
-          data: filterChunks.slice(i, i + step),
-          mode: TrainingModeEnum.index
-        });
-
-        success += insertLen;
-        setSuccessChunks(success);
-      }
+      // upload data
+      const { insertLen } = await chunksUpload({
+        kbId,
+        chunks,
+        mode: TrainingModeEnum.index,
+        onUploading: (insertLen) => {
+          setSuccessChunks(insertLen);
+        }
+      });
 
       toast({
-        title: `去重后共导入 ${success} 条数据，请耐心等待训练.`,
+        title: `去重后共导入 ${insertLen} 条数据，请耐心等待训练.`,
         status: 'success'
       });
 

client/src/pages/kb/detail/components/Import/FileSelect.tsx

@@ -2,22 +2,24 @@ import MyIcon from '@/components/Icon';
 import { useLoading } from '@/hooks/useLoading';
 import { useSelectFile } from '@/hooks/useSelectFile';
 import { useToast } from '@/hooks/useToast';
+import { simpleText, splitText2Chunks } from '@/utils/file';
 import {
+  uploadFiles,
   fileDownload,
   readCsvContent,
-  simpleText,
-  splitText2Chunks,
-  uploadFiles
-} from '@/utils/file';
+  readTxtContent,
+  readPdfContent,
+  readDocContent
+} from '@/utils/web/file';
 import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
 import { fileImgs } from '@/constants/common';
 import { DragEvent, useCallback, useState } from 'react';
 import { useTranslation } from 'next-i18next';
-import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
-import { FetchResultItem, DatasetItemType } from '@/types/plugin';
+import { FetchResultItem } from '@/types/plugin';
+import type { DatasetDataItemType } from '@/types/core/dataset/data';
 import { getErrText } from '@/utils/tools';
 import { useDatasetStore } from '@/store/dataset';
 
@@ -30,7 +32,7 @@ const csvTemplate = `index,content,source\n"被索引的内容","对应的答案
 export type FileItemType = {
   id: string;
   filename: string;
-  chunks: DatasetItemType[];
+  chunks: DatasetDataItemType[];
   text: string;
   icon: string;
   tokens: number;

client/src/pages/kb/detail/components/Import/Manual.tsx

@@ -4,7 +4,7 @@ import { useForm } from 'react-hook-form';
 import { useToast } from '@/hooks/useToast';
 import { useRequest } from '@/hooks/useRequest';
 import { getErrText } from '@/utils/tools';
-import { postKbDataFromList } from '@/api/plugins/kb';
+import { postChunks2Dataset } from '@/api/core/dataset/data';
 import { TrainingModeEnum } from '@/constants/plugin';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
@@ -38,7 +38,7 @@ const ManualImport = ({ kbId }: { kbId: string }) => {
           q: e.q,
           source: '手动录入'
         };
-        const { insertLen } = await postKbDataFromList({
+        const { insertLen } = await postChunks2Dataset({
           kbId,
           mode: TrainingModeEnum.index,
           data: [data]

client/src/pages/kb/detail/components/Import/QA.tsx

@@ -3,7 +3,6 @@ import { Box, Flex, Button, useTheme, Image, Input } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
 import { useConfirm } from '@/hooks/useConfirm';
 import { useMutation } from '@tanstack/react-query';
-import { postKbDataFromList } from '@/api/plugins/kb';
 import { splitText2Chunks } from '@/utils/file';
 import { getErrText } from '@/utils/tools';
 import { formatPrice } from '@/utils/user';
@@ -19,6 +18,7 @@ import { useRouter } from 'next/router';
 import { updateDatasetFile } from '@/api/core/dataset/file';
 import { Prompt_AgentQA } from '@/prompts/core/agent';
 import { replaceVariable } from '@/utils/common/tools/text';
+import { chunksUpload } from '@/utils/web/core/dataset';
 
 const fileExtension = '.txt, .doc, .docx, .pdf, .md';
 
@@ -74,23 +74,19 @@ const QAImport = ({ kbId }: { kbId: string }) => {
         )
       );
 
-      // subsection import
-      let success = 0;
-      const step = 200;
-      for (let i = 0; i < chunks.length; i += step) {
-        const { insertLen } = await postKbDataFromList({
-          kbId,
-          data: chunks.slice(i, i + step),
-          mode: TrainingModeEnum.qa,
-          prompt: previewQAPrompt
-        });
-
-        success += insertLen;
-        setSuccessChunks(success);
-      }
+      // upload data
+      const { insertLen } = await chunksUpload({
+        kbId,
+        chunks,
+        mode: TrainingModeEnum.qa,
+        prompt: previewQAPrompt,
+        onUploading: (insertLen) => {
+          setSuccessChunks(insertLen);
+        }
+      });
 
       toast({
-        title: `共导入 ${success} 条数据，请耐心等待训练.`,
+        title: `共导入 ${insertLen} 条数据，请耐心等待训练.`,
         status: 'success'
       });
 

client/src/pages/kb/detail/components/Info.tsx

@@ -9,14 +9,14 @@ import React, {
 import { useRouter } from 'next/router';
 import { Box, Flex, Button, FormControl, IconButton, Input } from '@chakra-ui/react';
 import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
-import { delKbById, putKbById } from '@/api/plugins/kb';
+import { delDatasetById, putDatasetById } from '@/api/core/dataset';
 import { useSelectFile } from '@/hooks/useSelectFile';
 import { useToast } from '@/hooks/useToast';
 import { useDatasetStore } from '@/store/dataset';
 import { useConfirm } from '@/hooks/useConfirm';
 import { UseFormReturn } from 'react-hook-form';
-import { compressImg } from '@/utils/file';
-import type { KbItemType } from '@/types/plugin';
+import { compressImg } from '@/utils/web/file';
+import type { DatasetItemType } from '@/types/core/dataset';
 import Avatar from '@/components/Avatar';
 import Tag from '@/components/Tag';
 import MyTooltip from '@/components/MyTooltip';
@@ -26,7 +26,7 @@ export interface ComponentRef {
 }
 
 const Info = (
-  { kbId, form }: { kbId: string; form: UseFormReturn<KbItemType, any> },
+  { kbId, form }: { kbId: string; form: UseFormReturn<DatasetItemType, any> },
   ref: ForwardedRef<ComponentRef>
 ) => {
   const { getValues, formState, setValue, register, handleSubmit } = form;
@@ -53,7 +53,7 @@ const Info = (
   const onclickDelKb = useCallback(async () => {
     setBtnLoading(true);
     try {
-      await delKbById(kbId);
+      await delDatasetById(kbId);
       toast({
         title: '删除成功',
         status: 'success'
@@ -70,10 +70,10 @@ const Info = (
   }, [setBtnLoading, kbId, toast, router, loadKbList]);
 
   const saveSubmitSuccess = useCallback(
-    async (data: KbItemType) => {
+    async (data: DatasetItemType) => {
       setBtnLoading(true);
       try {
-        await putKbById({
+        await putDatasetById({
           id: kbId,
           ...data
         });

client/src/pages/kb/detail/components/InputDataModal.tsx

@@ -1,7 +1,11 @@
 import React, { useState, useCallback } from 'react';
 import { Box, Flex, Button, Textarea, IconButton, BoxProps } from '@chakra-ui/react';
 import { useForm } from 'react-hook-form';
-import { insertData2Kb, putKbDataById, delOneKbDataByDataId } from '@/api/plugins/kb';
+import {
+  postData2Dataset,
+  putDatasetDataById,
+  delOneDatasetDataById
+} from '@/api/core/dataset/data';
 import { useToast } from '@/hooks/useToast';
 import { getErrText } from '@/utils/tools';
 import MyIcon from '@/components/Icon';
@@ -9,12 +13,12 @@ import MyModal from '@/components/MyModal';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useQuery } from '@tanstack/react-query';
-import { DatasetItemType } from '@/types/plugin';
+import { DatasetDataItemType } from '@/types/core/dataset/data';
 import { useTranslation } from 'react-i18next';
 import { useDatasetStore } from '@/store/dataset';
-import { getFileAndOpen } from '@/utils/common/file';
+import { getFileAndOpen } from '@/utils/web/file';
 
-export type FormData = { dataId?: string } & DatasetItemType;
+export type FormData = { dataId?: string } & DatasetDataItemType;
 
 const InputDataModal = ({
   onClose,
@@ -65,7 +69,7 @@ const InputDataModal = ({
           q: e.q,
           source: '手动录入'
         };
-        data.dataId = await insertData2Kb({
+        data.dataId = await postData2Dataset({
           kbId,
           data
         });
@@ -104,7 +108,7 @@ const InputDataModal = ({
             a: e.a,
             q: e.q === defaultValues.q ? '' : e.q
           };
-          await putKbDataById(data);
+          await putDatasetDataById(data);
           onSuccess(data);
         } catch (err) {
           toast({
@@ -211,7 +215,7 @@ const InputDataModal = ({
                 onClick={async () => {
                   if (!onDelete || !defaultValues.dataId) return;
                   try {
-                    await delOneKbDataByDataId(defaultValues.dataId);
+                    await delOneDatasetDataById(defaultValues.dataId);
                     onDelete();
                     onClose();
                     toast({

client/src/pages/kb/detail/components/Test.tsx

@@ -1,8 +1,8 @@
 import React, { useEffect, useMemo, useState } from 'react';
 import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react';
 import { useDatasetStore } from '@/store/dataset';
-import type { KbTestItemType } from '@/types/plugin';
-import { searchText, getKbDataItemById } from '@/api/plugins/kb';
+import type { SearchTestItemType } from '@/types/core/dataset';
+import { getDatasetDataItemById } from '@/api/core/dataset/data';
 import MyIcon from '@/components/Icon';
 import { useRequest } from '@/hooks/useRequest';
 import { formatTimeToChatTime } from '@/utils/tools';
@@ -13,6 +13,7 @@ import { useToast } from '@/hooks/useToast';
 import { customAlphabet } from 'nanoid';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
+import { postSearchText } from '@/api/core/dataset';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
 
 const Test = ({ kbId }: { kbId: string }) => {
@@ -22,7 +23,7 @@ const Test = ({ kbId }: { kbId: string }) => {
   const { kbDetail, kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } =
     useDatasetStore();
   const [inputText, setInputText] = useState('');
-  const [kbTestItem, setKbTestItem] = useState<KbTestItemType>();
+  const [kbTestItem, setKbTestItem] = useState<SearchTestItemType>();
   const [editData, setEditData] = useState<FormData>();
 
   const kbTestHistory = useMemo(
@@ -31,7 +32,7 @@ const Test = ({ kbId }: { kbId: string }) => {
   );
 
   const { mutate, isLoading } = useRequest({
-    mutationFn: () => searchText({ kbId, text: inputText.trim() }),
+    mutationFn: () => postSearchText({ kbId, text: inputText.trim() }),
     onSuccess(res) {
       const testItem = {
         id: nanoid(),
@@ -197,7 +198,7 @@ const Test = ({ kbId }: { kbId: string }) => {
                   onClick={async () => {
                     try {
                       setLoading(true);
-                      const data = await getKbDataItemById(item.id);
+                      const data = await getDatasetDataItemById(item.id);
 
                       if (!data) {
                         throw new Error('该数据已被删除');

client/src/pages/kb/detail/index.tsx

@@ -4,7 +4,7 @@ import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
 import { useForm } from 'react-hook-form';
 import { useQuery } from '@tanstack/react-query';
-import { KbItemType } from '@/types/plugin';
+import { DatasetItemType } from '@/types/core/dataset';
 import { getErrText } from '@/utils/tools';
 import { useGlobalStore } from '@/store/global';
 import { type ComponentRef } from './components/Info';
@@ -15,9 +15,9 @@ import SideTabs from '@/components/SideTabs';
 import PageContainer from '@/components/PageContainer';
 import Avatar from '@/components/Avatar';
 import Info from './components/Info';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { useTranslation } from 'react-i18next';
-import { getTrainingQueueLen } from '@/api/plugins/kb';
+import { getTrainingQueueLen } from '@/api/core/dataset/data';
 import { delDatasetEmptyFiles } from '@/api/core/dataset/file';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
@@ -72,7 +72,7 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
     [kbId, router]
   );
 
-  const form = useForm<KbItemType>({
+  const form = useForm<DatasetItemType>({
     defaultValues: kbDetail
   });
 

client/src/pages/kb/list/component/CreateModal.tsx

@@ -2,7 +2,7 @@ import React, { useCallback, useState, useRef } from 'react';
 import { Box, Flex, Button, ModalHeader, ModalFooter, ModalBody, Input } from '@chakra-ui/react';
 import { useSelectFile } from '@/hooks/useSelectFile';
 import { useForm } from 'react-hook-form';
-import { compressImg } from '@/utils/file';
+import { compressImg } from '@/utils/web/file';
 import { getErrText } from '@/utils/tools';
 import { useToast } from '@/hooks/useToast';
 import { useRouter } from 'next/router';
@@ -11,8 +11,8 @@ import { useRequest } from '@/hooks/useRequest';
 import Avatar from '@/components/Avatar';
 import MyTooltip from '@/components/MyTooltip';
 import MyModal from '@/components/MyModal';
-import { postCreateKb } from '@/api/plugins/kb';
-import type { CreateKbParams } from '@/api/request/kb';
+import { postCreateDataset } from '@/api/core/dataset';
+import type { CreateDatasetParams } from '@/api/core/dataset/index.d';
 import { vectorModelList } from '@/store/static';
 import MySelect from '@/components/Select';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
@@ -23,7 +23,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
   const { toast } = useToast();
   const router = useRouter();
   const { isPc } = useGlobalStore();
-  const { register, setValue, getValues, handleSubmit } = useForm<CreateKbParams>({
+  const { register, setValue, getValues, handleSubmit } = useForm<CreateDatasetParams>({
     defaultValues: {
       avatar: '/icon/logo.svg',
       name: '',
@@ -64,8 +64,8 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
 
   /* create a new kb and router to it */
   const { mutate: onclickCreate, isLoading: creating } = useRequest({
-    mutationFn: async (data: CreateKbParams) => {
-      const id = await postCreateKb(data);
+    mutationFn: async (data: CreateDatasetParams) => {
+      const id = await postCreateDataset(data);
       return id;
     },
     successToast: '创建成功',

client/src/pages/kb/list/component/EditFolderModal.tsx

@@ -3,8 +3,8 @@ import { ModalFooter, ModalBody, Input, Button } from '@chakra-ui/react';
 import MyModal from '@/components/MyModal';
 import { useTranslation } from 'react-i18next';
 import { useRequest } from '@/hooks/useRequest';
-import { postCreateKb, putKbById } from '@/api/plugins/kb';
-import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
+import { postCreateDataset, putDatasetById } from '@/api/core/dataset';
+import { FolderAvatarSrc, KbTypeEnum } from '@/constants/dataset';
 
 const EditFolderModal = ({
   onClose,
@@ -39,12 +39,12 @@ const EditFolderModal = ({
       const val = inputRef.current?.value;
       if (!val) return Promise.resolve('');
       if (id) {
-        return putKbById({
+        return putDatasetById({
           id,
           name: val
         });
       }
-      return postCreateKb({
+      return postCreateDataset({
         parentId,
         name: val,
         type: KbTypeEnum.folder,

client/src/pages/kb/list/component/MoveModal.tsx

@@ -10,15 +10,14 @@ import {
   useTheme,
   Grid
 } from '@chakra-ui/react';
-import { getKbPaths } from '@/api/plugins/kb';
 import Avatar from '@/components/Avatar';
 import MyTooltip from '@/components/MyTooltip';
 import MyModal from '@/components/MyModal';
 import MyIcon from '@/components/Icon';
-import { KbTypeEnum } from '@/constants/kb';
+import { KbTypeEnum } from '@/constants/dataset';
 import { useTranslation } from 'react-i18next';
 import { useQuery } from '@tanstack/react-query';
-import { getKbList, putKbById } from '@/api/plugins/kb';
+import { getDatasets, putDatasetById, getDatasetPaths } from '@/api/core/dataset';
 import { useRequest } from '@/hooks/useRequest';
 
 const MoveModal = ({
@@ -35,8 +34,8 @@ const MoveModal = ({
 
   const [parentId, setParentId] = useState<string>('');
 
-  const { data } = useQuery(['getKbList', parentId], () => {
-    return Promise.all([getKbList({ parentId, type: 'folder' }), getKbPaths(parentId)]);
+  const { data } = useQuery(['getDatasets', parentId], () => {
+    return Promise.all([getDatasets({ parentId, type: 'folder' }), getDatasetPaths(parentId)]);
   });
   const paths = useMemo(
     () => [
@@ -54,7 +53,7 @@ const MoveModal = ({
   );
 
   const { mutate, isLoading } = useRequest({
-    mutationFn: () => putKbById({ id: moveDataId, parentId }),
+    mutationFn: () => putDatasetById({ id: moveDataId, parentId }),
     onSuccess,
     errorToast: t('kb.Move Failed')
   });

client/src/pages/kb/list/index.tsx

@@ -15,13 +15,14 @@ import PageContainer from '@/components/PageContainer';
 import { useConfirm } from '@/hooks/useConfirm';
 import { AddIcon } from '@chakra-ui/icons';
 import { useQuery } from '@tanstack/react-query';
-import { delKbById, exportDataset, getKbPaths, putKbById } from '@/api/plugins/kb';
+import { delDatasetById, getDatasetPaths, putDatasetById } from '@/api/core/dataset';
+import { exportDatasetData } from '@/api/core/dataset/data';
 import { useTranslation } from 'react-i18next';
 import Avatar from '@/components/Avatar';
 import MyIcon from '@/components/Icon';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import dynamic from 'next/dynamic';
-import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
+import { FolderAvatarSrc, KbTypeEnum } from '@/constants/dataset';
 import Tag from '@/components/Tag';
 import MyMenu from '@/components/MyMenu';
 import { useRequest } from '@/hooks/useRequest';
@@ -71,7 +72,7 @@ const Kb = () => {
   const { mutate: onclickDelKb } = useRequest({
     mutationFn: async (id: string) => {
       setLoading(true);
-      await delKbById(id);
+      await delDatasetById(id);
       return id;
     },
     onSuccess(id: string) {
@@ -88,7 +89,7 @@ const Kb = () => {
   const { mutate: onclickExport } = useRequest({
     mutationFn: (kbId: string) => {
       setLoading(true);
-      return exportDataset({ kbId });
+      return exportDatasetData({ kbId });
     },
     onSettled() {
       setLoading(false);
@@ -98,7 +99,7 @@ const Kb = () => {
   });
 
   const { data, refetch } = useQuery(['loadDataset', parentId], () => {
-    return Promise.all([loadKbList(parentId), getKbPaths(parentId)]);
+    return Promise.all([loadKbList(parentId), getDatasetPaths(parentId)]);
   });
 
   const paths = useMemo(
@@ -240,7 +241,7 @@ const Kb = () => {
               if (!dragTargetId || !dragStartId || dragTargetId === dragStartId) return;
               // update parentId
               try {
-                await putKbById({
+                await putDatasetById({
                   id: dragStartId,
                   parentId: dragTargetId
                 });

client/src/pages/login/index.tsx

@@ -9,7 +9,7 @@ import { useUserStore } from '@/store/user';
 import { useChatStore } from '@/store/chat';
 import LoginForm from './components/LoginForm';
 import dynamic from 'next/dynamic';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { setToken } from '@/utils/user';
 import { feConfigs } from '@/store/static';
 import CommunityModal from '@/components/CommunityModal';

client/src/pages/login/provider.tsx

@@ -8,7 +8,7 @@ import { setToken } from '@/utils/user';
 import { oauthLogin } from '@/api/user';
 import { useToast } from '@/hooks/useToast';
 import Loading from '@/components/Loading';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { useQuery } from '@tanstack/react-query';
 import { getErrText } from '@/utils/tools';
 

client/src/pages/tools/index.tsx

@@ -4,7 +4,7 @@ import { ChevronRightIcon } from '@chakra-ui/icons';
 import MyIcon from '@/components/Icon';
 import { useRouter } from 'next/router';
 import { feConfigs } from '@/store/static';
-import { serviceSideProps } from '@/utils/i18n';
+import { serviceSideProps } from '@/utils/web/i18n';
 import { useTranslation } from 'react-i18next';
 
 const Tools = () => {

client/src/service/events/generateQA.ts

@@ -1,6 +1,6 @@
 import { TrainingData } from '@/service/mongo';
 import { pushQABill } from '@/service/events/pushBill';
-import { pushDataToKb } from '@/pages/api/openapi/kb/pushData';
+import { pushDataToKb } from '@/pages/api/core/dataset/data/pushData';
 import { TrainingModeEnum } from '@/constants/plugin';
 import { ERROR_ENUM } from '../errorCode';
 import { sendInform } from '@/pages/api/user/inform/send';

client/src/service/events/generateVector.ts

@@ -1,4 +1,4 @@
-import { insertKbItem } from '@/service/pg';
+import { insertData2Dataset } from '@/service/pg';
 import { getVector } from '@/pages/api/openapi/plugin/vector';
 import { TrainingData } from '../models/trainingData';
 import { ERROR_ENUM } from '../errorCode';
@@ -68,7 +68,7 @@ export async function generateVector(): Promise<any> {
     });
 
     // 生成结果插入到 pg
-    await insertKbItem({
+    await insertData2Dataset({
       userId,
       kbId,
       data: vectors.map((vector, i) => ({

client/src/service/lib/gridfs.ts

@@ -2,7 +2,7 @@ import mongoose, { Types } from 'mongoose';
 import fs from 'fs';
 import fsp from 'fs/promises';
 import { ERROR_ENUM } from '../errorCode';
-import type { FileInfo } from '@/types/plugin';
+import type { GSFileInfoType } from '@/types/common/file';
 
 enum BucketNameEnum {
   dataset = 'dataset'
@@ -60,7 +60,7 @@ export class GridFSStorage {
 
     return String(stream.id);
   }
-  async findAndAuthFile(id: string): Promise<FileInfo> {
+  async findAndAuthFile(id: string): Promise<GSFileInfoType> {
     if (!id) {
       return Promise.reject(`id is empty`);
     }

client/src/service/models/kb.ts

@@ -1,6 +1,6 @@
 import { Schema, model, models, Model } from 'mongoose';
 import { kbSchema as SchemaType } from '@/types/mongoSchema';
-import { KbTypeMap } from '@/constants/kb';
+import { KbTypeMap } from '@/constants/dataset';
 
 const kbSchema = new Schema({
   parentId: {

client/src/service/moduleDispatch/kb/search.ts

@@ -3,14 +3,14 @@ import type { ChatHistoryItemResType } from '@/types/chat';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { getVector } from '@/pages/api/openapi/plugin/vector';
 import { countModelPrice } from '@/service/events/pushBill';
-import type { SelectedKbType } from '@/types/plugin';
+import type { SelectedDatasetType } from '@/types/core/dataset';
 import type { QuoteItemType } from '@/types/chat';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { FlowModuleTypeEnum } from '@/constants/flow';
 import { ModuleDispatchProps } from '@/types/core/modules';
 
 type KBSearchProps = ModuleDispatchProps<{
-  kbList: SelectedKbType;
+  kbList: SelectedDatasetType;
   similarity: number;
   limit: number;
   userChatInput: string;

client/src/service/pg.ts

@@ -2,7 +2,7 @@ import { Pool } from 'pg';
 import type { QueryResultRow } from 'pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { addLog } from './utils/tools';
-import { DatasetItemType } from '@/types/plugin';
+import type { DatasetDataItemType } from '@/types/core/dataset/data';
 
 export const connectPg = async (): Promise<Pool> => {
   if (global.pgClient) {
@@ -161,16 +161,16 @@ class Pg {
 export const PgClient = new Pg();
 
 /**
- * data insert kb
+ * data insert dataset
  */
-export const insertKbItem = ({
+export const insertData2Dataset = ({
   userId,
   kbId,
   data
 }: {
   userId: string;
   kbId: string;
-  data: (DatasetItemType & {
+  data: (DatasetDataItemType & {
     vector: number[];
   })[];
 }) => {

client/src/store/dataset.ts

@@ -1,26 +1,26 @@
 import { create } from 'zustand';
 import { devtools, persist } from 'zustand/middleware';
 import { immer } from 'zustand/middleware/immer';
-import { type KbTestItemType } from '@/types/plugin';
-import type { KbItemType, KbListItemType } from '@/types/plugin';
-import { getKbList, getKbById, getAllDataset, putKbById } from '@/api/plugins/kb';
-import { defaultKbDetail } from '@/constants/kb';
-import { KbUpdateParams } from '@/api/request/kb';
+import type { SearchTestItemType } from '@/types/core/dataset';
+import type { DatasetItemType, DatasetsItemType } from '@/types/core/dataset';
+import { getAllDataset, getDatasets, getDatasetById, putDatasetById } from '@/api/core/dataset';
+import { defaultKbDetail } from '@/constants/dataset';
+import type { DatasetUpdateParams } from '@/api/core/dataset/index.d';
 
 type State = {
-  allDatasets: KbListItemType[];
-  loadAllDatasets: () => Promise<KbListItemType[]>;
-  myKbList: KbListItemType[];
+  allDatasets: DatasetsItemType[];
+  loadAllDatasets: () => Promise<DatasetsItemType[]>;
+  myKbList: DatasetsItemType[];
   loadKbList: (parentId?: string) => Promise<any>;
-  setKbList(val: KbListItemType[]): void;
-  kbDetail: KbItemType;
-  getKbDetail: (id: string, init?: boolean) => Promise<KbItemType>;
-  updateDataset: (data: KbUpdateParams) => Promise<any>;
+  setKbList(val: DatasetsItemType[]): void;
+  kbDetail: DatasetItemType;
+  getKbDetail: (id: string, init?: boolean) => Promise<DatasetItemType>;
+  updateDataset: (data: DatasetUpdateParams) => Promise<any>;
 
-  kbTestList: KbTestItemType[];
-  pushKbTestItem: (data: KbTestItemType) => void;
+  kbTestList: SearchTestItemType[];
+  pushKbTestItem: (data: SearchTestItemType) => void;
   delKbTestItemById: (id: string) => void;
-  updateKbItemById: (data: KbTestItemType) => void;
+  updateKbItemById: (data: SearchTestItemType) => void;
 };
 
 export const useDatasetStore = create<State>()(
@@ -37,7 +37,7 @@ export const useDatasetStore = create<State>()(
         },
         myKbList: [],
         async loadKbList(parentId = '') {
-          const res = await getKbList({ parentId });
+          const res = await getDatasets({ parentId });
           set((state) => {
             state.myKbList = res;
           });
@@ -52,7 +52,7 @@ export const useDatasetStore = create<State>()(
         async getKbDetail(id: string, init = false) {
           if (id === get().kbDetail._id && !init) return get().kbDetail;
 
-          const data = await getKbById(id);
+          const data = await getDatasetById(id);
 
           set((state) => {
             state.kbDetail = data;
@@ -80,7 +80,7 @@ export const useDatasetStore = create<State>()(
                 : item
             );
           });
-          await putKbById(data);
+          await putDatasetById(data);
         },
         kbTestList: [],
         pushKbTestItem(data) {
@@ -93,7 +93,7 @@ export const useDatasetStore = create<State>()(
             state.kbTestList = state.kbTestList.filter((item) => item.id !== id);
           });
         },
-        updateKbItemById(data: KbTestItemType) {
+        updateKbItemById(data: SearchTestItemType) {
           set((state) => {
             state.kbTestList = state.kbTestList.map((item) => (item.id === data.id ? data : item));
           });

client/src/types/chat.d.ts

@@ -3,7 +3,7 @@ import type { InitChatResponse, InitShareChatResponse } from '@/api/response/cha
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { ClassifyQuestionAgentItemType } from './app';
 import { ChatItemSchema } from './mongoSchema';
-import { KbDataItemType } from './plugin';
+import type { PgDataItemType } from '@/types/core/dataset/data';
 import { FlowModuleTypeEnum } from '@/constants/flow';
 
 export type ExportChatType = 'md' | 'pdf' | 'html';
@@ -43,7 +43,7 @@ export type ShareChatType = InitShareChatResponse & {
   history: ShareChatHistoryItemType;
 };
 
-export type QuoteItemType = KbDataItemType & {
+export type QuoteItemType = PgDataItemType & {
   kb_id: string;
 };
 

client/src/types/common/file.d.ts

@@ -0,0 +1,8 @@
+export type GSFileInfoType = {
+  id: string;
+  filename: string;
+  size: number;
+  contentType: string;
+  encoding: string;
+  uploadDate: Date;
+};

client/src/types/core/dataset/data.d.ts

@@ -0,0 +1,9 @@
+export type DatasetDataItemType = {
+  q: string; // 提问词
+  a: string; // 原文
+  source?: string;
+  file_id?: string;
+};
+export type PgDataItemType = DatasetItemType & {
+  id: string;
+};

client/src/types/core/dataset/file.d.ts

@@ -0,0 +1,10 @@
+import { FileStatusEnum } from '@/constants/dataset';
+
+export type DatasetFileItemType = {
+  id: string;
+  size: number;
+  filename: string;
+  uploadTime: Date;
+  chunkLength: number;
+  status: `${FileStatusEnum}`;
+};

client/src/types/core/dataset/index.d.ts

@@ -0,0 +1,32 @@
+import { FileStatusEnum } from '@/constants/dataset';
+import { PgDataItemType } from './data';
+import { VectorModelItemType } from '../../model';
+import type { kbSchema } from '../../mongoSchema';
+
+export type DatasetsItemType = Omit<kbSchema, 'vectorModel'> & {
+  vectorModel: VectorModelItemType;
+};
+
+export type DatasetItemType = {
+  _id: string;
+  avatar: string;
+  name: string;
+  userId: string;
+  vectorModel: VectorModelItemType;
+  tags: string;
+};
+
+export type DatasetPathItemType = {
+  parentId: string;
+  parentName: string;
+};
+
+export type SearchTestItemType = {
+  id: string;
+  kbId: string;
+  text: string;
+  time: Date;
+  results: (PgDataItemType & { score: number })[];
+};
+
+export type SelectedDatasetType = { kbId: string; vectorModel: VectorModelItemType }[];

client/src/types/mongoSchema.d.ts

@@ -6,7 +6,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
 import type { AppModuleItemType } from './app';
 import { ChatSourceEnum } from '@/constants/chat';
 import { AppTypeEnum } from '@/constants/app';
-import { KbTypeEnum } from '@/constants/kb';
+import { KbTypeEnum } from '@/constants/dataset';
 
 export interface UserModelSchema {
   _id: string;

client/src/types/plugin.d.ts

@@ -1,65 +1,4 @@
-import { FileStatusEnum } from '@/constants/kb';
-import { VectorModelItemType } from './model';
-import type { kbSchema } from './mongoSchema';
-
-export type SelectedKbType = { kbId: string; vectorModel: VectorModelItemType }[];
-
-export type KbListItemType = Omit<kbSchema, 'vectorModel'> & {
-  vectorModel: VectorModelItemType;
-};
-
-export type KbPathItemType = {
-  parentId: string;
-  parentName: string;
-};
-
-/* kb type */
-export interface KbItemType {
-  _id: string;
-  avatar: string;
-  name: string;
-  userId: string;
-  vectorModel: VectorModelItemType;
-  tags: string;
-}
-
-export type KbFileItemType = {
-  id: string;
-  size: number;
-  filename: string;
-  uploadTime: Date;
-  chunkLength: number;
-  status: `${FileStatusEnum}`;
-};
-
-export type DatasetItemType = {
-  q: string; // 提问词
-  a: string; // 原文
-  source?: string;
-  file_id?: string;
-};
-export type KbDataItemType = DatasetItemType & {
-  id: string;
-};
-
-export type KbTestItemType = {
-  id: string;
-  kbId: string;
-  text: string;
-  time: Date;
-  results: (KbDataItemType & { score: number })[];
-};
-
 export type FetchResultItem = {
   url: string;
   content: string;
 };
-
-export type FileInfo = {
-  id: string;
-  filename: string;
-  size: number;
-  contentType: string;
-  encoding: string;
-  uploadDate: Date;
-};

client/src/utils/app.ts

@@ -7,14 +7,14 @@ import {
   SpecialInputKeyEnum
 } from '@/constants/flow';
 import { SystemInputEnum } from '@/constants/app';
-import type { SelectedKbType } from '@/types/plugin';
+import type { SelectedDatasetType } from '@/types/core/dataset';
 import { FlowInputItemType } from '@/types/flow';
 import type { AIChatProps } from '@/types/core/aiChat';
 
 export type EditFormType = {
   chatModel: AIChatProps;
   kb: {
-    list: SelectedKbType;
+    list: SelectedDatasetType;
     searchSimilarity: number;
     searchLimit: number;
     searchEmptyText: string;

client/src/utils/common/file/index.tsx

@@ -1,7 +0,0 @@
-import { getFileViewUrl } from '@/api/support/file';
-
-export async function getFileAndOpen(fileId: string) {
-  const url = await getFileViewUrl(fileId);
-  const asPath = `${location.origin}${url}`;
-  window.open(asPath, '_blank');
-}

client/src/utils/file.ts

@@ -1,179 +1,6 @@
-import mammoth from 'mammoth';
-import Papa from 'papaparse';
 import { getErrText } from './tools';
-import { uploadImg, postUploadFiles } from '@/api/support/file';
 import { countPromptTokens } from './common/tiktoken';
 
-/**
- * upload file to mongo gridfs
- */
-export const uploadFiles = (
-  files: File[],
-  metadata: Record<string, any> = {},
-  percentListen?: (percent: number) => void
-) => {
-  const form = new FormData();
-  form.append('metadata', JSON.stringify(metadata));
-  files.forEach((file) => {
-    form.append('file', file, encodeURIComponent(file.name));
-  });
-  return postUploadFiles(form, (e) => {
-    if (!e.total) return;
-
-    const percent = Math.round((e.loaded / e.total) * 100);
-    percentListen && percentListen(percent);
-  });
-};
-
-/**
- * 读取 txt 文件内容
- */
-export const readTxtContent = (file: File) => {
-  return new Promise((resolve: (_: string) => void, reject) => {
-    try {
-      const reader = new FileReader();
-      reader.onload = () => {
-        resolve(reader.result as string);
-      };
-      reader.onerror = (err) => {
-        console.log('error txt read:', err);
-        reject('读取 txt 文件失败');
-      };
-      reader.readAsText(file);
-    } catch (error) {
-      reject('浏览器不支持文件内容读取');
-    }
-  });
-};
-
-/**
- * 读取 pdf 内容
- */
-export const readPdfContent = (file: File) =>
-  new Promise<string>((resolve, reject) => {
-    try {
-      const pdfjsLib = window['pdfjs-dist/build/pdf'];
-      pdfjsLib.workerSrc = '/js/pdf.worker.js';
-
-      const readPDFPage = async (doc: any, pageNo: number) => {
-        const page = await doc.getPage(pageNo);
-        const tokenizedText = await page.getTextContent();
-
-        const pageText = tokenizedText.items
-          .map((token: any) => token.str)
-          .filter((item: string) => item)
-          .join('');
-        return pageText;
-      };
-
-      let reader = new FileReader();
-      reader.readAsArrayBuffer(file);
-      reader.onload = async (event) => {
-        if (!event?.target?.result) return reject('解析 PDF 失败');
-        try {
-          const doc = await pdfjsLib.getDocument(event.target.result).promise;
-          const pageTextPromises = [];
-          for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
-            pageTextPromises.push(readPDFPage(doc, pageNo));
-          }
-          const pageTexts = await Promise.all(pageTextPromises);
-          resolve(pageTexts.join('\n'));
-        } catch (err) {
-          console.log(err, 'pdf load error');
-          reject('解析 PDF 失败');
-        }
-      };
-      reader.onerror = (err) => {
-        console.log(err, 'pdf load error');
-        reject('解析 PDF 失败');
-      };
-    } catch (error) {
-      reject('浏览器不支持文件内容读取');
-    }
-  });
-
-/**
- * 读取doc
- */
-export const readDocContent = (file: File) =>
-  new Promise<string>((resolve, reject) => {
-    try {
-      const reader = new FileReader();
-      reader.readAsArrayBuffer(file);
-      reader.onload = async ({ target }) => {
-        if (!target?.result) return reject('读取 doc 文件失败');
-        try {
-          const res = await mammoth.extractRawText({
-            arrayBuffer: target.result as ArrayBuffer
-          });
-          resolve(res?.value);
-        } catch (error) {
-          window.umami?.track('wordReadError', {
-            err: error?.toString()
-          });
-          console.log('error doc read:', error);
-
-          reject('读取 doc 文件失败, 请转换成 PDF');
-        }
-      };
-      reader.onerror = (err) => {
-        window.umami?.track('wordReadError', {
-          err: err?.toString()
-        });
-        console.log('error doc read:', err);
-
-        reject('读取 doc 文件失败');
-      };
-    } catch (error) {
-      reject('浏览器不支持文件内容读取');
-    }
-  });
-
-/**
- * 读取csv
- */
-export const readCsvContent = async (file: File) => {
-  try {
-    const textArr = await readTxtContent(file);
-    const csvArr = Papa.parse(textArr).data as string[][];
-    if (csvArr.length === 0) {
-      throw new Error('csv 解析失败');
-    }
-    return {
-      header: csvArr.shift() as string[],
-      data: csvArr.map((item) => item)
-    };
-  } catch (error) {
-    return Promise.reject('解析 csv 文件失败');
-  }
-};
-
-/**
- * file download
- */
-export const fileDownload = ({
-  text,
-  type,
-  filename
-}: {
-  text: string;
-  type: string;
-  filename: string;
-}) => {
-  // 导出为文件
-  const blob = new Blob([`\uFEFF${text}`], { type: `${type};charset=utf-8;` });
-
-  // 创建下载链接
-  const downloadLink = document.createElement('a');
-  downloadLink.href = window.URL.createObjectURL(blob);
-  downloadLink.download = filename;
-
-  // 添加链接到页面并触发下载
-  document.body.appendChild(downloadLink);
-  downloadLink.click();
-  document.body.removeChild(downloadLink);
-};
-
 /**
  * text split into chunks
  * maxLen - one chunk len. max: 3500
@@ -217,89 +44,6 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
   }
 };
 
-export const fileToBase64 = (file: File) => {
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader();
-    reader.readAsDataURL(file);
-    reader.onload = () => resolve(reader.result);
-    reader.onerror = (error) => reject(error);
-  });
-};
-
-/**
- * compress image. response base64
- * @param maxSize The max size of the compressed image
- */
-export const compressImg = ({
-  file,
-  maxW = 200,
-  maxH = 200,
-  maxSize = 1024 * 100
-}: {
-  file: File;
-  maxW?: number;
-  maxH?: number;
-  maxSize?: number;
-}) =>
-  new Promise<string>((resolve, reject) => {
-    const reader = new FileReader();
-    reader.readAsDataURL(file);
-    reader.onload = async () => {
-      const img = new Image();
-      // @ts-ignore
-      img.src = reader.result;
-      img.onload = async () => {
-        let width = img.width;
-        let height = img.height;
-
-        if (width > height) {
-          if (width > maxW) {
-            height *= maxW / width;
-            width = maxW;
-          }
-        } else {
-          if (height > maxH) {
-            width *= maxH / height;
-            height = maxH;
-          }
-        }
-
-        const canvas = document.createElement('canvas');
-        canvas.width = width;
-        canvas.height = height;
-        const ctx = canvas.getContext('2d');
-
-        if (!ctx) {
-          return reject('压缩图片异常');
-        }
-
-        ctx.drawImage(img, 0, 0, width, height);
-        const compressedDataUrl = canvas.toDataURL(file.type, 0.8);
-        // 移除 canvas 元素
-        canvas.remove();
-
-        if (compressedDataUrl.length > maxSize) {
-          return reject('图片太大了');
-        }
-
-        const src = await (async () => {
-          try {
-            const src = await uploadImg(compressedDataUrl);
-            return src;
-          } catch (error) {
-            return compressedDataUrl;
-          }
-        })();
-
-        resolve(src);
-      };
-    };
-    reader.onerror = (err) => {
-      console.log(err);
-      reject('压缩图片异常');
-    };
-  });
-
 /* simple text, remove chinese space and extra \n */
 export const simpleText = (text: string) => {
   text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');

client/src/utils/tools.ts

@@ -1,44 +1,5 @@
 import crypto from 'crypto';
-import { useToast } from '@/hooks/useToast';
 import dayjs from 'dayjs';
-import { useTranslation } from 'react-i18next';
-
-/**
- * copy text data
- */
-export const useCopyData = () => {
-  const { t } = useTranslation();
-  const { toast } = useToast();
-
-  return {
-    copyData: async (
-      data: string,
-      title: string | null = t('common.Copy Successful'),
-      duration = 1000
-    ) => {
-      try {
-        if (navigator.clipboard) {
-          await navigator.clipboard.writeText(data);
-        } else {
-          throw new Error('');
-        }
-      } catch (error) {
-        const textarea = document.createElement('textarea');
-        textarea.value = data;
-        document.body.appendChild(textarea);
-        textarea.select();
-        document.execCommand('copy');
-        document.body.removeChild(textarea);
-      }
-
-      toast({
-        title,
-        status: 'success',
-        duration
-      });
-    }
-  };
-};
 
 /**
  * 密码加密
@@ -138,35 +99,6 @@ export const formatFileSize = (bytes: number): string => {
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 };
 
-export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
-/**
- * voice broadcast
- */
-export const voiceBroadcast = ({ text }: { text: string }) => {
-  window.speechSynthesis?.cancel();
-  const msg = new SpeechSynthesisUtterance(text);
-  const voices = window.speechSynthesis?.getVoices?.(); // 获取语言包
-  const voice = voices.find((item) => {
-    return item.name === 'Microsoft Yaoyao - Chinese (Simplified, PRC)';
-  });
-  if (voice) {
-    msg.voice = voice;
-  }
-
-  window.speechSynthesis?.speak(msg);
-
-  msg.onerror = (e) => {
-    console.log(e);
-  };
-
-  return {
-    cancel: () => window.speechSynthesis?.cancel()
-  };
-};
-export const cancelBroadcast = () => {
-  window.speechSynthesis?.cancel();
-};
-
 export const getErrText = (err: any, def = '') => {
   const msg: string = typeof err === 'string' ? err : err?.message || def || '';
   msg && console.log('error =>', msg);

client/src/utils/web/core/dataset.ts

@@ -0,0 +1,48 @@
+import { postChunks2Dataset } from '@/api/core/dataset/data';
+import { TrainingModeEnum } from '@/constants/plugin';
+import type { DatasetDataItemType } from '@/types/core/dataset/data';
+import { delay } from '@/utils/tools';
+
+export async function chunksUpload({
+  kbId,
+  mode,
+  chunks,
+  prompt,
+  rate = 200,
+  onUploading
+}: {
+  kbId: string;
+  mode: `${TrainingModeEnum}`;
+  chunks: DatasetDataItemType[];
+  prompt?: string;
+  rate?: number;
+  onUploading?: (insertLen: number, total: number) => void;
+}) {
+  async function upload(data: DatasetDataItemType[]) {
+    return postChunks2Dataset({
+      kbId,
+      data,
+      mode,
+      prompt
+    });
+  }
+
+  let successInsert = 0;
+  let retryTimes = 10;
+  for (let i = 0; i < chunks.length; i += rate) {
+    try {
+      const { insertLen } = await upload(chunks.slice(i, i + rate));
+      onUploading && onUploading(i + rate, chunks.length);
+      successInsert += insertLen;
+    } catch (error) {
+      if (retryTimes === 0) {
+        return Promise.reject(error);
+      }
+      await delay(1000);
+      retryTimes--;
+      i -= rate;
+    }
+  }
+
+  return { insertLen: successInsert };
+}

client/src/utils/web/file.ts

@@ -0,0 +1,261 @@
+import mammoth from 'mammoth';
+import Papa from 'papaparse';
+import { uploadImg, postUploadFiles, getFileViewUrl } from '@/api/support/file';
+/**
+ * upload file to mongo gridfs
+ */
+export const uploadFiles = (
+  files: File[],
+  metadata: Record<string, any> = {},
+  percentListen?: (percent: number) => void
+) => {
+  const form = new FormData();
+  form.append('metadata', JSON.stringify(metadata));
+  files.forEach((file) => {
+    form.append('file', file, encodeURIComponent(file.name));
+  });
+  return postUploadFiles(form, (e) => {
+    if (!e.total) return;
+
+    const percent = Math.round((e.loaded / e.total) * 100);
+    percentListen && percentListen(percent);
+  });
+};
+
+/**
+ * 读取 txt 文件内容
+ */
+export const readTxtContent = (file: File) => {
+  return new Promise((resolve: (_: string) => void, reject) => {
+    try {
+      const reader = new FileReader();
+      reader.onload = () => {
+        resolve(reader.result as string);
+      };
+      reader.onerror = (err) => {
+        console.log('error txt read:', err);
+        reject('读取 txt 文件失败');
+      };
+      reader.readAsText(file);
+    } catch (error) {
+      reject('浏览器不支持文件内容读取');
+    }
+  });
+};
+
+/**
+ * 读取 pdf 内容
+ */
+export const readPdfContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    try {
+      const pdfjsLib = window['pdfjs-dist/build/pdf'];
+      pdfjsLib.workerSrc = '/js/pdf.worker.js';
+
+      const readPDFPage = async (doc: any, pageNo: number) => {
+        const page = await doc.getPage(pageNo);
+        const tokenizedText = await page.getTextContent();
+
+        const pageText = tokenizedText.items
+          .map((token: any) => token.str)
+          .filter((item: string) => item)
+          .join('');
+        return pageText;
+      };
+
+      let reader = new FileReader();
+      reader.readAsArrayBuffer(file);
+      reader.onload = async (event) => {
+        if (!event?.target?.result) return reject('解析 PDF 失败');
+        try {
+          const doc = await pdfjsLib.getDocument(event.target.result).promise;
+          const pageTextPromises = [];
+          for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
+            pageTextPromises.push(readPDFPage(doc, pageNo));
+          }
+          const pageTexts = await Promise.all(pageTextPromises);
+          resolve(pageTexts.join('\n'));
+        } catch (err) {
+          console.log(err, 'pdf load error');
+          reject('解析 PDF 失败');
+        }
+      };
+      reader.onerror = (err) => {
+        console.log(err, 'pdf load error');
+        reject('解析 PDF 失败');
+      };
+    } catch (error) {
+      reject('浏览器不支持文件内容读取');
+    }
+  });
+
+/**
+ * 读取doc
+ */
+export const readDocContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    try {
+      const reader = new FileReader();
+      reader.readAsArrayBuffer(file);
+      reader.onload = async ({ target }) => {
+        if (!target?.result) return reject('读取 doc 文件失败');
+        try {
+          const res = await mammoth.extractRawText({
+            arrayBuffer: target.result as ArrayBuffer
+          });
+          resolve(res?.value);
+        } catch (error) {
+          window.umami?.track('wordReadError', {
+            err: error?.toString()
+          });
+          console.log('error doc read:', error);
+
+          reject('读取 doc 文件失败, 请转换成 PDF');
+        }
+      };
+      reader.onerror = (err) => {
+        window.umami?.track('wordReadError', {
+          err: err?.toString()
+        });
+        console.log('error doc read:', err);
+
+        reject('读取 doc 文件失败');
+      };
+    } catch (error) {
+      reject('浏览器不支持文件内容读取');
+    }
+  });
+
+/**
+ * 读取csv
+ */
+export const readCsvContent = async (file: File) => {
+  try {
+    const textArr = await readTxtContent(file);
+    const csvArr = Papa.parse(textArr).data as string[][];
+    if (csvArr.length === 0) {
+      throw new Error('csv 解析失败');
+    }
+    return {
+      header: csvArr.shift() as string[],
+      data: csvArr.map((item) => item)
+    };
+  } catch (error) {
+    return Promise.reject('解析 csv 文件失败');
+  }
+};
+
+/**
+ * file download
+ */
+export const fileDownload = ({
+  text,
+  type,
+  filename
+}: {
+  text: string;
+  type: string;
+  filename: string;
+}) => {
+  // 导出为文件
+  const blob = new Blob([`\uFEFF${text}`], { type: `${type};charset=utf-8;` });
+
+  // 创建下载链接
+  const downloadLink = document.createElement('a');
+  downloadLink.href = window.URL.createObjectURL(blob);
+  downloadLink.download = filename;
+
+  // 添加链接到页面并触发下载
+  document.body.appendChild(downloadLink);
+  downloadLink.click();
+  document.body.removeChild(downloadLink);
+};
+
+export async function getFileAndOpen(fileId: string) {
+  const url = await getFileViewUrl(fileId);
+  const asPath = `${location.origin}${url}`;
+  window.open(asPath, '_blank');
+}
+
+export const fileToBase64 = (file: File) => {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file);
+    reader.onload = () => resolve(reader.result);
+    reader.onerror = (error) => reject(error);
+  });
+};
+
+/**
+ * compress image. response base64
+ * @param maxSize The max size of the compressed image
+ */
+export const compressImg = ({
+  file,
+  maxW = 200,
+  maxH = 200,
+  maxSize = 1024 * 100
+}: {
+  file: File;
+  maxW?: number;
+  maxH?: number;
+  maxSize?: number;
+}) =>
+  new Promise<string>((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file);
+    reader.onload = async () => {
+      const img = new Image();
+      // @ts-ignore
+      img.src = reader.result;
+      img.onload = async () => {
+        let width = img.width;
+        let height = img.height;
+
+        if (width > height) {
+          if (width > maxW) {
+            height *= maxW / width;
+            width = maxW;
+          }
+        } else {
+          if (height > maxH) {
+            width *= maxH / height;
+            height = maxH;
+          }
+        }
+
+        const canvas = document.createElement('canvas');
+        canvas.width = width;
+        canvas.height = height;
+        const ctx = canvas.getContext('2d');
+
+        if (!ctx) {
+          return reject('压缩图片异常');
+        }
+
+        ctx.drawImage(img, 0, 0, width, height);
+        const compressedDataUrl = canvas.toDataURL(file.type, 0.8);
+        // 移除 canvas 元素
+        canvas.remove();
+
+        if (compressedDataUrl.length > maxSize) {
+          return reject('图片太大了');
+        }
+
+        const src = await (async () => {
+          try {
+            const src = await uploadImg(compressedDataUrl);
+            return src;
+          } catch (error) {
+            return compressedDataUrl;
+          }
+        })();
+
+        resolve(src);
+      };
+    };
+    reader.onerror = (err) => {
+      console.log(err);
+      reject('压缩图片异常');
+    };
+  });

client/src/utils/web/voice.ts

@@ -0,0 +1,28 @@
+export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
+/**
+ * voice broadcast
+ */
+export const voiceBroadcast = ({ text }: { text: string }) => {
+  window.speechSynthesis?.cancel();
+  const msg = new SpeechSynthesisUtterance(text);
+  const voices = window.speechSynthesis?.getVoices?.(); // 获取语言包
+  const voice = voices.find((item) => {
+    return item.name === 'Microsoft Yaoyao - Chinese (Simplified, PRC)';
+  });
+  if (voice) {
+    msg.voice = voice;
+  }
+
+  window.speechSynthesis?.speak(msg);
+
+  msg.onerror = (e) => {
+    console.log(e);
+  };
+
+  return {
+    cancel: () => window.speechSynthesis?.cancel()
+  };
+};
+export const cancelBroadcast = () => {
+  window.speechSynthesis?.cancel();
+};