Change embedding (#1463)

* rebuild embedding queue

* dataset menu

* feat: rebuild data api

* feat: ui change embedding model

* dataset ui

* feat: rebuild index ui

* rename collection

projects/app/src/pages/api/admin/initv481.ts

@@ -0,0 +1,178 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
+import { NextAPI } from '@/service/middle/entry';
+import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
+import { connectionMongo } from '@fastgpt/service/common/mongo';
+import { addLog } from '@fastgpt/service/common/system/log';
+
+/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
+async function handler(req: NextApiRequest, res: NextApiResponse) {
+  await authCert({ req, authRoot: true });
+
+  // 重命名 dataset.trainigns -> dataset_trainings
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'dataset.trainings' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('dataset.trainings');
+      const targetCol = connectionMongo.connection.db.collection('dataset_trainings');
+
+      if ((await targetCol.countDocuments()) > 0) {
+        console.log(
+          'dataset_trainings 中有数据，无法自动将 dataset.trainings 迁移到 dataset_trainings，请手动操作'
+        );
+      } else {
+        await sourceCol.rename('dataset_trainings', { dropTarget: true });
+        console.log('success rename dataset.trainings -> dataset_trainings');
+      }
+    }
+  } catch (error) {
+    console.log('error： rename dataset.trainings -> dataset_trainings', error);
+  }
+
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'dataset.collections' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('dataset.collections');
+      const targetCol = connectionMongo.connection.db.collection('dataset_collections');
+
+      if ((await targetCol.countDocuments()) > 0) {
+        console.log(
+          'dataset_collections 中有数据，无法自动将 dataset.collections 迁移到 dataset_collections，请手动操作'
+        );
+      } else {
+        await sourceCol.rename('dataset_collections', { dropTarget: true });
+        console.log('success rename dataset.collections -> dataset_collections');
+      }
+    }
+  } catch (error) {
+    console.log('error： rename dataset.collections -> dataset_collections', error);
+  }
+
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'dataset.datas' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('dataset.datas');
+      const targetCol = connectionMongo.connection.db.collection('dataset_datas');
+
+      if ((await targetCol.countDocuments()) > 0) {
+        console.log(
+          'dataset_datas 中有数据，无法自动将 dataset.datas 迁移到 dataset_datas，请手动操作'
+        );
+      } else {
+        await sourceCol.rename('dataset_datas', { dropTarget: true });
+        console.log('success rename dataset.datas -> dataset_datas');
+      }
+    }
+  } catch (error) {
+    console.log('error： rename dataset.datas -> dataset_datas', error);
+  }
+
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'app.versions' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('app.versions');
+      const targetCol = connectionMongo.connection.db.collection('app_versions');
+
+      if ((await targetCol.countDocuments()) > 0) {
+        console.log(
+          'app_versions 中有数据，无法自动将 app.versions 迁移到 app_versions，请手动操作'
+        );
+      } else {
+        await sourceCol.rename('app_versions', { dropTarget: true });
+        console.log('success rename app.versions -> app_versions');
+      }
+    }
+  } catch (error) {
+    console.log('error： rename app.versions -> app_versions', error);
+  }
+
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'buffer.rawtexts' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('buffer.rawtexts');
+      const targetCol = connectionMongo.connection.db.collection('buffer_rawtexts');
+
+      if ((await targetCol.countDocuments()) > 0) {
+        console.log(
+          'buffer_rawtexts 中有数据，无法自动将 buffer.rawtexts 迁移到 buffer_rawtexts，请手动操作'
+        );
+      } else {
+        await sourceCol.rename('buffer_rawtexts', { dropTarget: true });
+        console.log('success rename buffer.rawtexts -> buffer_rawtexts');
+      }
+    }
+  } catch (error) {
+    console.log('error： rename buffer.rawtext -> buffer_rawtext', error);
+  }
+
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'buffer.tts' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('buffer.tts');
+      const targetCol = connectionMongo.connection.db.collection('buffer_tts');
+
+      if ((await targetCol.countDocuments()) > 0) {
+        console.log('buffer_tts 中有数据，无法自动将 buffer.tts 迁移到 buffer_tts，请手动操作');
+      } else {
+        await sourceCol.rename('buffer_tts', { dropTarget: true });
+        console.log('success rename buffer.tts -> buffer_tts');
+      }
+    }
+  } catch (error) {
+    console.log('error： rename buffer.tts -> buffer_tts', error);
+  }
+
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'team.members' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('team.members');
+
+      await sourceCol.rename('team_members', { dropTarget: true });
+      console.log('success rename team.members -> team_members');
+    }
+  } catch (error) {
+    console.log('error： rename team.members -> team_members', error);
+  }
+
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'team.tags' })
+      .toArray();
+    if (collections.length > 0) {
+      const sourceCol = connectionMongo.connection.db.collection('team.tags');
+      const targetCol = connectionMongo.connection.db.collection('team_tags');
+
+      if ((await targetCol.countDocuments()) > 0) {
+        console.log('team_tags 中有数据，无法自动将 team.tags 迁移到 team_tags，请手动操作');
+      } else {
+        await sourceCol.rename('team_tags', { dropTarget: true });
+        console.log('success rename team.tags -> team_tags');
+      }
+    }
+  } catch (error) {
+    console.log('error： rename team.tags -> team_tags', error);
+  }
+
+  jsonRes(res, {
+    message: 'success'
+  });
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts

@@ -0,0 +1,39 @@
+import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middle/entry';
+import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+
+type Props = {};
+
+export type getDatasetTrainingQueueResponse = {
+  rebuildingCount: number;
+  trainingCount: number;
+};
+
+async function handler(
+  req: ApiRequestProps<any, { datasetId: string }>,
+  res: ApiResponseType<any>
+): Promise<getDatasetTrainingQueueResponse> {
+  const { datasetId } = req.query;
+
+  const { teamId } = await authDataset({
+    req,
+    authToken: true,
+    authApiKey: true,
+    datasetId,
+    per: 'r'
+  });
+
+  const [rebuildingCount, trainingCount] = await Promise.all([
+    MongoDatasetData.countDocuments({ teamId, datasetId, rebuilding: true }),
+    MongoDatasetTraining.countDocuments({ teamId, datasetId })
+  ]);
+
+  return {
+    rebuildingCount,
+    trainingCount
+  };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts

@@ -0,0 +1,133 @@
+import { NextAPI } from '@/service/middle/entry';
+import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
+import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/controller';
+import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
+import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
+
+export type rebuildEmbeddingBody = {
+  datasetId: string;
+  vectorModel: string;
+};
+
+export type Response = {};
+
+async function handler(
+  req: ApiRequestProps<rebuildEmbeddingBody>,
+  res: ApiResponseType<any>
+): Promise<Response> {
+  const { datasetId, vectorModel } = req.body;
+
+  const { teamId, tmbId, dataset } = await authDataset({
+    req,
+    authToken: true,
+    authApiKey: true,
+    datasetId,
+    per: 'owner'
+  });
+
+  // check vector model
+  if (!vectorModel || dataset.vectorModel === vectorModel) {
+    return Promise.reject('vectorModel 不合法');
+  }
+
+  // check rebuilding or training
+  const [rebuilding, training] = await Promise.all([
+    MongoDatasetData.findOne({ teamId, datasetId, rebuilding: true }),
+    MongoDatasetTraining.findOne({ teamId, datasetId })
+  ]);
+
+  if (rebuilding || training) {
+    return Promise.reject('数据集正在训练或者重建中，请稍后再试');
+  }
+
+  const { billId } = await createTrainingUsage({
+    teamId,
+    tmbId,
+    appName: '切换索引模型',
+    billSource: UsageSourceEnum.training,
+    vectorModel: getVectorModel(dataset.vectorModel)?.name,
+    agentModel: getLLMModel(dataset.agentModel)?.name
+  });
+
+  // update vector model and dataset.data rebuild field
+  await mongoSessionRun(async (session) => {
+    await MongoDataset.findByIdAndUpdate(
+      datasetId,
+      {
+        vectorModel
+      },
+      { session }
+    );
+    await MongoDatasetData.updateMany(
+      {
+        teamId,
+        datasetId
+      },
+      {
+        $set: {
+          rebuilding: true
+        }
+      },
+      {
+        session
+      }
+    );
+  });
+
+  // get 10 init dataset.data
+  const arr = new Array(10).fill(0);
+  for await (const _ of arr) {
+    await mongoSessionRun(async (session) => {
+      const data = await MongoDatasetData.findOneAndUpdate(
+        {
+          teamId,
+          datasetId,
+          rebuilding: true
+        },
+        {
+          $unset: {
+            rebuilding: null
+          },
+          updateTime: new Date()
+        },
+        {
+          session
+        }
+      ).select({
+        _id: 1,
+        collectionId: 1
+      });
+
+      if (data) {
+        await MongoDatasetTraining.create(
+          [
+            {
+              teamId,
+              tmbId,
+              datasetId,
+              collectionId: data.collectionId,
+              billId,
+              mode: TrainingModeEnum.chunk,
+              model: vectorModel,
+              q: '1',
+              dataId: data._id
+            }
+          ],
+          {
+            session
+          }
+        );
+      }
+    });
+  }
+
+  return {};
+}
+
+export default NextAPI(handler);

projects/app/src/pages/dataset/detail/components/Info.tsx

@@ -16,25 +16,47 @@ import PermissionRadio from '@/components/support/permission/Radio';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { useRequest } from '@fastgpt/web/hooks/useRequest';
 import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
-import MySelect from '@fastgpt/web/components/common/MySelect';
 import AIModelSelector from '@/components/Select/AIModelSelector';
+import { postRebuildEmbedding } from '@/web/core/dataset/api';
+import { useI18n } from '@/web/context/I18n';
+import type { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
+import { useContextSelector } from 'use-context-selector';
+import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import MyDivider from '@fastgpt/web/components/common/MyDivider/index';
 
 const Info = ({ datasetId }: { datasetId: string }) => {
   const { t } = useTranslation();
-  const { datasetDetail, loadDatasets, updateDataset } = useDatasetStore();
-  const { getValues, setValue, register, handleSubmit } = useForm<DatasetItemType>({
+  const { datasetT } = useI18n();
+  const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore();
+  const rebuildingCount = useContextSelector(DatasetPageContext, (v) => v.rebuildingCount);
+  const trainingCount = useContextSelector(DatasetPageContext, (v) => v.trainingCount);
+  const refetchDatasetTraining = useContextSelector(
+    DatasetPageContext,
+    (v) => v.refetchDatasetTraining
+  );
+
+  const { setValue, register, handleSubmit, watch } = useForm<DatasetItemType>({
     defaultValues: datasetDetail
   });
+
+  const avatar = watch('avatar');
+  const vectorModel = watch('vectorModel');
+  const agentModel = watch('agentModel');
+  const permission = watch('permission');
+
   const { datasetModelList, vectorModelList } = useSystemStore();
 
   const router = useRouter();
 
-  const [refresh, setRefresh] = useState(false);
-
-  const { openConfirm, ConfirmModal } = useConfirm({
+  const { openConfirm: onOpenConfirmDel, ConfirmModal: ConfirmDelModal } = useConfirm({
     content: t('core.dataset.Delete Confirm'),
     type: 'delete'
   });
+  const { openConfirm: onOpenConfirmRebuild, ConfirmModal: ConfirmRebuildModal } = useConfirm({
+    title: t('common.confirm.Common Tip'),
+    content: datasetT('Confirm to rebuild embedding tip'),
+    type: 'delete'
+  });
 
   const { File, onOpen: onOpenSelectFile } = useSelectFile({
     fileType: '.jpg,.png',
@@ -81,13 +103,27 @@ const Info = ({ datasetId }: { datasetId: string }) => {
     onSuccess(src: string | null) {
       if (src) {
         setValue('avatar', src);
-        setRefresh((state) => !state);
       }
     },
     errorToast: t('common.avatar.Select Failed')
   });
 
-  const btnLoading = useMemo(() => isDeleting || isSaving, [isDeleting, isSaving]);
+  const { mutate: onRebuilding, isLoading: isRebuilding } = useRequest({
+    mutationFn: (vectorModel: VectorModelItemType) => {
+      return postRebuildEmbedding({
+        datasetId,
+        vectorModel: vectorModel.model
+      });
+    },
+    onSuccess() {
+      refetchDatasetTraining();
+      loadDatasetDetail(datasetId, true);
+    },
+    successToast: datasetT('Rebuild embedding start tip'),
+    errorToast: t('common.Update Failed')
+  });
+
+  const btnLoading = isSelecting || isDeleting || isSaving || isRebuilding;
 
   return (
     <Box py={5} px={[5, 10]}>
@@ -97,6 +133,62 @@ const Info = ({ datasetId }: { datasetId: string }) => {
         </Box>
         <Box flex={1}>{datasetDetail._id}</Box>
       </Flex>
+      <Flex mt={8} w={'100%'} alignItems={'center'} flexWrap={'wrap'}>
+        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+          {t('core.ai.model.Vector Model')}
+        </Box>
+        <Box flex={[1, '0 0 300px']}>
+          <AIModelSelector
+            w={'100%'}
+            value={vectorModel.model}
+            disableTip={
+              rebuildingCount > 0 || trainingCount > 0
+                ? datasetT('The knowledge base has indexes that are being trained or being rebuilt')
+                : undefined
+            }
+            list={vectorModelList.map((item) => ({
+              label: item.name,
+              value: item.model
+            }))}
+            onchange={(e) => {
+              const vectorModel = vectorModelList.find((item) => item.model === e);
+              if (!vectorModel) return;
+              onOpenConfirmRebuild(() => {
+                setValue('vectorModel', vectorModel);
+                onRebuilding(vectorModel);
+              })();
+            }}
+          />
+        </Box>
+      </Flex>
+      <Flex mt={8} w={'100%'} alignItems={'center'}>
+        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+          {t('core.Max Token')}
+        </Box>
+        <Box flex={[1, '0 0 300px']}>{vectorModel.maxToken}</Box>
+      </Flex>
+      <Flex mt={6} alignItems={'center'} flexWrap={'wrap'}>
+        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+          {t('core.ai.model.Dataset Agent Model')}
+        </Box>
+        <Box flex={[1, '0 0 300px']}>
+          <AIModelSelector
+            w={'100%'}
+            value={agentModel.model}
+            list={datasetModelList.map((item) => ({
+              label: item.name,
+              value: item.model
+            }))}
+            onchange={(e) => {
+              const agentModel = datasetModelList.find((item) => item.model === e);
+              if (!agentModel) return;
+              setValue('agentModel', agentModel);
+            }}
+          />
+        </Box>
+      </Flex>
+
+      <MyDivider my={4} h={'2px'} maxW={'500px'} />
 
       <Flex mt={5} w={'100%'} alignItems={'center'}>
         <Box flex={['0 0 90px', '0 0 160px']} w={0}>
@@ -106,7 +198,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
           <MyTooltip label={t('common.avatar.Select Avatar')}>
             <Avatar
               m={'auto'}
-              src={getValues('avatar')}
+              src={avatar}
               w={['32px', '40px']}
               h={['32px', '40px']}
               cursor={'pointer'}
@@ -121,40 +213,6 @@ const Info = ({ datasetId }: { datasetId: string }) => {
         </Box>
         <Input flex={[1, '0 0 300px']} maxLength={30} {...register('name')} />
       </Flex>
-      <Flex mt={8} w={'100%'} alignItems={'center'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          {t('core.ai.model.Vector Model')}
-        </Box>
-        <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
-      </Flex>
-      <Flex mt={8} w={'100%'} alignItems={'center'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          {t('core.Max Token')}
-        </Box>
-        <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
-      </Flex>
-      <Flex mt={6} alignItems={'center'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          {t('core.ai.model.Dataset Agent Model')}
-        </Box>
-        <Box flex={[1, '0 0 300px']}>
-          <AIModelSelector
-            w={'100%'}
-            value={getValues('agentModel').model}
-            list={datasetModelList.map((item) => ({
-              label: item.name,
-              value: item.model
-            }))}
-            onchange={(e) => {
-              const agentModel = datasetModelList.find((item) => item.model === e);
-              if (!agentModel) return;
-              setValue('agentModel', agentModel);
-              setRefresh((state) => !state);
-            }}
-          />
-        </Box>
-      </Flex>
-
       <Flex mt={8} alignItems={'center'} w={'100%'}>
         <Box flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</Box>
         <Textarea flex={[1, '0 0 300px']} {...register('intro')} placeholder={t('common.Intro')} />
@@ -166,10 +224,9 @@ const Info = ({ datasetId }: { datasetId: string }) => {
           </Box>
           <Box>
             <PermissionRadio
-              value={getValues('permission')}
+              value={permission}
               onChange={(e) => {
                 setValue('permission', e);
-                setRefresh(!refresh);
               }}
             />
           </Box>
@@ -193,12 +250,14 @@ const Info = ({ datasetId }: { datasetId: string }) => {
             aria-label={''}
             variant={'whiteDanger'}
             size={'smSquare'}
-            onClick={openConfirm(onclickDelete)}
+            onClick={onOpenConfirmDel(onclickDelete)}
           />
         )}
       </Flex>
+
       <File onSelect={onSelectFile} />
-      <ConfirmModal />
+      <ConfirmDelModal />
+      <ConfirmRebuildModal countDown={10} />
     </Box>
   );
 };

projects/app/src/pages/dataset/detail/components/Slider.tsx

@@ -0,0 +1,215 @@
+import React, { useCallback } from 'react';
+import { useTranslation } from 'next-i18next';
+import { useDatasetStore } from '@/web/core/dataset/store/dataset';
+import { useUserStore } from '@/web/support/user/useUserStore';
+import { Box, Flex, IconButton, useTheme, Progress } from '@chakra-ui/react';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import Avatar from '@/components/Avatar';
+import {
+  DatasetStatusEnum,
+  DatasetTypeEnum,
+  DatasetTypeMap
+} from '@fastgpt/global/core/dataset/constants';
+import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
+import SideTabs from '@/components/SideTabs';
+import { useRequest } from '@fastgpt/web/hooks/useRequest';
+import { useRouter } from 'next/router';
+import Tabs from '@/components/Tabs';
+import { useContextSelector } from 'use-context-selector';
+import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import { useI18n } from '@/web/context/I18n';
+
+export enum TabEnum {
+  dataCard = 'dataCard',
+  collectionCard = 'collectionCard',
+  test = 'test',
+  info = 'info',
+  import = 'import'
+}
+
+const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
+  const theme = useTheme();
+  const { t } = useTranslation();
+  const { datasetT } = useI18n();
+  const router = useRouter();
+  const query = router.query;
+  const { datasetDetail, startWebsiteSync } = useDatasetStore();
+  const { userInfo } = useUserStore();
+  const { isPc, setLoading } = useSystemStore();
+  const vectorTrainingMap = useContextSelector(DatasetPageContext, (v) => v.vectorTrainingMap);
+  const agentTrainingMap = useContextSelector(DatasetPageContext, (v) => v.agentTrainingMap);
+  const rebuildingCount = useContextSelector(DatasetPageContext, (v) => v.rebuildingCount);
+
+  const tabList = [
+    {
+      label: t('core.dataset.Collection'),
+      id: TabEnum.collectionCard,
+      icon: 'common/overviewLight'
+    },
+    { label: t('core.dataset.test.Search Test'), id: TabEnum.test, icon: 'kbTest' },
+    ...(userInfo?.team.canWrite && datasetDetail.isOwner
+      ? [{ label: t('common.Config'), id: TabEnum.info, icon: 'common/settingLight' }]
+      : [])
+  ];
+
+  const setCurrentTab = useCallback(
+    (tab: TabEnum) => {
+      router.replace({
+        query: {
+          ...query,
+          currentTab: tab
+        }
+      });
+    },
+    [query, router]
+  );
+
+  const { ConfirmModal: ConfirmSyncModal, openConfirm: openConfirmSync } = useConfirm({
+    type: 'common'
+  });
+  const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({
+    mutationFn: () => {
+      setLoading(true);
+      return startWebsiteSync();
+    },
+    onSettled() {
+      setLoading(false);
+    },
+    errorToast: t('common.Update Failed')
+  });
+
+  return (
+    <>
+      {isPc ? (
+        <Flex
+          flexDirection={'column'}
+          py={4}
+          h={'100%'}
+          flex={'0 0 200px'}
+          borderRight={theme.borders.base}
+        >
+          <Box px={4} borderBottom={'1px'} borderColor={'myGray.200'} pb={4} mb={4}>
+            <Flex mb={4} alignItems={'center'}>
+              <Avatar src={datasetDetail.avatar} w={'34px'} borderRadius={'md'} />
+              <Box ml={2}>
+                <Box fontWeight={'bold'}>{datasetDetail.name}</Box>
+              </Box>
+            </Flex>
+            {DatasetTypeMap[datasetDetail.type] && (
+              <Flex alignItems={'center'} pl={2} justifyContent={'space-between'}>
+                <DatasetTypeTag type={datasetDetail.type} />
+                {datasetDetail.type === DatasetTypeEnum.websiteDataset &&
+                  datasetDetail.status === DatasetStatusEnum.active && (
+                    <MyTooltip label={t('core.dataset.website.Start Sync')}>
+                      <MyIcon
+                        mt={1}
+                        name={'common/refreshLight'}
+                        w={'12px'}
+                        color={'myGray.500'}
+                        cursor={'pointer'}
+                        onClick={() =>
+                          openConfirmSync(
+                            onUpdateDatasetWebsiteConfig,
+                            undefined,
+                            t('core.dataset.website.Confirm Create Tips')
+                          )()
+                        }
+                      />
+                    </MyTooltip>
+                  )}
+              </Flex>
+            )}
+          </Box>
+          <SideTabs
+            px={4}
+            flex={1}
+            mx={'auto'}
+            w={'100%'}
+            list={tabList}
+            activeId={currentTab}
+            onChange={(e: any) => {
+              setCurrentTab(e);
+            }}
+          />
+          <Box px={4}>
+            {rebuildingCount > 0 && (
+              <Box mb={3}>
+                <Box fontSize={'sm'}>
+                  {datasetT('Rebuilding index count', { count: rebuildingCount })}
+                </Box>
+              </Box>
+            )}
+            <Box mb={3}>
+              <Box fontSize={'sm'}>
+                {t('core.dataset.training.Agent queue')}({agentTrainingMap.tip})
+              </Box>
+              <Progress
+                value={100}
+                size={'xs'}
+                colorScheme={agentTrainingMap.colorSchema}
+                borderRadius={'10px'}
+                isAnimated
+                hasStripe
+              />
+            </Box>
+            <Box mb={3}>
+              <Box fontSize={'sm'}>
+                {t('core.dataset.training.Vector queue')}({vectorTrainingMap.tip})
+              </Box>
+              <Progress
+                value={100}
+                size={'xs'}
+                colorScheme={vectorTrainingMap.colorSchema}
+                borderRadius={'10px'}
+                isAnimated
+                hasStripe
+              />
+            </Box>
+          </Box>
+
+          <Flex
+            alignItems={'center'}
+            cursor={'pointer'}
+            py={2}
+            px={3}
+            borderRadius={'md'}
+            _hover={{ bg: 'myGray.100' }}
+            onClick={() => router.replace('/dataset/list')}
+          >
+            <IconButton
+              mr={3}
+              icon={<MyIcon name={'common/backFill'} w={'18px'} color={'primary.500'} />}
+              bg={'white'}
+              boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
+              size={'smSquare'}
+              borderRadius={'50%'}
+              aria-label={''}
+            />
+            {t('core.dataset.All Dataset')}
+          </Flex>
+        </Flex>
+      ) : (
+        <Box mb={3}>
+          <Tabs
+            m={'auto'}
+            w={'260px'}
+            size={isPc ? 'md' : 'sm'}
+            list={tabList.map((item) => ({
+              id: item.id,
+              label: item.label
+            }))}
+            activeId={currentTab}
+            onChange={(e: any) => setCurrentTab(e)}
+          />
+        </Box>
+      )}
+
+      <ConfirmSyncModal />
+    </>
+  );
+};
+
+export default Slider;

projects/app/src/pages/dataset/detail/index.tsx

@@ -1,33 +1,22 @@
 import React, { useCallback, useMemo } from 'react';
 import { useRouter } from 'next/router';
-import { Box, Flex, IconButton, useTheme, Progress } from '@chakra-ui/react';
+import { Box } from '@chakra-ui/react';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import { useQuery } from '@tanstack/react-query';
 import { getErrText } from '@fastgpt/global/common/error/utils';
-import { useSystemStore } from '@/web/common/system/useSystemStore';
-import Tabs from '@/components/Tabs';
 import dynamic from 'next/dynamic';
-import MyIcon from '@fastgpt/web/components/common/Icon';
-import SideTabs from '@/components/SideTabs';
 import PageContainer from '@/components/PageContainer';
-import Avatar from '@/components/Avatar';
 import { serviceSideProps } from '@/web/common/utils/i18n';
 import { useTranslation } from 'next-i18next';
-import { getTrainingQueueLen } from '@/web/core/dataset/api';
-import MyTooltip from '@/components/MyTooltip';
+
 import CollectionCard from './components/CollectionCard';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-import { useUserStore } from '@/web/support/user/useUserStore';
-import {
-  DatasetStatusEnum,
-  DatasetTypeEnum,
-  DatasetTypeMap
-} from '@fastgpt/global/core/dataset/constants';
+
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
-import { useRequest } from '@fastgpt/web/hooks/useRequest';
-import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag';
 import Head from 'next/head';
+import Slider from './components/Slider';
 import MyBox from '@fastgpt/web/components/common/MyBox';
+import { DatasetPageContextProvider } from '@/web/core/dataset/context/datasetPageContext';
 
 const DataCard = dynamic(() => import('./components/DataCard'));
 const Test = dynamic(() => import('./components/Test'));
@@ -42,48 +31,16 @@ export enum TabEnum {
   import = 'import'
 }
 
-const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${TabEnum}` }) => {
-  const theme = useTheme();
+const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: TabEnum }) => {
   const { t } = useTranslation();
   const { toast } = useToast();
   const router = useRouter();
-  const { isPc } = useSystemStore();
-  const { datasetDetail, loadDatasetDetail, startWebsiteSync } = useDatasetStore();
-  const { userInfo } = useUserStore();
-
-  const tabList = [
-    {
-      label: t('core.dataset.Collection'),
-      id: TabEnum.collectionCard,
-      icon: 'common/overviewLight'
-    },
-    { label: t('core.dataset.test.Search Test'), id: TabEnum.test, icon: 'kbTest' },
-    ...(userInfo?.team.canWrite && datasetDetail.isOwner
-      ? [{ label: t('common.Config'), id: TabEnum.info, icon: 'common/settingLight' }]
-      : [])
-  ];
+  const { datasetDetail, loadDatasetDetail } = useDatasetStore();
 
   const { ConfirmModal: ConfirmSyncModal, openConfirm: openConfirmSync } = useConfirm({
     type: 'common'
   });
 
-  const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({
-    mutationFn: () => startWebsiteSync(),
-    errorToast: t('common.Update Failed')
-  });
-
-  const setCurrentTab = useCallback(
-    (tab: `${TabEnum}`) => {
-      router.replace({
-        query: {
-          datasetId,
-          currentTab: tab
-        }
-      });
-    },
-    [datasetId, router]
-  );
-
   useQuery([datasetId], () => loadDatasetDetail(datasetId), {
     onError(err: any) {
       router.replace(`/dataset/list`);
@@ -94,197 +51,33 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
     }
   });
 
-  const { data: { vectorTrainingCount = 0, agentTrainingCount = 0 } = {} } = useQuery(
-    ['getTrainingQueueLen'],
-    () =>
-      getTrainingQueueLen({
-        vectorModel: datasetDetail.vectorModel.model,
-        agentModel: datasetDetail.agentModel.model
-      }),
-    {
-      refetchInterval: 10000
-    }
-  );
-  const { vectorTrainingMap, agentTrainingMap } = useMemo(() => {
-    const vectorTrainingMap = (() => {
-      if (vectorTrainingCount < 1000)
-        return {
-          colorSchema: 'green',
-          tip: t('core.dataset.training.Leisure')
-        };
-      if (vectorTrainingCount < 10000)
-        return {
-          colorSchema: 'yellow',
-          tip: t('core.dataset.training.Waiting')
-        };
-      return {
-        colorSchema: 'red',
-        tip: t('core.dataset.training.Full')
-      };
-    })();
-    const agentTrainingMap = (() => {
-      if (agentTrainingCount < 100)
-        return {
-          colorSchema: 'green',
-          tip: t('core.dataset.training.Leisure')
-        };
-      if (agentTrainingCount < 1000)
-        return {
-          colorSchema: 'yellow',
-          tip: t('core.dataset.training.Waiting')
-        };
-      return {
-        colorSchema: 'red',
-        tip: t('core.dataset.training.Full')
-      };
-    })();
-    return {
-      vectorTrainingMap,
-      agentTrainingMap
-    };
-  }, [agentTrainingCount, t, vectorTrainingCount]);
-
   return (
     <>
       <Head>
         <title>{datasetDetail?.name}</title>
       </Head>
-      <PageContainer>
-        <MyBox
-          isLoading={isUpdating}
-          display={'flex'}
-          flexDirection={['column', 'row']}
-          h={'100%'}
-          pt={[4, 0]}
-        >
-          {isPc ? (
-            <Flex
-              flexDirection={'column'}
-              py={4}
-              h={'100%'}
-              flex={'0 0 200px'}
-              borderRight={theme.borders.base}
-            >
-              <Box px={4} borderBottom={'1px'} borderColor={'myGray.200'} pb={4} mb={4}>
-                <Flex mb={4} alignItems={'center'}>
-                  <Avatar src={datasetDetail.avatar} w={'34px'} borderRadius={'md'} />
-                  <Box ml={2}>
-                    <Box fontWeight={'bold'}>{datasetDetail.name}</Box>
-                  </Box>
-                </Flex>
-                {DatasetTypeMap[datasetDetail.type] && (
-                  <Flex alignItems={'center'} pl={2} justifyContent={'space-between'}>
-                    <DatasetTypeTag type={datasetDetail.type} />
-                    {datasetDetail.type === DatasetTypeEnum.websiteDataset &&
-                      datasetDetail.status === DatasetStatusEnum.active && (
-                        <MyTooltip label={t('core.dataset.website.Start Sync')}>
-                          <MyIcon
-                            mt={1}
-                            name={'common/refreshLight'}
-                            w={'12px'}
-                            color={'myGray.500'}
-                            cursor={'pointer'}
-                            onClick={() =>
-                              openConfirmSync(
-                                onUpdateDatasetWebsiteConfig,
-                                undefined,
-                                t('core.dataset.website.Confirm Create Tips')
-                              )()
-                            }
-                          />
-                        </MyTooltip>
-                      )}
-                  </Flex>
-                )}
-              </Box>
-              <SideTabs
-                px={4}
-                flex={1}
-                mx={'auto'}
-                w={'100%'}
-                list={tabList}
-                activeId={currentTab}
-                onChange={(e: any) => {
-                  setCurrentTab(e);
-                }}
-              />
-              <Box px={4}>
-                <Box mb={3}>
-                  <Box fontSize={'sm'}>
-                    {t('core.dataset.training.Agent queue')}({agentTrainingMap.tip})
-                  </Box>
-                  <Progress
-                    value={100}
-                    size={'xs'}
-                    colorScheme={agentTrainingMap.colorSchema}
-                    borderRadius={'10px'}
-                    isAnimated
-                    hasStripe
-                  />
-                </Box>
-                <Box mb={3}>
-                  <Box fontSize={'sm'}>
-                    {t('core.dataset.training.Vector queue')}({vectorTrainingMap.tip})
-                  </Box>
-                  <Progress
-                    value={100}
-                    size={'xs'}
-                    colorScheme={vectorTrainingMap.colorSchema}
-                    borderRadius={'10px'}
-                    isAnimated
-                    hasStripe
-                  />
-                </Box>
-              </Box>
+      <DatasetPageContextProvider
+        value={{
+          datasetId
+        }}
+      >
+        <PageContainer>
+          <MyBox display={'flex'} flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
+            <Slider currentTab={currentTab} />
 
-              <Flex
-                alignItems={'center'}
-                cursor={'pointer'}
-                py={2}
-                px={3}
-                borderRadius={'md'}
-                _hover={{ bg: 'myGray.100' }}
-                onClick={() => router.replace('/dataset/list')}
-              >
-                <IconButton
-                  mr={3}
-                  icon={<MyIcon name={'common/backFill'} w={'18px'} color={'primary.500'} />}
-                  bg={'white'}
-                  boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
-                  size={'smSquare'}
-                  borderRadius={'50%'}
-                  aria-label={''}
-                />
-                {t('core.dataset.All Dataset')}
-              </Flex>
-            </Flex>
-          ) : (
-            <Box mb={3}>
-              <Tabs
-                m={'auto'}
-                w={'260px'}
-                size={isPc ? 'md' : 'sm'}
-                list={tabList.map((item) => ({
-                  id: item.id,
-                  label: item.label
-                }))}
-                activeId={currentTab}
-                onChange={(e: any) => setCurrentTab(e)}
-              />
-            </Box>
-          )}
+            {!!datasetDetail._id && (
+              <Box flex={'1 0 0'} pb={0}>
+                {currentTab === TabEnum.collectionCard && <CollectionCard />}
+                {currentTab === TabEnum.dataCard && <DataCard />}
+                {currentTab === TabEnum.test && <Test datasetId={datasetId} />}
+                {currentTab === TabEnum.info && <Info datasetId={datasetId} />}
+                {currentTab === TabEnum.import && <Import />}
+              </Box>
+            )}
+          </MyBox>
+        </PageContainer>
+      </DatasetPageContextProvider>
 
-          {!!datasetDetail._id && (
-            <Box flex={'1 0 0'} pb={0}>
-              {currentTab === TabEnum.collectionCard && <CollectionCard />}
-              {currentTab === TabEnum.dataCard && <DataCard />}
-              {currentTab === TabEnum.test && <Test datasetId={datasetId} />}
-              {currentTab === TabEnum.info && <Info datasetId={datasetId} />}
-              {currentTab === TabEnum.import && <Import />}
-            </Box>
-          )}
-        </MyBox>
-      </PageContainer>
       <ConfirmSyncModal />
     </>
   );
@@ -295,7 +88,7 @@ export async function getServerSideProps(context: any) {
   const datasetId = context?.query?.datasetId;
 
   return {
-    props: { currentTab, datasetId, ...(await serviceSideProps(context, ['file'])) }
+    props: { currentTab, datasetId, ...(await serviceSideProps(context, ['dataset', 'file'])) }
   };
 }
 

projects/app/src/pages/dataset/list/index.tsx

@@ -305,6 +305,42 @@ const Kb = () => {
                     </Box>
                   }
                   menuList={[
+                    {
+                      label: (
+                        <Flex alignItems={'center'}>
+                          <MyIcon name={'edit'} w={'14px'} mr={2} />
+                          {t('Rename')}
+                        </Flex>
+                      ),
+                      onClick: () =>
+                        onOpenTitleModal({
+                          defaultVal: dataset.name,
+                          onSuccess: (val) => {
+                            if (val === dataset.name || !val) return;
+                            updateDataset({ id: dataset._id, name: val });
+                          }
+                        })
+                    },
+                    {
+                      label: (
+                        <Flex alignItems={'center'}>
+                          <MyIcon name={'common/file/move'} w={'14px'} mr={2} />
+                          {t('Move')}
+                        </Flex>
+                      ),
+                      onClick: () => setMoveDataId(dataset._id)
+                    },
+                    {
+                      label: (
+                        <Flex alignItems={'center'}>
+                          <MyIcon name={'export'} w={'14px'} mr={2} />
+                          {t('Export')}
+                        </Flex>
+                      ),
+                      onClick: () => {
+                        exportDataset(dataset);
+                      }
+                    },
                     ...(dataset.permission === PermissionTypeEnum.private
                       ? [
                           {
@@ -342,42 +378,6 @@ const Kb = () => {
                             }
                           }
                         ]),
-                    {
-                      label: (
-                        <Flex alignItems={'center'}>
-                          <MyIcon name={'edit'} w={'14px'} mr={2} />
-                          {t('Rename')}
-                        </Flex>
-                      ),
-                      onClick: () =>
-                        onOpenTitleModal({
-                          defaultVal: dataset.name,
-                          onSuccess: (val) => {
-                            if (val === dataset.name || !val) return;
-                            updateDataset({ id: dataset._id, name: val });
-                          }
-                        })
-                    },
-                    {
-                      label: (
-                        <Flex alignItems={'center'}>
-                          <MyIcon name={'common/file/move'} w={'14px'} mr={2} />
-                          {t('Move')}
-                        </Flex>
-                      ),
-                      onClick: () => setMoveDataId(dataset._id)
-                    },
-                    {
-                      label: (
-                        <Flex alignItems={'center'}>
-                          <MyIcon name={'export'} w={'14px'} mr={2} />
-                          {t('Export')}
-                        </Flex>
-                      ),
-                      onClick: () => {
-                        exportDataset(dataset);
-                      }
-                    },
                     {
                       label: (
                         <Flex alignItems={'center'}>