feat: Sync collection (#3368)

* feat: sync collection * feat: sync collection * perf: website selector * update doc
2025-07-23 05:12:39 +00:00 · 2024-12-11 15:03:41 +08:00
parent 048f5a2d53
commit d5752ddbaa
40 changed files with 365 additions and 191 deletions
--- a/projects/app/src/components/Select/AIModelSelector.tsx
+++ b/projects/app/src/components/Select/AIModelSelector.tsx
@@ -29,6 +29,16 @@ const AIModelSelector = ({ list, onchange, disableTip, ...props }: Props) => {
    onOpen: onOpenAiPointsModal
  } = useDisclosure();

+  const avatarSize = useMemo(() => {
+    const size = {
+      sm: '1rem',
+      md: '1.2rem',
+      lg: '1.4rem'
+    };
+    //@ts-ignore
+    return props.size ? size[props.size] : size['md'];
+  }, [props.size]);
+
  const avatarList = list.map((item) => {
    const modelData =
      llmModelList.find((model) => model.model === item.value) ||
@@ -43,7 +53,7 @@ const AIModelSelector = ({ list, onchange, disableTip, ...props }: Props) => {
            mr={2}
            src={modelData?.avatar || HUGGING_FACE_ICON}
            fallbackSrc={HUGGING_FACE_ICON}
-            w={'18px'}
+            w={avatarSize}
          />
          <Box>{item.label}</Box>
        </Flex>
@@ -56,14 +66,14 @@ const AIModelSelector = ({ list, onchange, disableTip, ...props }: Props) => {
      ? avatarList.concat({
          label: (
            <Flex alignItems={'center'}>
-              <Avatar borderRadius={'0'} mr={2} src={LOGO_ICON} w={'18px'} />
+              <Avatar borderRadius={'0'} mr={2} src={LOGO_ICON} w={avatarSize} />
              <Box>{t('common:support.user.Price')}</Box>
            </Flex>
          ),
          value: 'price'
        })
      : avatarList;
-  }, [feConfigs.show_pay, avatarList, t]);
+  }, [feConfigs.show_pay, avatarList, avatarSize, t]);

  const onSelect = useCallback(
    (e: string) => {
@@ -73,7 +83,7 @@ const AIModelSelector = ({ list, onchange, disableTip, ...props }: Props) => {
      }
      return onchange?.(e);
    },
-    [onchange, router]
+    [onOpenAiPointsModal, onchange]
  );

  return (
--- a/projects/app/src/pages/api/core/dataset/update.ts
+++ b/projects/app/src/pages/api/core/dataset/update.ts
@@ -9,7 +9,11 @@ import {
 } from '@fastgpt/global/support/permission/constant';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
-import { DatasetTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import {
+  DatasetCollectionTypeEnum,
+  DatasetTypeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constants';
 import { ClientSession } from 'mongoose';
 import { parseParentIdInMongo } from '@fastgpt/global/common/parentFolder/utils';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
@@ -22,6 +26,8 @@ import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
 import { TeamWritePermissionVal } from '@fastgpt/global/support/permission/user/constant';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { addDays } from 'date-fns';

 export type DatasetUpdateQuery = {};
 export type DatasetUpdateResponse = any;
@@ -51,7 +57,8 @@ async function handler(
    websiteConfig,
    externalReadUrl,
    apiServer,
-    status
+    status,
+    autoSync
  } = req.body;

  if (!id) {
@@ -101,7 +108,7 @@ async function handler(
    agentModel: agentModel?.model
  });

-  const onUpdate = async (session?: ClientSession) => {
+  const onUpdate = async (session: ClientSession) => {
    await MongoDataset.findByIdAndUpdate(
      id,
      {
@@ -117,14 +124,21 @@ async function handler(
        ...(!!apiServer?.authorization && {
          'apiServer.authorization': apiServer.authorization
        }),
-        ...(isMove && { inheritPermission: true })
+        ...(isMove && { inheritPermission: true }),
+        ...(typeof autoSync === 'boolean' && { autoSync })
      },
      { session }
    );
+    await updateSyncSchedule({
+      teamId: dataset.teamId,
+      datasetId: dataset._id,
+      autoSync,
+      session
+    });
  };

-  if (isMove) {
-    await mongoSessionRun(async (session) => {
+  await mongoSessionRun(async (session) => {
+    if (isMove) {
      if (isFolder && dataset.inheritPermission) {
        const parentClbsAndGroups = await getResourceClbsAndGroups({
          teamId: dataset.teamId,
@@ -149,17 +163,16 @@ async function handler(
          collaborators: parentClbsAndGroups,
          session
        });
-        return onUpdate(session);
      }
      return onUpdate(session);
-    });
-  } else {
-    return onUpdate();
-  }
+    } else {
+      return onUpdate(session);
+    }
+  });
 }
 export default NextAPI(handler);

-async function updateTraining({
+const updateTraining = async ({
  teamId,
  datasetId,
  agentModel
@@ -167,7 +180,7 @@ async function updateTraining({
  teamId: string;
  datasetId: string;
  agentModel?: string;
-}) {
+}) => {
  if (!agentModel) return;

  await MongoDatasetTraining.updateMany(
@@ -184,4 +197,48 @@ async function updateTraining({
      }
    }
  );
-}
+};
+
+const updateSyncSchedule = async ({
+  teamId,
+  datasetId,
+  autoSync,
+  session
+}: {
+  teamId: string;
+  datasetId: string;
+  autoSync?: boolean;
+  session: ClientSession;
+}) => {
+  if (typeof autoSync !== 'boolean') return;
+
+  // Update all collection nextSyncTime
+  if (autoSync) {
+    await MongoDatasetCollection.updateMany(
+      {
+        teamId,
+        datasetId,
+        type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
+      },
+      {
+        $set: {
+          nextSyncTime: addDays(new Date(), 1)
+        }
+      },
+      { session }
+    );
+  } else {
+    await MongoDatasetCollection.updateMany(
+      {
+        teamId,
+        datasetId
+      },
+      {
+        $unset: {
+          nextSyncTime: 1
+        }
+      },
+      { session }
+    );
+  }
+};
--- a/projects/app/src/pages/dataset/detail/components/Info/index.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Info/index.tsx
@@ -1,5 +1,5 @@
 import React, { useEffect, useState } from 'react';
-import { Box, Flex, Input } from '@chakra-ui/react';
+import { Box, Flex, Switch, Input } from '@chakra-ui/react';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
 import { useForm } from 'react-hook-form';
@@ -33,6 +33,8 @@ import EditAPIDatasetInfoModal, {
  EditAPIDatasetInfoFormType
 } from './components/EditApiServiceModal';
 import { EditResourceInfoFormType } from '@/components/common/Modal/EditResourceModal';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+
 const EditResourceModal = dynamic(() => import('@/components/common/Modal/EditResourceModal'));

 const Info = ({ datasetId }: { datasetId: string }) => {
@@ -52,7 +54,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
  const vectorModel = watch('vectorModel');
  const agentModel = watch('agentModel');

-  const { datasetModelList, vectorModelList } = useSystemStore();
+  const { feConfigs, datasetModelList, vectorModelList } = useSystemStore();
  const { ConfirmModal: ConfirmDelModal } = useConfirm({
    content: t('common:core.dataset.Delete Confirm'),
    type: 'delete'
@@ -62,6 +64,10 @@ const Info = ({ datasetId }: { datasetId: string }) => {
    content: t('dataset:confirm_to_rebuild_embedding_tip'),
    type: 'delete'
  });
+  const { openConfirm: onOpenConfirmSyncSchedule, ConfirmModal: ConfirmSyncScheduleModal } =
+    useConfirm({
+      title: t('common:common.confirm.Common Tip')
+    });

  const { File } = useSelectFile({
    fileType: '.jpg,.png',
@@ -132,6 +138,8 @@ const Info = ({ datasetId }: { datasetId: string }) => {
    reset(datasetDetail);
  }, [datasetDetail, datasetDetail._id, reset]);

+  const isTraining = rebuildingCount > 0 || trainingCount > 0;
+
  return (
    <Box w={'100%'} h={'100%'} p={6}>
      <Box>
@@ -177,7 +185,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {

      <MyDivider my={4} h={'2px'} maxW={'500px'} />

-      <Box overflow={'hidden'}>
+      <Box>
        <Flex w={'100%'} flexDir={'column'}>
          <FormLabel fontSize={'mini'} fontWeight={'500'}>
            {t('common:core.dataset.Dataset ID')}
@@ -186,16 +194,23 @@ const Info = ({ datasetId }: { datasetId: string }) => {
        </Flex>

        <Box mt={5} w={'100%'}>
-          <FormLabel fontSize={'mini'} fontWeight={'500'}>
-            {t('common:core.ai.model.Vector Model')}
-          </FormLabel>
+          <Flex alignItems={'center'} fontSize={'mini'}>
+            <FormLabel fontWeight={'500'} flex={'1 0 0'}>
+              {t('common:core.ai.model.Vector Model')}
+            </FormLabel>
+            <MyTooltip label={t('dataset:vector_model_max_tokens_tip')}>
+              <Box>
+                {t('dataset:chunk_max_tokens')}: {vectorModel.maxToken}
+              </Box>
+            </MyTooltip>
+          </Flex>
          <Box pt={2}>
            <AIModelSelector
              w={'100%'}
              value={vectorModel.model}
              fontSize={'mini'}
              disableTip={
-                rebuildingCount > 0 || trainingCount > 0
+                isTraining
                  ? t(
                      'dataset:the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt'
                    )
@@ -217,13 +232,6 @@ const Info = ({ datasetId }: { datasetId: string }) => {
          </Box>
        </Box>

-        <Flex mt={2} w={'100%'} alignItems={'center'}>
-          <FormLabel flex={1} fontSize={'mini'} w={0} fontWeight={'500'}>
-            {t('common:core.Max Token')}
-          </FormLabel>
-          <Box fontSize={'mini'}>{vectorModel.maxToken}</Box>
-        </Flex>
-
        <Box pt={5}>
          <FormLabel fontSize={'mini'} fontWeight={'500'}>
            {t('common:core.ai.model.Dataset Agent Model')}
@@ -247,7 +255,34 @@ const Info = ({ datasetId }: { datasetId: string }) => {
          </Box>
        </Box>

-        {/* <MyDivider my={4} h={'2px'} maxW={'500px'} /> */}
+        {feConfigs?.isPlus && (
+          <Flex alignItems={'center'} pt={5}>
+            <FormLabel fontSize={'mini'} fontWeight={'500'}>
+              {t('dataset:sync_schedule')}
+            </FormLabel>
+            <QuestionTip ml={1} label={t('dataset:sync_schedule_tip')} />
+            <Box flex={1} />
+            <Switch
+              isChecked={!!datasetDetail.autoSync}
+              onChange={(e) => {
+                e.preventDefault();
+                const autoSync = e.target.checked;
+                const text = autoSync ? t('dataset:open_auto_sync') : t('dataset:close_auto_sync');
+
+                onOpenConfirmSyncSchedule(
+                  async () => {
+                    return updateDataset({
+                      id: datasetId,
+                      autoSync
+                    });
+                  },
+                  undefined,
+                  text
+                )();
+              }}
+            />
+          </Flex>
+        )}

        {datasetDetail.type === DatasetTypeEnum.externalFile && (
          <>
@@ -330,6 +365,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
      <File onSelect={onSelectFile} />
      <ConfirmDelModal />
      <ConfirmRebuildModal countDown={10} />
+      <ConfirmSyncScheduleModal />
      {editedDataset && (
        <EditResourceModal
          {...editedDataset}
--- a/projects/app/src/pages/dataset/detail/components/MetaDataCard.tsx
+++ b/projects/app/src/pages/dataset/detail/components/MetaDataCard.tsx
@@ -38,8 +38,7 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
  const metadataList = useMemo<{ label?: string; value?: any }[]>(() => {
    if (!collection) return [];

-    const webSelector =
-      collection?.datasetId?.websiteConfig?.selector || collection?.metadata?.webPageSelector;
+    const webSelector = collection?.metadata?.webPageSelector;

    return [
      {