4.6.3-website dataset (#532)

projects/app/src/pages/dataset/detail/components/CollectionCard.tsx

@@ -11,14 +11,18 @@ import {
   Tbody,
   Image,
   MenuButton,
-  useDisclosure
+  useDisclosure,
+  Button,
+  Link,
+  useTheme
 } from '@chakra-ui/react';
 import {
   getDatasetCollections,
   delDatasetCollectionById,
   putDatasetCollectionById,
   postDatasetCollection,
-  getDatasetCollectionPathById
+  getDatasetCollectionPathById,
+  postWebsiteSync
 } from '@/web/core/dataset/api';
 import { useQuery } from '@tanstack/react-query';
 import { debounce } from 'lodash';
@@ -39,7 +43,10 @@ import EmptyTip from '@/components/EmptyTip';
 import {
   FolderAvatarSrc,
   DatasetCollectionTypeEnum,
-  TrainingModeEnum
+  DatasetCollectionTrainingModeEnum,
+  DatasetTypeEnum,
+  DatasetTypeMap,
+  DatasetStatusEnum
 } from '@fastgpt/global/core/dataset/constant';
 import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
 import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
@@ -52,13 +59,18 @@ import { useToast } from '@/web/common/hooks/useToast';
 import MyTooltip from '@/components/MyTooltip';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
+import { useDatasetStore } from '@/web/core/dataset/store/dataset';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
 
 const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
+const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {});
 
 const CollectionCard = () => {
   const BoxRef = useRef<HTMLDivElement>(null);
   const lastSearch = useRef('');
   const router = useRouter();
+  const theme = useTheme();
   const { toast } = useToast();
   const { parentId = '', datasetId } = router.query as { parentId: string; datasetId: string };
   const { t } = useTranslation();
@@ -66,7 +78,7 @@ const CollectionCard = () => {
   const { isPc } = useSystemStore();
   const { userInfo } = useUserStore();
   const [searchText, setSearchText] = useState('');
-  const { setLoading } = useSystemStore();
+  const { datasetDetail, updateDataset, loadDatasetDetail } = useDatasetStore();
 
   const { openConfirm, ConfirmModal } = useConfirm({
     content: t('dataset.Confirm to delete the file')
@@ -76,11 +88,18 @@ const CollectionCard = () => {
     onOpen: onOpenFileImportModal,
     onClose: onCloseFileImportModal
   } = useDisclosure();
+  const {
+    isOpen: isOpenWebsiteModal,
+    onOpen: onOpenWebsiteModal,
+    onClose: onCloseWebsiteModal
+  } = useDisclosure();
   const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
     useEditTitle({
       title: t('dataset.Create Virtual File'),
-      tip: t('dataset.Virtual File Tip')
+      tip: t('dataset.Virtual File Tip'),
+      canEmpty: false
     });
+
   const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
     title: t('Rename')
   });
@@ -128,48 +147,60 @@ const CollectionCard = () => {
     () =>
       collections.map((collection) => {
         const icon = getCollectionIcon(collection.type, collection.name);
+        const status = (() => {
+          if (collection.trainingAmount > 0) {
+            return {
+              statusText: t('dataset.collections.Collection Embedding', {
+                total: collection.trainingAmount
+              }),
+              color: 'myGray.500'
+            };
+          }
+          return {
+            statusText: t('core.dataset.collection.status.active'),
+            color: 'green.500'
+          };
+        })();
 
         return {
           ...collection,
           icon,
-          ...(collection.trainingAmount > 0
-            ? {
-                statusText: t('dataset.collections.Collection Embedding', {
-                  total: collection.trainingAmount
-                }),
-                color: 'myGray.500'
-              }
-            : {
-                statusText: t('dataset.collections.Ready'),
-                color: 'green.500'
-              })
+          ...status
         };
       }),
     [collections, t]
   );
-  const hasTrainingData = useMemo(
-    () => !!formatCollections.find((item) => item.trainingAmount > 0),
-    [formatCollections]
-  );
 
-  const { mutate: onCreateVirtualFile } = useRequest({
-    mutationFn: ({ name }: { name: string }) => {
-      setLoading(true);
-      return postDatasetCollection({
+  const { mutate: onCreateCollection, isLoading: isCreating } = useRequest({
+    mutationFn: async ({
+      name,
+      type,
+      callback,
+      ...props
+    }: {
+      name: string;
+      type: `${DatasetCollectionTypeEnum}`;
+      callback?: (id: string) => void;
+      trainingType?: `${DatasetCollectionTrainingModeEnum}`;
+      rawLink?: string;
+      chunkSize?: number;
+    }) => {
+      const id = await postDatasetCollection({
         parentId,
         datasetId,
         name,
-        type: DatasetCollectionTypeEnum.virtual
+        type,
+        ...props
       });
+      callback?.(id);
+      return id;
     },
     onSuccess() {
       getData(pageNum);
     },
-    onSettled() {
-      setLoading(false);
-    },
-    successToast: t('dataset.collections.Create Virtual File Success'),
-    errorToast: t('common.Create Virtual File Failed')
+
+    successToast: t('common.Create Success'),
+    errorToast: t('common.Create Failed')
   });
   const { mutate: onUpdateCollectionName } = useRequest({
     mutationFn: ({ collectionId, name }: { collectionId: string; name: string }) => {
@@ -185,9 +216,8 @@ const CollectionCard = () => {
     successToast: t('common.Rename Success'),
     errorToast: t('common.Rename Failed')
   });
-  const { mutate: onDelCollection } = useRequest({
+  const { mutate: onDelCollection, isLoading: isDeleting } = useRequest({
     mutationFn: (collectionId: string) => {
-      setLoading(true);
       return delDatasetCollectionById({
         collectionId
       });
@@ -195,26 +225,54 @@ const CollectionCard = () => {
     onSuccess() {
       getData(pageNum);
     },
-    onSettled() {
-      setLoading(false);
-    },
     successToast: t('common.Delete Success'),
     errorToast: t('common.Delete Failed')
   });
+  const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({
+    mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
+      onCloseWebsiteModal();
+      const [_, billId] = await Promise.all([
+        updateDataset({
+          id: datasetDetail._id,
+          websiteConfig,
+          status: DatasetStatusEnum.syncing
+        }),
+        postCreateTrainingBill({
+          name: 'core.dataset.training.Website Sync',
+          vectorModel: datasetDetail.vectorModel.model,
+          agentModel: datasetDetail.agentModel.model
+        })
+      ]);
+      return billId;
+    },
+    onSuccess(billId: string) {
+      try {
+        postWebsiteSync({ datasetId: datasetDetail._id, billId });
+      } catch (error) {}
+    },
+    errorToast: t('common.Update Failed')
+  });
 
   const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
     getDatasetCollectionPathById(parentId)
   );
 
+  const hasTrainingData = useMemo(
+    () => !!formatCollections.find((item) => item.trainingAmount > 0),
+    [formatCollections]
+  );
   useQuery(
     ['refreshCollection'],
     () => {
       getData(1);
+      if (datasetDetail.status === DatasetStatusEnum.syncing) {
+        loadDatasetDetail(datasetId, true);
+      }
       return null;
     },
     {
       refetchInterval: 6000,
-      enabled: hasTrainingData
+      enabled: hasTrainingData || datasetDetail.status === DatasetStatusEnum.syncing
     }
   );
 
@@ -224,17 +282,33 @@ const CollectionCard = () => {
 
   return (
     <Flex flexDirection={'column'} ref={BoxRef} py={[1, 3]} h={'100%'}>
-      <Flex px={[2, 5]} alignItems={['flex-start', 'center']}>
+      <Flex px={[2, 5]} alignItems={['flex-start', 'center']} h={'35px'}>
         <Box flex={1}>
           <ParentPath
             paths={paths.map((path, i) => ({
               parentId: path.parentId,
-              parentName: i === paths.length - 1 ? `${path.parentName}(${total})` : path.parentName
+              parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName
             }))}
             FirstPathDom={
-              <Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
-                {t('common.File')}({total})
-              </Box>
+              <>
+                <Box fontWeight={'bold'} fontSize={['sm', 'lg']}>
+                  {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total})
+                </Box>
+                {datasetDetail?.websiteConfig?.url && (
+                  <Flex fontSize={'sm'}>
+                    {t('core.dataset.website.Base Url')}:
+                    <Link
+                      href={datasetDetail.websiteConfig.url}
+                      target="_blank"
+                      mr={2}
+                      textDecoration={'underline'}
+                      color={'myBlue.700'}
+                    >
+                      {datasetDetail.websiteConfig.url}
+                    </Link>
+                  </Flex>
+                )}
+              </>
             }
             onClick={(e) => {
               router.replace({
@@ -279,68 +353,109 @@ const CollectionCard = () => {
             />
           </Flex>
         )}
-        {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
-          <MyMenu
-            offset={[-40, 10]}
-            width={120}
-            Button={
-              <MenuButton
-                _hover={{
-                  color: 'myBlue.600'
-                }}
-                fontSize={['sm', 'md']}
-              >
-                <Flex
-                  alignItems={'center'}
-                  px={5}
-                  py={2}
-                  borderRadius={'md'}
-                  cursor={'pointer'}
-                  bg={'myBlue.600'}
-                  overflow={'hidden'}
-                  color={'white'}
-                  h={['28px', '35px']}
-                >
-                  <MyIcon name={'importLight'} mr={2} w={'14px'} />
-                  <Box>{t('dataset.collections.Create And Import')}</Box>
-                </Flex>
-              </MenuButton>
-            }
-            menuList={[
-              {
-                child: (
-                  <Flex>
-                    <Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
-                    {t('Folder')}
-                  </Flex>
-                ),
-                onClick: () => setEditFolderData({})
-              },
-              {
-                child: (
-                  <Flex>
-                    <Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
-                    {t('dataset.Create Virtual File')}
-                  </Flex>
-                ),
-                onClick: () => {
-                  onOpenCreateVirtualFileModal({
-                    defaultVal: '',
-                    onSuccess: (name) => onCreateVirtualFile({ name })
-                  });
+        {datasetDetail?.type === DatasetTypeEnum.dataset && (
+          <>
+            {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
+              <MyMenu
+                offset={[-40, 10]}
+                width={120}
+                Button={
+                  <MenuButton
+                    _hover={{
+                      color: 'myBlue.600'
+                    }}
+                    fontSize={['sm', 'md']}
+                  >
+                    <Flex
+                      alignItems={'center'}
+                      px={5}
+                      py={2}
+                      borderRadius={'md'}
+                      cursor={'pointer'}
+                      bg={'myBlue.600'}
+                      overflow={'hidden'}
+                      color={'white'}
+                      h={['28px', '35px']}
+                    >
+                      <MyIcon name={'importLight'} mr={2} w={'14px'} />
+                      <Box>{t('dataset.collections.Create And Import')}</Box>
+                    </Flex>
+                  </MenuButton>
                 }
-              },
-              {
-                child: (
-                  <Flex>
-                    <Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
-                    {t('dataset.File Input')}
+                menuList={[
+                  {
+                    child: (
+                      <Flex>
+                        <Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
+                        {t('Folder')}
+                      </Flex>
+                    ),
+                    onClick: () => setEditFolderData({})
+                  },
+                  {
+                    child: (
+                      <Flex>
+                        <Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
+                        {t('dataset.Create Virtual File')}
+                      </Flex>
+                    ),
+                    onClick: () => {
+                      onOpenCreateVirtualFileModal({
+                        defaultVal: '',
+                        onSuccess: (name) => {
+                          onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual });
+                        }
+                      });
+                    }
+                  },
+                  {
+                    child: (
+                      <Flex>
+                        <Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
+                        {t('dataset.File Input')}
+                      </Flex>
+                    ),
+                    onClick: onOpenFileImportModal
+                  }
+                ]}
+              />
+            )}
+          </>
+        )}
+        {datasetDetail?.type === DatasetTypeEnum.websiteDataset && (
+          <>
+            {datasetDetail?.websiteConfig?.url ? (
+              <Flex alignItems={'center'}>
+                {datasetDetail.status === DatasetStatusEnum.active && (
+                  <Button onClick={onOpenWebsiteModal}>{t('common.Config')}</Button>
+                )}
+                {datasetDetail.status === DatasetStatusEnum.syncing && (
+                  <Flex
+                    ml={3}
+                    alignItems={'center'}
+                    px={3}
+                    py={1}
+                    borderRadius="md"
+                    border={theme.borders.base}
+                  >
+                    <Box
+                      animation={'zoomStopIcon 0.5s infinite alternate'}
+                      bg={'myGray.700'}
+                      w="8px"
+                      h="8px"
+                      borderRadius={'50%'}
+                      mt={'1px'}
+                    ></Box>
+                    <Box ml={2} color={'myGray.600'}>
+                      {t('core.dataset.status.syncing')}
+                    </Box>
                   </Flex>
-                ),
-                onClick: onOpenFileImportModal
-              }
-            ]}
-          />
+                )}
+              </Flex>
+            ) : (
+              <Button onClick={onOpenWebsiteModal}>{t('core.dataset.Set Website Config')}</Button>
+            )}
+          </>
         )}
       </Flex>
 
@@ -428,11 +543,7 @@ const CollectionCard = () => {
                     </MyTooltip>
                   </Flex>
                 </Td>
-                <Td fontSize={'md'}>
-                  {collection.type === DatasetCollectionTypeEnum.folder
-                    ? '-'
-                    : collection.dataAmount}
-                </Td>
+                <Td fontSize={'md'}>{collection.dataAmount || '-'}</Td>
                 <Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
                 <Td>
                   <Flex
@@ -443,10 +554,10 @@ const CollectionCard = () => {
                       h: '10px',
                       mr: 2,
                       borderRadius: 'lg',
-                      bg: collection?.color
+                      bg: collection.color
                     }}
                   >
-                    {collection?.statusText}
+                    {t(collection.statusText)}
                   </Flex>
                 </Td>
                 <Td onClick={(e) => e.stopPropagation()}>
@@ -536,14 +647,31 @@ const CollectionCard = () => {
             ))}
           </Tbody>
         </Table>
-        <Loading loading={isLoading && collections.length === 0} fixed={false} />
         {total > pageSize && (
           <Flex mt={2} justifyContent={'center'}>
             <Pagination />
           </Flex>
         )}
-        {total === 0 && <EmptyTip text="数据集空空如也" />}
+        {total === 0 && (
+          <EmptyTip
+            text={
+              datasetDetail.type === DatasetTypeEnum.dataset ? (
+                t('core.dataset.collection.Empty Tip')
+              ) : (
+                <Flex>
+                  {t('core.dataset.collection.Website Empty Tip')}
+                  <Box textDecoration={'underline'} cursor={'pointer'} onClick={onOpenWebsiteModal}>
+                    {t('core.dataset.collection.Click top config website')}
+                  </Box>
+                </Flex>
+              )
+            }
+          />
+        )}
       </TableContainer>
+      <Loading
+        loading={isCreating || isDeleting || isUpdating || (isLoading && collections.length === 0)}
+      />
 
       <ConfirmModal />
       <EditTitleModal />
@@ -559,7 +687,6 @@ const CollectionCard = () => {
           onClose={onCloseFileImportModal}
         />
       )}
-
       {!!editFolderData && (
         <EditFolderModal
           onClose={() => setEditFolderData(undefined)}
@@ -570,15 +697,13 @@ const CollectionCard = () => {
                   id: editFolderData.id,
                   name
                 });
+                getData(pageNum);
               } else {
-                await postDatasetCollection({
-                  parentId,
-                  datasetId,
+                onCreateCollection({
                   name,
                   type: DatasetCollectionTypeEnum.folder
                 });
               }
-              getData(pageNum);
             } catch (error) {
               return Promise.reject(error);
             }
@@ -587,7 +712,6 @@ const CollectionCard = () => {
           name={editFolderData.name}
         />
       )}
-
       {!!moveCollectionData && (
         <SelectCollections
           datasetId={datasetId}
@@ -608,6 +732,16 @@ const CollectionCard = () => {
           }}
         />
       )}
+      {isOpenWebsiteModal && (
+        <WebSiteConfigModal
+          onClose={onCloseWebsiteModal}
+          onSuccess={onUpdateDatasetWebsiteConfig}
+          defaultValue={{
+            url: datasetDetail?.websiteConfig?.url,
+            selector: datasetDetail?.websiteConfig?.selector
+          }}
+        />
+      )}
     </Flex>
   );
 };

projects/app/src/pages/dataset/detail/components/DataCard.tsx

@@ -55,7 +55,10 @@ const DataCard = () => {
   const router = useRouter();
   const { userInfo } = useUserStore();
   const { isPc } = useSystemStore();
-  const { collectionId = '' } = router.query as { collectionId: string };
+  const { collectionId = '', datasetId } = router.query as {
+    collectionId: string;
+    datasetId: string;
+  };
   const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
   const { t } = useTranslation();
   const [searchText, setSearchText] = useState('');
@@ -99,8 +102,18 @@ const DataCard = () => {
   );
 
   // get file info
-  const { data: collection } = useQuery(['getDatasetCollectionById', collectionId], () =>
-    getDatasetCollectionById(collectionId)
+  const { data: collection } = useQuery(
+    ['getDatasetCollectionById', collectionId],
+    () => getDatasetCollectionById(collectionId),
+    {
+      onError: () => {
+        router.replace({
+          query: {
+            datasetId
+          }
+        });
+      }
+    }
   );
 
   const canWrite = useMemo(
@@ -290,6 +303,7 @@ const DataCard = () => {
             </Flex>
             <Box
               maxH={'135px'}
+              minH={'90px'}
               overflow={'hidden'}
               wordBreak={'break-all'}
               pt={1}

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -18,13 +18,13 @@ import { useTranslation } from 'next-i18next';
 import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
-import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
 import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
+import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
 
 const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
 const CreateFileModal = dynamic(() => import('./CreateFileModal'));
@@ -215,7 +215,7 @@ const FileSelect = ({
   );
   // link fetch
   const onUrlFetch = useCallback(
-    (e: FetchResultItem[]) => {
+    (e: UrlFetchResponse) => {
       const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
         const splitRes = splitText2Chunks({
           text: content,

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -188,6 +188,8 @@ const Provider = ({
 
   const onReSplitChunks = useCallback(async () => {
     try {
+      setPreviewFile(undefined);
+
       setFiles((state) =>
         state.map((file) => {
           const splitRes = splitText2Chunks({
@@ -490,6 +492,7 @@ export const SelectorContainer = ({
                 display={['block', 'none']}
                 onClick={(e) => {
                   e.stopPropagation();
+                  setPreviewFile(undefined);
                   setFiles((state) => state.filter((file) => file.id !== item.id));
                 }}
               />

projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx

@@ -1,31 +1,39 @@
-import React, { useRef } from 'react';
+import React from 'react';
 import { useTranslation } from 'next-i18next';
 import MyModal from '@/components/MyModal';
-import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
-import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
+import { Box, Button, Input, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
 import { useRequest } from '@/web/common/hooks/useRequest';
-import { postFetchUrls } from '@/web/common/plugin/api';
+import { postFetchUrls } from '@/web/common/tools/api';
+import { useForm } from 'react-hook-form';
+import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
 
 const UrlFetchModal = ({
   onClose,
   onSuccess
 }: {
   onClose: () => void;
-  onSuccess: (e: FetchResultItem[]) => void;
+  onSuccess: (e: UrlFetchResponse) => void;
 }) => {
   const { t } = useTranslation();
-  const Dom = useRef<HTMLTextAreaElement>(null);
+  const { register, handleSubmit } = useForm({
+    defaultValues: {
+      urls: '',
+      selector: ''
+    }
+  });
 
   const { mutate, isLoading } = useRequest({
-    mutationFn: async () => {
-      const val = Dom.current?.value || '';
-      const urls = val.split('\n').filter((e) => e);
-      const res = await postFetchUrls(urls);
+    mutationFn: async ({ urls, selector }: { urls: string; selector: string }) => {
+      const urlList = urls.split('\n').filter((e) => e);
+      const res = await postFetchUrls({
+        urlList,
+        selector
+      });
 
       onSuccess(res);
       onClose();
     },
-    errorToast: '获取链接失败'
+    errorToast: t('core.dataset.import.Fetch Error')
   });
 
   return (
@@ -34,8 +42,8 @@ const UrlFetchModal = ({
       title={
         <Box>
           <Box>{t('file.Fetch Url')}</Box>
-          <Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
-            目前仅支持读取静态链接，请注意检查结果
+          <Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'}>
+            {t('core.dataset.import.Fetch url tip')}
           </Box>
         </Box>
       }
@@ -45,20 +53,31 @@ const UrlFetchModal = ({
       w={'600px'}
     >
       <ModalBody>
-        <Textarea
-          ref={Dom}
-          rows={12}
-          whiteSpace={'nowrap'}
-          resize={'both'}
-          placeholder={'最多10个链接，每行一个。'}
-        />
+        <Box>
+          <Box fontWeight={'bold'}>{t('core.dataset.import.Fetch Url')}</Box>
+          <Textarea
+            {...register('urls', {
+              required: true
+            })}
+            rows={11}
+            whiteSpace={'nowrap'}
+            resize={'both'}
+            placeholder={t('core.dataset.import.Fetch url placeholder')}
+          />
+        </Box>
+        <Box mt={4}>
+          <Box fontWeight={'bold'}>
+            {t('core.dataset.website.Selector')}({t('common.choosable')})
+          </Box>{' '}
+          <Input {...register('selector')} placeholder="body .content #document" />
+        </Box>
       </ModalBody>
       <ModalFooter>
         <Button variant={'base'} mr={4} onClick={onClose}>
-          取消
+          {t('common.Close')}
         </Button>
-        <Button isLoading={isLoading} onClick={mutate}>
-          确认
+        <Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>
+          {t('common.Confirm')}
         </Button>
       </ModalFooter>
     </MyModal>

projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx

@@ -0,0 +1,101 @@
+import React from 'react';
+import MyModal from '@/components/MyModal';
+import { useTranslation } from 'next-i18next';
+import { Box, Button, Input, ModalBody, ModalFooter } from '@chakra-ui/react';
+import { strIsLink } from '@fastgpt/global/common/string/tools';
+import { useToast } from '@/web/common/hooks/useToast';
+import { useForm } from 'react-hook-form';
+import { useConfirm } from '@/web/common/hooks/useConfirm';
+
+type FormType = {
+  url?: string | undefined;
+  selector?: string | undefined;
+};
+
+const WebsiteConfigModal = ({
+  onClose,
+  onSuccess,
+  defaultValue = {
+    url: '',
+    selector: ''
+  }
+}: {
+  onClose: () => void;
+  onSuccess: (data: FormType) => void;
+  defaultValue?: FormType;
+}) => {
+  const { t } = useTranslation();
+  const { toast } = useToast();
+  const { register, handleSubmit } = useForm({
+    defaultValues: defaultValue
+  });
+  const isEdit = !!defaultValue.url;
+  const confirmTip = isEdit
+    ? t('core.dataset.website.Confirm Update Tips')
+    : t('core.dataset.website.Confirm Create Tips');
+
+  const { ConfirmModal, openConfirm } = useConfirm({
+    type: 'common'
+  });
+
+  return (
+    <MyModal
+      isOpen
+      iconSrc="core/dataset/websiteDataset"
+      title={t('core.dataset.website.Config')}
+      onClose={onClose}
+      maxW={'500px'}
+    >
+      <ModalBody>
+        <Box fontSize={'sm'} color={'myGray.600'}>
+          {t('core.dataset.website.Config Description')}
+        </Box>
+        <Box mt={2}>
+          <Box>{t('core.dataset.website.Base Url')}</Box>
+          <Input
+            placeholder={t('core.dataset.collection.Website Link')}
+            {...register('url', {
+              required: true
+            })}
+          />
+        </Box>
+        <Box mt={3}>
+          <Box>
+            {t('core.dataset.website.Selector')}({t('common.choosable')})
+          </Box>
+          <Input {...register('selector')} placeholder="body .content #document" />
+        </Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button variant={'base'} onClick={onClose}>
+          {t('common.Close')}
+        </Button>
+        <Button
+          ml={2}
+          onClick={handleSubmit((data) => {
+            if (!data.url) return;
+            // check is link
+            if (!strIsLink(data.url)) {
+              return toast({
+                status: 'warning',
+                title: t('common.link.UnValid')
+              });
+            }
+            openConfirm(
+              () => {
+                onSuccess(data);
+              },
+              undefined,
+              confirmTip
+            )();
+          })}
+        >
+          {t('core.dataset.website.Start Sync')}
+        </Button>
+      </ModalFooter>
+      <ConfirmModal />
+    </MyModal>
+  );
+};
+
+export default WebsiteConfigModal;

projects/app/src/pages/dataset/detail/components/Info.tsx

@@ -7,7 +7,7 @@ import React, {
   ForwardedRef
 } from 'react';
 import { useRouter } from 'next/router';
-import { Box, Flex, Button, FormControl, IconButton, Input } from '@chakra-ui/react';
+import { Box, Flex, Button, FormControl, IconButton, Input, Textarea } from '@chakra-ui/react';
 import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
 import { delDatasetById } from '@/web/core/dataset/api';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
@@ -18,21 +18,19 @@ import { UseFormReturn } from 'react-hook-form';
 import { compressImgFileAndUpload } from '@/web/common/file/controller';
 import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
 import Avatar from '@/components/Avatar';
-import Tag from '@/components/Tag';
 import MyTooltip from '@/components/MyTooltip';
 import { useTranslation } from 'next-i18next';
 import PermissionRadio from '@/components/support/permission/Radio';
 import MySelect from '@/components/Select';
 import { qaModelList } from '@/web/common/system/staticData';
 
-export interface ComponentRef {
-  initInput: (tags: string) => void;
-}
-
-const Info = (
-  { datasetId, form }: { datasetId: string; form: UseFormReturn<DatasetItemType, any> },
-  ref: ForwardedRef<ComponentRef>
-) => {
+const Info = ({
+  datasetId,
+  form
+}: {
+  datasetId: string;
+  form: UseFormReturn<DatasetItemType, any>;
+}) => {
   const { t } = useTranslation();
   const { getValues, formState, setValue, register, handleSubmit } = form;
   const InputRef = useRef<HTMLInputElement>(null);
@@ -52,7 +50,7 @@ const Info = (
     multiple: false
   });
 
-  const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore();
+  const { datasetDetail, loadDatasets, updateDataset } = useDatasetStore();
 
   /* 点击删除 */
   const onclickDelKb = useCallback(async () => {
@@ -121,8 +119,8 @@ const Info = (
       try {
         const src = await compressImgFileAndUpload({
           file,
-          maxW: 100,
-          maxH: 100
+          maxW: 300,
+          maxH: 300
         });
 
         setValue('avatar', src);
@@ -138,14 +136,6 @@ const Info = (
     [setRefresh, setValue, toast]
   );
 
-  useImperativeHandle(ref, () => ({
-    initInput: (tags: string) => {
-      if (InputRef.current) {
-        InputRef.current.value = tags;
-      }
-    }
-  }));
-
   return (
     <Box py={5} px={[5, 10]}>
       <Flex mt={5} w={'100%'} alignItems={'center'}>
@@ -154,18 +144,7 @@ const Info = (
         </Box>
         <Box flex={1}>{datasetDetail._id}</Box>
       </Flex>
-      <Flex mt={8} w={'100%'} alignItems={'center'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          索引模型
-        </Box>
-        <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
-      </Flex>
-      <Flex mt={8} w={'100%'} alignItems={'center'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          MaxTokens
-        </Box>
-        <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
-      </Flex>
+
       <Flex mt={5} w={'100%'} alignItems={'center'}>
         <Box flex={['0 0 90px', '0 0 160px']} w={0}>
           知识库头像
@@ -183,7 +162,7 @@ const Info = (
           </MyTooltip>
         </Box>
       </Flex>
-      <FormControl mt={8} w={'100%'} display={'flex'} alignItems={'center'}>
+      <Flex mt={8} w={'100%'} alignItems={'center'}>
         <Box flex={['0 0 90px', '0 0 160px']} w={0}>
           知识库名称
         </Box>
@@ -194,7 +173,19 @@ const Info = (
             required: '知识库名称不能为空'
           })}
         />
-      </FormControl>
+      </Flex>
+      <Flex mt={8} w={'100%'} alignItems={'center'}>
+        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+          索引模型
+        </Box>
+        <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
+      </Flex>
+      <Flex mt={8} w={'100%'} alignItems={'center'}>
+        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+          {t('core.Max Token')}
+        </Box>
+        <Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
+      </Flex>
       <Flex mt={6} alignItems={'center'}>
         <Box flex={['0 0 90px', '0 0 160px']} w={0}>
           {t('dataset.Agent Model')}
@@ -216,33 +207,9 @@ const Info = (
           />
         </Box>
       </Flex>
-      <Flex mt={8} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
-        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
-          标签
-          <MyTooltip label={'用空格隔开多个标签，便于搜索'} forceShow>
-            <QuestionOutlineIcon ml={1} />
-          </MyTooltip>
-        </Box>
-        <Input
-          flex={[1, '0 0 300px']}
-          ref={InputRef}
-          defaultValue={getValues('tags')}
-          placeholder={'标签,使用空格分割。'}
-          maxLength={30}
-          onChange={(e) => {
-            setValue('tags', e.target.value.split(' ').filter(Boolean));
-            setRefresh(!refresh);
-          }}
-        />
-        <Flex w={'100%'} pl={['90px', '160px']} mt={2}>
-          {getValues('tags')
-            .filter(Boolean)
-            .map((item, i) => (
-              <Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
-                {item}
-              </Tag>
-            ))}
-        </Flex>
+      <Flex mt={8} alignItems={'center'} w={'100%'}>
+        <Box flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</Box>
+        <Textarea flex={[1, '0 0 300px']} {...register('intro')} placeholder={t('common.Intro')} />
       </Flex>
       {datasetDetail.isOwner && (
         <Flex mt={5} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
@@ -292,4 +259,4 @@ const Info = (
   );
 };
 
-export default forwardRef(Info);
+export default React.memo(Info);

projects/app/src/pages/dataset/detail/components/InputDataModal.tsx

@@ -199,12 +199,12 @@ const InputDataModal = ({
         <Box p={5} borderRight={theme.borders.base}>
           <RawSourceText
             w={'200px'}
-            className=""
+            className="textEllipsis3"
             whiteSpace={'pre-wrap'}
             sourceName={collection.sourceName}
             sourceId={collection.sourceId}
             mb={6}
-            fontSize={['14px', '16px']}
+            fontSize={'sm'}
           />
           <SideTabs
             list={tabList}
@@ -220,14 +220,14 @@ const InputDataModal = ({
             }}
           />
         </Box>
-        <Flex flexDirection={'column'} px={5} py={3} flex={1} h={'100%'}>
-          <Box fontSize={'lg'} fontWeight={'bold'} mb={4}>
+        <Flex flexDirection={'column'} py={3} flex={1} h={'100%'}>
+          <Box fontSize={'lg'} px={5} fontWeight={'bold'} mb={4}>
             {currentTab === TabEnum.content && (
               <>{defaultValue.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')}</>
             )}
             {currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}</>}
           </Box>
-          <Box flex={1} overflow={'auto'}>
+          <Box flex={1} px={5} overflow={'auto'}>
             {currentTab === TabEnum.content && (
               <>
                 <Box>
@@ -358,7 +358,7 @@ const InputDataModal = ({
               </Grid>
             )}
           </Box>
-          <Flex justifyContent={'flex-end'} mt={4}>
+          <Flex justifyContent={'flex-end'} px={5} mt={4}>
             <Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
               {t('common.Close')}
             </Button>

projects/app/src/pages/dataset/detail/components/Test.tsx

@@ -187,7 +187,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
         </Box>
       </Box>
       {/* result show */}
-      <Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={1}>
+      <Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={'1 0 0'}>
         {!datasetTestItem?.results || datasetTestItem.results.length === 0 ? (
           <Flex
             mt={[10, 0]}
@@ -275,7 +275,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                     <MyIcon name={'kbTest'} w={'14px'} />
                     <Progress
                       mx={2}
-                      flex={1}
+                      flex={'1 0 0'}
                       value={item.score * 100}
                       size="sm"
                       borderRadius={'20px'}
@@ -283,7 +283,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                     />
                     <Box>{item.score.toFixed(4)}</Box>
                   </Flex>
-                  <Box px={2} fontSize={'xs'} color={'myGray.600'}>
+                  <Box px={2} fontSize={'xs'} color={'myGray.600'} wordBreak={'break-word'}>
                     <Box>{item.q}</Box>
                     <Box>{item.a}</Box>
                   </Box>