4.6.3-website dataset (#532)

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -18,13 +18,13 @@ import { useTranslation } from 'next-i18next';
 import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
-import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
 import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
+import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
 
 const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
 const CreateFileModal = dynamic(() => import('./CreateFileModal'));
@@ -215,7 +215,7 @@ const FileSelect = ({
   );
   // link fetch
   const onUrlFetch = useCallback(
-    (e: FetchResultItem[]) => {
+    (e: UrlFetchResponse) => {
       const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
         const splitRes = splitText2Chunks({
           text: content,

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -188,6 +188,8 @@ const Provider = ({
 
   const onReSplitChunks = useCallback(async () => {
     try {
+      setPreviewFile(undefined);
+
       setFiles((state) =>
         state.map((file) => {
           const splitRes = splitText2Chunks({
@@ -490,6 +492,7 @@ export const SelectorContainer = ({
                 display={['block', 'none']}
                 onClick={(e) => {
                   e.stopPropagation();
+                  setPreviewFile(undefined);
                   setFiles((state) => state.filter((file) => file.id !== item.id));
                 }}
               />

projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx

@@ -1,31 +1,39 @@
-import React, { useRef } from 'react';
+import React from 'react';
 import { useTranslation } from 'next-i18next';
 import MyModal from '@/components/MyModal';
-import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
-import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
+import { Box, Button, Input, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
 import { useRequest } from '@/web/common/hooks/useRequest';
-import { postFetchUrls } from '@/web/common/plugin/api';
+import { postFetchUrls } from '@/web/common/tools/api';
+import { useForm } from 'react-hook-form';
+import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
 
 const UrlFetchModal = ({
   onClose,
   onSuccess
 }: {
   onClose: () => void;
-  onSuccess: (e: FetchResultItem[]) => void;
+  onSuccess: (e: UrlFetchResponse) => void;
 }) => {
   const { t } = useTranslation();
-  const Dom = useRef<HTMLTextAreaElement>(null);
+  const { register, handleSubmit } = useForm({
+    defaultValues: {
+      urls: '',
+      selector: ''
+    }
+  });
 
   const { mutate, isLoading } = useRequest({
-    mutationFn: async () => {
-      const val = Dom.current?.value || '';
-      const urls = val.split('\n').filter((e) => e);
-      const res = await postFetchUrls(urls);
+    mutationFn: async ({ urls, selector }: { urls: string; selector: string }) => {
+      const urlList = urls.split('\n').filter((e) => e);
+      const res = await postFetchUrls({
+        urlList,
+        selector
+      });
 
       onSuccess(res);
       onClose();
     },
-    errorToast: '获取链接失败'
+    errorToast: t('core.dataset.import.Fetch Error')
   });
 
   return (
@@ -34,8 +42,8 @@ const UrlFetchModal = ({
       title={
         <Box>
           <Box>{t('file.Fetch Url')}</Box>
-          <Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
-            目前仅支持读取静态链接，请注意检查结果
+          <Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'}>
+            {t('core.dataset.import.Fetch url tip')}
           </Box>
         </Box>
       }
@@ -45,20 +53,31 @@ const UrlFetchModal = ({
       w={'600px'}
     >
       <ModalBody>
-        <Textarea
-          ref={Dom}
-          rows={12}
-          whiteSpace={'nowrap'}
-          resize={'both'}
-          placeholder={'最多10个链接，每行一个。'}
-        />
+        <Box>
+          <Box fontWeight={'bold'}>{t('core.dataset.import.Fetch Url')}</Box>
+          <Textarea
+            {...register('urls', {
+              required: true
+            })}
+            rows={11}
+            whiteSpace={'nowrap'}
+            resize={'both'}
+            placeholder={t('core.dataset.import.Fetch url placeholder')}
+          />
+        </Box>
+        <Box mt={4}>
+          <Box fontWeight={'bold'}>
+            {t('core.dataset.website.Selector')}({t('common.choosable')})
+          </Box>{' '}
+          <Input {...register('selector')} placeholder="body .content #document" />
+        </Box>
       </ModalBody>
       <ModalFooter>
         <Button variant={'base'} mr={4} onClick={onClose}>
-          取消
+          {t('common.Close')}
         </Button>
-        <Button isLoading={isLoading} onClick={mutate}>
-          确认
+        <Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>
+          {t('common.Confirm')}
         </Button>
       </ModalFooter>
     </MyModal>

projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx

@@ -0,0 +1,101 @@
+import React from 'react';
+import MyModal from '@/components/MyModal';
+import { useTranslation } from 'next-i18next';
+import { Box, Button, Input, ModalBody, ModalFooter } from '@chakra-ui/react';
+import { strIsLink } from '@fastgpt/global/common/string/tools';
+import { useToast } from '@/web/common/hooks/useToast';
+import { useForm } from 'react-hook-form';
+import { useConfirm } from '@/web/common/hooks/useConfirm';
+
+type FormType = {
+  url?: string | undefined;
+  selector?: string | undefined;
+};
+
+const WebsiteConfigModal = ({
+  onClose,
+  onSuccess,
+  defaultValue = {
+    url: '',
+    selector: ''
+  }
+}: {
+  onClose: () => void;
+  onSuccess: (data: FormType) => void;
+  defaultValue?: FormType;
+}) => {
+  const { t } = useTranslation();
+  const { toast } = useToast();
+  const { register, handleSubmit } = useForm({
+    defaultValues: defaultValue
+  });
+  const isEdit = !!defaultValue.url;
+  const confirmTip = isEdit
+    ? t('core.dataset.website.Confirm Update Tips')
+    : t('core.dataset.website.Confirm Create Tips');
+
+  const { ConfirmModal, openConfirm } = useConfirm({
+    type: 'common'
+  });
+
+  return (
+    <MyModal
+      isOpen
+      iconSrc="core/dataset/websiteDataset"
+      title={t('core.dataset.website.Config')}
+      onClose={onClose}
+      maxW={'500px'}
+    >
+      <ModalBody>
+        <Box fontSize={'sm'} color={'myGray.600'}>
+          {t('core.dataset.website.Config Description')}
+        </Box>
+        <Box mt={2}>
+          <Box>{t('core.dataset.website.Base Url')}</Box>
+          <Input
+            placeholder={t('core.dataset.collection.Website Link')}
+            {...register('url', {
+              required: true
+            })}
+          />
+        </Box>
+        <Box mt={3}>
+          <Box>
+            {t('core.dataset.website.Selector')}({t('common.choosable')})
+          </Box>
+          <Input {...register('selector')} placeholder="body .content #document" />
+        </Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button variant={'base'} onClick={onClose}>
+          {t('common.Close')}
+        </Button>
+        <Button
+          ml={2}
+          onClick={handleSubmit((data) => {
+            if (!data.url) return;
+            // check is link
+            if (!strIsLink(data.url)) {
+              return toast({
+                status: 'warning',
+                title: t('common.link.UnValid')
+              });
+            }
+            openConfirm(
+              () => {
+                onSuccess(data);
+              },
+              undefined,
+              confirmTip
+            )();
+          })}
+        >
+          {t('core.dataset.website.Start Sync')}
+        </Button>
+      </ModalFooter>
+      <ConfirmModal />
+    </MyModal>
+  );
+};
+
+export default WebsiteConfigModal;