4.6.3-website dataset (#532)

2025-08-01 20:27:45 +00:00 · 2023-12-03 20:45:57 +08:00
parent b916183848
commit a9ae270335
122 changed files with 3793 additions and 1360 deletions
--- a/projects/app/src/web/common/api/request.ts
+++ b/projects/app/src/web/common/api/request.ts
@@ -160,18 +160,18 @@ function request(
 * @param {Object} config
 * @returns
 */
-export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+export function GET<T = undefined>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
  return request(url, params, config, 'GET');
 }

-export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function POST<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
  return request(url, data, config, 'POST');
 }

-export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function PUT<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
  return request(url, data, config, 'PUT');
 }

-export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function DELETE<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
  return request(url, data, config, 'DELETE');
 }
--- a/projects/app/src/web/common/file/controller.ts
+++ b/projects/app/src/web/common/file/controller.ts
@@ -35,9 +35,9 @@ export const uploadFiles = ({
 */
 export const compressBase64ImgAndUpload = ({
  base64,
-  maxW = 200,
-  maxH = 200,
-  maxSize = 1024 * 100, // 100kb
+  maxW = 1080,
+  maxH = 1080,
+  maxSize = 1024 * 500, // 300kb
  expiredTime
 }: {
  base64: string;
@@ -77,7 +77,7 @@ export const compressBase64ImgAndUpload = ({
      }

      ctx.drawImage(img, 0, 0, width, height);
-      const compressedDataUrl = canvas.toDataURL(fileType, 0.8);
+      const compressedDataUrl = canvas.toDataURL(fileType, 1);
      // 移除 canvas 元素
      canvas.remove();

--- a/projects/app/src/web/common/file/utils.ts
+++ b/projects/app/src/web/common/file/utils.ts
@@ -1,6 +1,7 @@
 import mammoth from 'mammoth';
 import Papa from 'papaparse';
 import { compressBase64ImgAndUpload } from './controller';
+import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';

 /**
 * 读取 txt 文件内容
@@ -182,9 +183,9 @@ export const formatMarkdown = async (rawText: string = '') => {
      try {
        const str = await compressBase64ImgAndUpload({
          base64,
-          maxW: 800,
-          maxH: 800,
-          maxSize: 1024 * 1024 * 2
+          maxW: 4329,
+          maxH: 4329,
+          maxSize: 1024 * 1024 * 5
        });
        rawText = rawText.replace(base64, str);
      } catch (error) {
@@ -199,14 +200,7 @@ export const formatMarkdown = async (rawText: string = '') => {
    rawText = rawText.replace(/\s*(!\[.*\]\(.*\))\s*/g, '$1');
  }

-  // replace \
-  const reg1 = /\\([-.!`_(){}\[\]])/g;
-  if (reg1.test(rawText)) {
-    rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
-  }
-  rawText = rawText.replace(/\\\\n/g, '\\n');
-
-  return rawText;
+  return simpleMarkdownText(rawText);
 };

 /**
--- a/projects/app/src/web/common/hooks/useConfirm.tsx
+++ b/projects/app/src/web/common/hooks/useConfirm.tsx
@@ -1,29 +1,46 @@
-import { useCallback, useRef, useState } from 'react';
-import {
-  AlertDialog,
-  AlertDialogBody,
-  AlertDialogFooter,
-  AlertDialogHeader,
-  AlertDialogContent,
-  AlertDialogOverlay,
-  useDisclosure,
-  Button
-} from '@chakra-ui/react';
+import { useCallback, useMemo, useRef, useState } from 'react';
+import { useDisclosure, Button, ModalBody, ModalFooter } from '@chakra-ui/react';
 import { useTranslation } from 'next-i18next';
+import MyModal from '@/components/MyModal';

 export const useConfirm = (props?: {
-  title?: string | null;
-  content?: string | null;
+  title?: string;
+  iconSrc?: string | '';
+  content?: string;
  bg?: string;
  showCancel?: boolean;
+  type?: 'common' | 'delete';
 }) => {
  const { t } = useTranslation();
-  const { title = t('Warning'), content, bg, showCancel = true } = props || {};
+
+  const map = useMemo(() => {
+    const map = {
+      common: {
+        title: t('common.confirm.Common Tip'),
+        bg: undefined,
+        iconSrc: 'common/confirm/commonTip'
+      },
+      delete: {
+        title: t('common.Delete Warning'),
+        bg: 'red.600',
+        iconSrc: 'common/confirm/deleteTip'
+      }
+    };
+    if (props?.type && map[props.type]) return map[props.type];
+    return map.common;
+  }, [props?.type, t]);
+
+  const {
+    title = map?.title || t('Warning'),
+    iconSrc = map?.iconSrc,
+    content,
+    bg = map?.bg,
+    showCancel = true
+  } = props || {};
  const [customContent, setCustomContent] = useState(content);

  const { isOpen, onOpen, onClose } = useDisclosure();

-  const cancelRef = useRef(null);
  const confirmCb = useRef<any>();
  const cancelCb = useRef<any>();

@@ -41,51 +58,41 @@ export const useConfirm = (props?: {
    ),
    ConfirmModal: useCallback(
      () => (
-        <AlertDialog
+        <MyModal
          isOpen={isOpen}
-          leastDestructiveRef={cancelRef}
-          autoFocus={false}
          onClose={onClose}
+          iconSrc={iconSrc}
+          title={title}
+          maxW={['90vw', '500px']}
        >
-          <AlertDialogOverlay>
-            <AlertDialogContent maxW={'min(90vw,400px)'}>
-              <AlertDialogHeader fontSize="lg" fontWeight="bold">
-                {title}
-              </AlertDialogHeader>
+          <ModalBody pt={5}>{customContent}</ModalBody>
+          <ModalFooter>
+            {showCancel && (
+              <Button
+                variant={'base'}
+                onClick={() => {
+                  onClose();
+                  typeof cancelCb.current === 'function' && cancelCb.current();
+                }}
+              >
+                {t('Cancel')}
+              </Button>
+            )}

-              <AlertDialogBody whiteSpace={'pre-wrap'} py={0}>
-                {customContent}
-              </AlertDialogBody>
-
-              <AlertDialogFooter>
-                {showCancel && (
-                  <Button
-                    variant={'base'}
-                    onClick={() => {
-                      onClose();
-                      typeof cancelCb.current === 'function' && cancelCb.current();
-                    }}
-                  >
-                    {t('Cancel')}
-                  </Button>
-                )}
-
-                <Button
-                  {...(bg && { bg: `${bg} !important` })}
-                  ml={4}
-                  onClick={() => {
-                    onClose();
-                    typeof confirmCb.current === 'function' && confirmCb.current();
-                  }}
-                >
-                  {t('Confirm')}
-                </Button>
-              </AlertDialogFooter>
-            </AlertDialogContent>
-          </AlertDialogOverlay>
-        </AlertDialog>
+            <Button
+              {...(bg && { bg: `${bg} !important` })}
+              ml={4}
+              onClick={() => {
+                onClose();
+                typeof confirmCb.current === 'function' && confirmCb.current();
+              }}
+            >
+              {t('Confirm')}
+            </Button>
+          </ModalFooter>
+        </MyModal>
      ),
-      [bg, customContent, isOpen, onClose, t, title]
+      [bg, customContent, iconSrc, isOpen, onClose, showCancel, t, title]
    )
  };
 };
--- a/projects/app/src/web/common/hooks/useEditTitle.tsx
+++ b/projects/app/src/web/common/hooks/useEditTitle.tsx
@@ -1,21 +1,27 @@
 import React, { useCallback, useRef } from 'react';
 import { ModalFooter, ModalBody, Input, useDisclosure, Button, Box } from '@chakra-ui/react';
 import MyModal from '@/components/MyModal';
+import { useToast } from './useToast';

 export const useEditTitle = ({
  title,
  tip,
-  placeholder = ''
+  placeholder = '',
+  canEmpty = true,
+  valueRule
 }: {
  title: string;
  tip?: string;
  placeholder?: string;
+  canEmpty?: boolean;
+  valueRule?: (val: string) => string | void;
 }) => {
  const { isOpen, onOpen, onClose } = useDisclosure();

  const inputRef = useRef<HTMLInputElement | null>(null);
  const onSuccessCb = useRef<(content: string) => void | Promise<void>>();
  const onErrorCb = useRef<(err: any) => void>();
+  const { toast } = useToast();
  const defaultValue = useRef('');

  const onOpenModal = useCallback(
@@ -37,21 +43,43 @@ export const useEditTitle = ({
  );

  const onclickConfirm = useCallback(async () => {
-    if (!inputRef.current) return;
+    if (!inputRef.current || !onSuccessCb.current) return;
+    const val = inputRef.current.value;
+
+    if (!canEmpty && !val) {
+      inputRef.current.focus();
+      return;
+    }
+
+    if (valueRule) {
+      const result = valueRule(val);
+      if (result) {
+        return toast({
+          status: 'warning',
+          title: result
+        });
+      }
+    }
+
    try {
-      const val = inputRef.current.value;
-      await onSuccessCb.current?.(val);
+      await onSuccessCb.current(val);

      onClose();
    } catch (err) {
      onErrorCb.current?.(err);
    }
-  }, [onClose]);
+  }, [canEmpty, onClose]);

  // eslint-disable-next-line react/display-name
  const EditModal = useCallback(
-    ({ maxLength = 30 }: { maxLength?: number }) => (
-      <MyModal isOpen={isOpen} onClose={onClose} iconSrc="/imgs/modal/edit.svg" title={title}>
+    ({
+      maxLength = 30,
+      iconSrc = '/imgs/modal/edit.svg'
+    }: {
+      maxLength?: number;
+      iconSrc?: string;
+    }) => (
+      <MyModal isOpen={isOpen} onClose={onClose} iconSrc={iconSrc} title={title} maxW={'500px'}>
        <ModalBody>
          {!!tip && (
            <Box mb={2} color={'myGray.500'} fontSize={'sm'}>
--- a/projects/app/src/web/common/plugin/api.ts
+++ b/projects/app/src/web/common/plugin/api.ts
@@ -1,6 +0,0 @@
-import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
-
-import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
-
-export const postFetchUrls = (urlList: string[]) =>
-  POST<FetchResultItem[]>(`/plugins/urlFetch`, { urlList });
--- a/projects/app/src/web/common/system/staticData.ts
+++ b/projects/app/src/web/common/system/staticData.ts
@@ -1,6 +1,6 @@
 import type { InitDateResponse } from '@/global/common/api/systemRes';
 import { getSystemInitData } from '@/web/common/system/api';
-import { delay } from '@/utils/tools';
+import { delay } from '@fastgpt/global/common/system/utils';
 import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
 import {
  defaultChatModels,
--- a/projects/app/src/web/common/system/useSystemStore.ts
+++ b/projects/app/src/web/common/system/useSystemStore.ts
@@ -59,7 +59,7 @@ export const useSystemStore = create<State>()(
            state.isPc = val;
          });
        },
-        gitStar: 3700,
+        gitStar: 6100,
        async loadGitStar() {
          try {
            const { data: git } = await axios.get('https://api.github.com/repos/labring/FastGPT');
--- a/projects/app/src/web/common/tools/api.ts
+++ b/projects/app/src/web/common/tools/api.ts
@@ -0,0 +1,5 @@
+import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
+import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
+
+export const postFetchUrls = (data: UrlFetchParams) =>
+  POST<UrlFetchResponse>(`/tools/urlFetch`, data);
--- a/projects/app/src/web/core/app/templates.ts
+++ b/projects/app/src/web/core/app/templates.ts
@@ -1095,7 +1095,7 @@ export const appTemplates: (AppItemType & {
            key: 'text',
            type: 'textarea',
            valueType: 'string',
-            value: '你好，我是 laf 助手，有什么可以帮助你的？',
+            value: '你好，有什么可以帮助你的？',
            label: '回复的内容',
            description:
              '可以使用 \\n 来实现连续换行。\n\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容',
--- a/projects/app/src/web/core/dataset/api.ts
+++ b/projects/app/src/web/core/dataset/api.ts
@@ -1,13 +1,16 @@
 import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
 import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d';
-import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
+import type { DatasetItemType, DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
 import type {
-  DatasetUpdateParams,
  GetDatasetCollectionsProps,
  GetDatasetDataListProps,
-  CreateDatasetCollectionParams,
  UpdateDatasetCollectionParams
 } from '@/global/core/api/datasetReq.d';
+import type {
+  CreateDatasetCollectionParams,
+  DatasetUpdateBody,
+  PostWebsiteSyncParams
+} from '@fastgpt/global/core/dataset/api.d';
 import type { SearchTestProps, SearchTestResponse } from '@/global/core/dataset/api.d';
 import type {
  PushDatasetDataProps,
@@ -24,12 +27,12 @@ import { PagingData } from '@/types';

 /* ======================== dataset ======================= */
 export const getDatasets = (data: { parentId?: string; type?: `${DatasetTypeEnum}` }) =>
-  GET<DatasetItemType[]>(`/core/dataset/list`, data);
+  GET<DatasetListItemType[]>(`/core/dataset/list`, data);

 /**
 * get type=dataset list
 */
-export const getAllDataset = () => GET<DatasetItemType[]>(`/core/dataset/allDataset`);
+export const getAllDataset = () => GET<DatasetListItemType[]>(`/core/dataset/allDataset`);

 export const getDatasetPaths = (parentId?: string) =>
  GET<ParentTreePathItemType[]>('/core/dataset/paths', { parentId });
@@ -39,7 +42,7 @@ export const getDatasetById = (id: string) => GET<DatasetItemType>(`/core/datase
 export const postCreateDataset = (data: CreateDatasetParams) =>
  POST<string>(`/core/dataset/create`, data);

-export const putDatasetById = (data: DatasetUpdateParams) => PUT(`/core/dataset/update`, data);
+export const putDatasetById = (data: DatasetUpdateBody) => PUT<void>(`/core/dataset/update`, data);

 export const delDatasetById = (id: string) => DELETE(`/core/dataset/delete?id=${id}`);

@@ -62,7 +65,11 @@ export const postDatasetCollection = (data: CreateDatasetCollectionParams) =>
 export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
  POST(`/core/dataset/collection/update`, data);
 export const delDatasetCollectionById = (params: { collectionId: string }) =>
-  DELETE(`/core/dataset/collection/delById`, params);
+  DELETE(`/core/dataset/collection/delete`, params);
+export const postWebsiteSync = (data: PostWebsiteSyncParams) =>
+  POST(`/plusApi/core/dataset/websiteSync`, data, {
+    timeout: 600000
+  }).catch();

 /* =============================== data ==================================== */
 /* get dataset list */
--- a/projects/app/src/web/core/dataset/store/dataset.ts
+++ b/projects/app/src/web/core/dataset/store/dataset.ts
@@ -1,20 +1,20 @@
 import { create } from 'zustand';
 import { devtools, persist } from 'zustand/middleware';
 import { immer } from 'zustand/middleware/immer';
-import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
+import type { DatasetItemType, DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
 import { getAllDataset, getDatasets, getDatasetById, putDatasetById } from '@/web/core/dataset/api';
 import { defaultDatasetDetail } from '@/constants/dataset';
-import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
+import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';

 type State = {
-  allDatasets: DatasetItemType[];
-  loadAllDatasets: () => Promise<DatasetItemType[]>;
-  myDatasets: DatasetItemType[];
+  allDatasets: DatasetListItemType[];
+  loadAllDatasets: () => Promise<DatasetListItemType[]>;
+  myDatasets: DatasetListItemType[];
  loadDatasets: (parentId?: string) => Promise<any>;
-  setDatasets(val: DatasetItemType[]): void;
+  setDatasets(val: DatasetListItemType[]): void;
  datasetDetail: DatasetItemType;
  loadDatasetDetail: (id: string, init?: boolean) => Promise<DatasetItemType>;
-  updateDataset: (data: DatasetUpdateParams) => Promise<any>;
+  updateDataset: (data: DatasetUpdateBody) => Promise<any>;
 };

 export const useDatasetStore = create<State>()(
@@ -55,10 +55,12 @@ export const useDatasetStore = create<State>()(
          return data;
        },
        async updateDataset(data) {
+          await putDatasetById(data);
+
          if (get().datasetDetail._id === data.id) {
            set((state) => {
              state.datasetDetail = {
-                ...state.datasetDetail,
+                ...get().datasetDetail,
                ...data
              };
            });
@@ -74,7 +76,6 @@ export const useDatasetStore = create<State>()(
                : item
            );
          });
-          await putDatasetById(data);
        }
      })),
      {
--- a/projects/app/src/web/core/dataset/utils.ts
+++ b/projects/app/src/web/core/dataset/utils.ts
@@ -1,6 +1,6 @@
 import { getFileViewUrl, postChunks2Dataset } from '@/web/core/dataset/api';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
-import { delay } from '@/utils/tools';
+import { delay } from '@fastgpt/global/common/system/utils';
 import { strIsLink } from '@fastgpt/global/common/string/tools';
 import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';

--- a/projects/app/src/web/styles/default.scss
+++ b/projects/app/src/web/styles/default.scss
@@ -26,3 +26,12 @@
 span[tabindex='0'] {
  line-height: 1;
 }
+
+@keyframes zoomStopIcon {
+  0% {
+    transform: scale(0.8);
+  }
+  100% {
+    transform: scale(1.2);
+  }
+}
--- a/projects/app/src/web/support/wallet/pay/api.ts
+++ b/projects/app/src/web/support/wallet/pay/api.ts
@@ -1,6 +1,6 @@
 import { GET } from '@/web/common/api/request';
 import type { PaySchema } from '@fastgpt/global/support/wallet/pay/type.d';
-import { delay } from '@/utils/tools';
+import { delay } from '@fastgpt/global/common/system/utils';

 export const getPayOrders = () => GET<PaySchema[]>(`/plusApi/support/wallet/pay/getPayOrders`);