v4.6-4 (#473)

2025-08-03 05:19:51 +00:00 · 2023-11-15 21:35:50 +08:00
parent bfd8be5df0
commit cd3acb44ab
39 changed files with 457 additions and 160 deletions
--- a/projects/app/src/pages/dataset/detail/components/DataCard.tsx
+++ b/projects/app/src/pages/dataset/detail/components/DataCard.tsx
@@ -170,7 +170,7 @@ const DataCard = () => {
      </Flex>
      <Grid
        minH={'100px'}
-        gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)']}
+        gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
        gridGap={4}
      >
        {datasetDataList.map((item) => (
--- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
@@ -34,10 +34,10 @@ const ImportData = ({
  const theme = useTheme();
  const { datasetDetail } = useDatasetStore();
  const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.chunk);
+  const vectorModel = datasetDetail.vectorModel;
+  const agentModel = datasetDetail.agentModel;

  const typeMap = useMemo(() => {
-    const vectorModel = datasetDetail.vectorModel;
-    const qaModel = qaModelList[0];
    const map = {
      [ImportTypeEnum.chunk]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
@@ -45,8 +45,8 @@ const ImportData = ({
        mode: TrainingModeEnum.chunk
      },
      [ImportTypeEnum.qa]: {
-        defaultChunkLen: qaModel?.maxContext * 0.5 || 8000,
-        unitPrice: qaModel?.price || 3,
+        defaultChunkLen: agentModel?.maxContext * 0.6 || 9000,
+        unitPrice: agentModel?.price || 3,
        mode: TrainingModeEnum.qa
      },
      [ImportTypeEnum.csv]: {
@@ -56,7 +56,13 @@ const ImportData = ({
      }
    };
    return map[importType];
-  }, [datasetDetail.vectorModel, importType]);
+  }, [
+    agentModel?.maxContext,
+    agentModel?.price,
+    importType,
+    vectorModel?.defaultToken,
+    vectorModel?.price
+  ]);

  const TitleStyle: BoxProps = {
    fontWeight: 'bold',
@@ -104,8 +110,10 @@ const ImportData = ({

        <Provider
          {...typeMap}
+          vectorModel={vectorModel.model}
+          agentModel={agentModel.model}
+          datasetId={datasetDetail._id}
          importType={importType}
-          datasetId={datasetId}
          parentId={parentId}
          onUploadSuccess={uploadSuccess}
        >
--- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
@@ -90,6 +90,8 @@ const Provider = ({
  parentId,
  unitPrice,
  mode,
+  vectorModel,
+  agentModel,
  defaultChunkLen = 500,
  importType,
  onUploadSuccess,
@@ -99,6 +101,8 @@ const Provider = ({
  parentId: string;
  unitPrice: number;
  mode: `${TrainingModeEnum}`;
+  vectorModel: string;
+  agentModel: string;
  defaultChunkLen: number;
  importType: `${ImportTypeEnum}`;
  onUploadSuccess: () => void;
@@ -132,7 +136,9 @@ const Provider = ({
        const chunks = file.chunks;
        // create training bill
        const billId = await postCreateTrainingBill({
-          name: t('dataset.collections.Create Training Data', { filename: file.filename })
+          name: t('dataset.collections.Create Training Data', { filename: file.filename }),
+          vectorModel,
+          agentModel
        });
        // create a file collection and training bill
        const collectionId = await postDatasetCollection({
--- a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
@@ -13,8 +13,8 @@ const fileExtension = '.txt, .doc, .docx, .pdf, .md';

 const QAImport = () => {
  const { datasetDetail } = useDatasetStore();
-  const vectorModel = datasetDetail.vectorModel;
-  const unitPrice = vectorModel?.price || 0.2;
+  const agentModel = datasetDetail.agentModel;
+  const unitPrice = agentModel?.price || 3;

  const {
    successChunks,
--- a/projects/app/src/pages/dataset/detail/components/Info.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Info.tsx
@@ -9,7 +9,7 @@ import React, {
 import { useRouter } from 'next/router';
 import { Box, Flex, Button, FormControl, IconButton, Input } from '@chakra-ui/react';
 import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
-import { delDatasetById, putDatasetById } from '@/web/core/dataset/api';
+import { delDatasetById } from '@/web/core/dataset/api';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
@@ -22,6 +22,8 @@ import Tag from '@/components/Tag';
 import MyTooltip from '@/components/MyTooltip';
 import { useTranslation } from 'next-i18next';
 import PermissionRadio from '@/components/support/permission/Radio';
+import MySelect from '@/components/Select';
+import { qaModelList } from '@/web/common/system/staticData';

 export interface ComponentRef {
  initInput: (tags: string) => void;
@@ -50,7 +52,7 @@ const Info = (
    multiple: false
  });

-  const { datasetDetail, loadDatasetDetail, loadDatasets } = useDatasetStore();
+  const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore();

  /* 点击删除 */
  const onclickDelKb = useCallback(async () => {
@@ -76,11 +78,10 @@ const Info = (
    async (data: DatasetItemType) => {
      setBtnLoading(true);
      try {
-        await putDatasetById({
+        await updateDataset({
          id: datasetId,
          ...data
        });
-        await loadDatasetDetail(datasetId, true);
        toast({
          title: '更新成功',
          status: 'success'
@@ -94,7 +95,7 @@ const Info = (
      }
      setBtnLoading(false);
    },
-    [loadDatasetDetail, datasetId, loadDatasets, toast]
+    [updateDataset, datasetId, loadDatasetDetail, toast, loadDatasets]
  );
  const saveSubmitError = useCallback(() => {
    // deep search message
@@ -194,6 +195,27 @@ const Info = (
          })}
        />
      </FormControl>
+      <Flex mt={6} alignItems={'center'}>
+        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
+          {t('dataset.Agent Model')}
+        </Box>
+        <Box flex={[1, '0 0 300px']}>
+          <MySelect
+            w={'100%'}
+            value={getValues('agentModel').model}
+            list={qaModelList.map((item) => ({
+              label: item.name,
+              value: item.model
+            }))}
+            onchange={(e) => {
+              const agentModel = qaModelList.find((item) => item.model === e);
+              if (!agentModel) return;
+              setValue('agentModel', agentModel);
+              setRefresh((state) => !state);
+            }}
+          />
+        </Box>
+      </Flex>
      <Flex mt={8} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
        <Box flex={['0 0 90px', '0 0 160px']} w={0}>
          标签
--- a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
@@ -196,7 +196,7 @@ const InputDataModal = ({
  const loading = useMemo(() => isImporting || isUpdating, [isImporting, isUpdating]);

  return (
-    <MyModal isOpen={true} isCentered w={'90vw'} maxW={'90vw'} h={'90vh'}>
+    <MyModal isOpen={true} isCentered w={'90vw'} maxW={'1440px'} h={'90vh'}>
      <Flex h={'100%'}>
        <Box p={5} borderRight={theme.borders.base}>
          <RawSourceText
@@ -250,7 +250,7 @@ const InputDataModal = ({
                    mt={1}
                    placeholder={`该输入框是必填项，该内容通常是对于知识点的描述，也可以是用户的问题，最多 ${maxToken} 字。`}
                    maxLength={maxToken}
-                    rows={10}
+                    rows={12}
                    bg={'myWhite.400'}
                    {...register(`q`, {
                      required: true
@@ -274,7 +274,7 @@ const InputDataModal = ({
                      maxToken * 1.5
                    } 字。`}
                    bg={'myWhite.400'}
-                    rows={10}
+                    rows={12}
                    maxLength={maxToken * 1.5}
                    {...register('a')}
                  />
--- a/projects/app/src/pages/dataset/list/component/CreateModal.tsx
+++ b/projects/app/src/pages/dataset/list/component/CreateModal.tsx
@@ -15,10 +15,12 @@ import { postCreateDataset } from '@/web/core/dataset/api';
 import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
 import MySelect from '@/components/Select';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { vectorModelList } from '@/web/common/system/staticData';
+import { vectorModelList, qaModelList } from '@/web/common/system/staticData';
 import Tag from '@/components/Tag';
+import { useTranslation } from 'next-i18next';

 const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
+  const { t } = useTranslation();
  const [refresh, setRefresh] = useState(false);
  const { toast } = useToast();
  const router = useRouter();
@@ -29,6 +31,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
      name: '',
      tags: '',
      vectorModel: vectorModelList[0].model,
+      agentModel: qaModelList[0].model,
      type: 'dataset',
      parentId
    }
@@ -76,7 +79,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
  });

  return (
-    <MyModal isOpen onClose={onClose} isCentered={!isPc} w={'400px'}>
+    <MyModal isOpen onClose={onClose} isCentered={!isPc} w={'450px'}>
      <ModalHeader fontSize={'2xl'}>创建一个知识库</ModalHeader>
      <ModalBody>
        <Box color={'myGray.800'} fontWeight={'bold'}>
@@ -106,7 +109,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
          />
        </Flex>
        <Flex mt={6} alignItems={'center'}>
-          <Box flex={'0 0 80px'}>索引模型</Box>
+          <Box flex={'0 0 100px'}>索引模型</Box>
          <Box flex={1}>
            <MySelect
              w={'100%'}
@@ -122,8 +125,25 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
            />
          </Box>
        </Flex>
+        <Flex mt={6} alignItems={'center'}>
+          <Box flex={'0 0 100px'}>{t('dataset.Agent Model')}</Box>
+          <Box flex={1}>
+            <MySelect
+              w={'100%'}
+              value={getValues('agentModel')}
+              list={qaModelList.map((item) => ({
+                label: item.name,
+                value: item.model
+              }))}
+              onchange={(e) => {
+                setValue('agentModel', e);
+                setRefresh((state) => !state);
+              }}
+            />
+          </Box>
+        </Flex>
        <Flex mt={6} alignItems={'center'} w={'100%'}>
-          <Box flex={'0 0 80px'}>
+          <Box flex={'0 0 100px'}>
            标签
            <MyTooltip label={'用空格隔开多个标签，便于搜索'} forceShow>
              <QuestionOutlineIcon ml={1} />
--- a/projects/app/src/pages/dataset/list/index.tsx
+++ b/projects/app/src/pages/dataset/list/index.tsx
@@ -20,7 +20,8 @@ import {
  delDatasetById,
  getDatasetPaths,
  putDatasetById,
-  postCreateDataset
+  postCreateDataset,
+  getCheckExportLimit
 } from '@/web/core/dataset/api';
 import { useTranslation } from 'next-i18next';
 import Avatar from '@/components/Avatar';
@@ -38,6 +39,7 @@ import { useDrag } from '@/web/common/hooks/useDrag';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import PermissionIconText from '@/components/support/permission/IconText';
 import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
+import { DatasetItemType } from '@fastgpt/global/core/dataset/type';

 const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false });
 const MoveModal = dynamic(() => import('./component/MoveModal'), { ssr: false });
@@ -89,6 +91,23 @@ const Kb = () => {
    successToast: t('common.Delete Success'),
    errorToast: t('dataset.Delete Dataset Error')
  });
+  // check export limit
+  const { mutate: exportDataset } = useRequest({
+    mutationFn: async (dataset: DatasetItemType) => {
+      setLoading(true);
+      await getCheckExportLimit(dataset._id);
+      const a = document.createElement('a');
+      a.href = `/api/core/dataset/exportAll?datasetId=${dataset._id}`;
+      a.download = `${dataset.name}.csv`;
+      document.body.appendChild(a);
+      a.click();
+      document.body.removeChild(a);
+    },
+    onSettled() {
+      setLoading(false);
+    },
+    errorToast: t('dataset.Export Dataset Limit Error')
+  });

  const { data, refetch } = useQuery(['loadDataset', parentId], () => {
    return Promise.all([loadDatasets(parentId), getDatasetPaths(parentId)]);
@@ -371,12 +390,7 @@ const Kb = () => {
                      </Flex>
                    ),
                    onClick: () => {
-                      const a = document.createElement('a');
-                      a.href = `/api/core/dataset/exportAll?datasetId=${dataset._id}`;
-                      a.download = `${dataset.name}.csv`;
-                      document.body.appendChild(a);
-                      a.click();
-                      document.body.removeChild(a);
+                      exportDataset(dataset);
                    }
                  },
                  {