4.6.3-alpha1 (#529)

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -19,7 +19,6 @@ import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
 import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
-import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
@@ -40,7 +39,8 @@ export type FileItemType = {
   icon: string;
   tokens: number; // total tokens
   type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
-  metadata: DatasetCollectionSchemaType['metadata'];
+  fileId?: string;
+  rawLink?: string;
 };
 
 export interface Props extends BoxProps {
@@ -157,9 +157,7 @@ const FileSelect = ({
                 .join('\n')}`,
               chunks: filterData,
               type: DatasetCollectionTypeEnum.file,
-              metadata: {
-                fileId
-              }
+              fileId
             };
 
             onPushFiles([fileItem]);
@@ -195,9 +193,7 @@ const FileSelect = ({
               text,
               tokens: splitRes.tokens,
               type: DatasetCollectionTypeEnum.file,
-              metadata: {
-                fileId
-              },
+              fileId,
               chunks: splitRes.chunks.map((chunk) => ({
                 q: chunk,
                 a: ''
@@ -220,7 +216,7 @@ const FileSelect = ({
   // link fetch
   const onUrlFetch = useCallback(
     (e: FetchResultItem[]) => {
-      const result: FileItemType[] = e.map(({ url, content }) => {
+      const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
         const splitRes = splitText2Chunks({
           text: content,
           chunkLen,
@@ -233,9 +229,7 @@ const FileSelect = ({
           text: content,
           tokens: splitRes.tokens,
           type: DatasetCollectionTypeEnum.link,
-          metadata: {
-            rawLink: url
-          },
+          rawLink: url,
           chunks: splitRes.chunks.map((chunk) => ({
             q: chunk,
             a: ''
@@ -277,9 +271,7 @@ const FileSelect = ({
           text: content,
           tokens: splitRes.tokens,
           type: DatasetCollectionTypeEnum.file,
-          metadata: {
-            fileId: fileIds[0]
-          },
+          fileId: fileIds[0],
           chunks: splitRes.chunks.map((chunk) => ({
             q: chunk,
             a: ''

projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx

@@ -11,7 +11,10 @@ import MyModal from '@/components/MyModal';
 import Provider from './Provider';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { qaModelList } from '@/web/common/system/staticData';
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionTrainingModeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constant';
 
 export enum ImportTypeEnum {
   chunk = 'chunk',
@@ -43,19 +46,22 @@ const ImportData = ({
         defaultChunkLen: vectorModel?.defaultToken || 500,
         chunkOverlapRatio: 0.2,
         unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.chunk
+        mode: TrainingModeEnum.chunk,
+        collectionTrainingType: DatasetCollectionTrainingModeEnum.chunk
       },
       [ImportTypeEnum.qa]: {
         defaultChunkLen: agentModel?.maxContext * 0.6 || 8000,
         chunkOverlapRatio: 0,
         unitPrice: agentModel?.price || 3,
-        mode: TrainingModeEnum.qa
+        mode: TrainingModeEnum.qa,
+        collectionTrainingType: DatasetCollectionTrainingModeEnum.qa
       },
       [ImportTypeEnum.csv]: {
-        defaultChunkLen: vectorModel?.defaultToken || 500,
+        defaultChunkLen: 0,
         chunkOverlapRatio: 0,
         unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.chunk
+        mode: TrainingModeEnum.chunk,
+        collectionTrainingType: DatasetCollectionTrainingModeEnum.manual
       }
     };
     return map[importType];

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -15,7 +15,10 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { useToast } from '@/web/common/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionTrainingModeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import { Box, Flex, Image, useTheme } from '@chakra-ui/react';
 import { CloseIcon } from '@chakra-ui/icons';
 import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
@@ -92,6 +95,7 @@ const Provider = ({
   parentId,
   unitPrice,
   mode,
+  collectionTrainingType,
   vectorModel,
   agentModel,
   defaultChunkLen = 500,
@@ -104,6 +108,7 @@ const Provider = ({
   parentId: string;
   unitPrice: number;
   mode: `${TrainingModeEnum}`;
+  collectionTrainingType: `${DatasetCollectionTrainingModeEnum}`;
   vectorModel: string;
   agentModel: string;
   defaultChunkLen: number;
@@ -150,7 +155,10 @@ const Provider = ({
           parentId,
           name: file.filename,
           type: file.type,
-          metadata: file.metadata
+          fileId: file.fileId,
+          rawLink: file.rawLink,
+          chunkSize: chunkLen,
+          trainingType: collectionTrainingType
         });
 
         // upload data