v4.6-3 (#471)

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -2,7 +2,7 @@ import MyIcon from '@/components/Icon';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useToast } from '@/web/common/hooks/useToast';
-import { splitText2Chunks } from '@/global/common/string/tools';
+import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { simpleText } from '@fastgpt/global/common/string/tools';
 import {
   fileDownload,
@@ -19,15 +19,13 @@ import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
 import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
-import type {
-  DatasetChunkItemType,
-  DatasetCollectionSchemaType
-} from '@fastgpt/global/core/dataset/type';
+import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
-import { countPromptTokens } from '@/global/common/tiktoken';
+import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
 
 const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
 const CreateFileModal = dynamic(() => import('./CreateFileModal'));
@@ -37,7 +35,7 @@ const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
 export type FileItemType = {
   id: string; // fileId / raw Link
   filename: string;
-  chunks: DatasetChunkItemType[];
+  chunks: PushDatasetDataChunkProps[];
   text: string; // raw text
   icon: string;
   tokens: number; // total tokens
@@ -152,7 +150,6 @@ const FileSelect = ({
                 fileId
               }
             };
-            console.log(fileItem);
 
             onPushFiles([fileItem]);
             continue;

projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx

@@ -3,7 +3,7 @@ import { Box, type BoxProps, Flex, useTheme, ModalCloseButton } from '@chakra-ui
 import MyRadio from '@/components/Radio/index';
 import dynamic from 'next/dynamic';
 import ChunkImport from './Chunk';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 
 const QAImport = dynamic(() => import('./QA'), {});
 const CsvImport = dynamic(() => import('./Csv'), {});
@@ -14,7 +14,7 @@ import { qaModelList } from '@/web/common/system/staticData';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
 
 export enum ImportTypeEnum {
-  index = 'index',
+  chunk = 'chunk',
   qa = 'qa',
   csv = 'csv'
 }
@@ -33,16 +33,16 @@ const ImportData = ({
   const { t } = useTranslation();
   const theme = useTheme();
   const { datasetDetail } = useDatasetStore();
-  const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.index);
+  const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.chunk);
 
   const typeMap = useMemo(() => {
     const vectorModel = datasetDetail.vectorModel;
     const qaModel = qaModelList[0];
     const map = {
-      [ImportTypeEnum.index]: {
+      [ImportTypeEnum.chunk]: {
         defaultChunkLen: vectorModel?.defaultToken || 500,
         unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.index
+        mode: TrainingModeEnum.chunk
       },
       [ImportTypeEnum.qa]: {
         defaultChunkLen: qaModel?.maxContext * 0.5 || 8000,
@@ -52,7 +52,7 @@ const ImportData = ({
       [ImportTypeEnum.csv]: {
         defaultChunkLen: vectorModel?.defaultToken || 500,
         unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.index
+        mode: TrainingModeEnum.chunk
       }
     };
     return map[importType];
@@ -82,7 +82,7 @@ const ImportData = ({
                 icon: 'indexImport',
                 title: '直接分段',
                 desc: '选择文本文件，直接将其按分段进行处理',
-                value: ImportTypeEnum.index
+                value: ImportTypeEnum.chunk
               },
               {
                 icon: 'qaImport',
@@ -110,7 +110,7 @@ const ImportData = ({
           onUploadSuccess={uploadSuccess}
         >
           <Box flex={'1 0 0'} h={0}>
-            {importType === ImportTypeEnum.index && <ChunkImport />}
+            {importType === ImportTypeEnum.chunk && <ChunkImport />}
             {importType === ImportTypeEnum.qa && <QAImport />}
             {importType === ImportTypeEnum.csv && <CsvImport />}
           </Box>

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -12,7 +12,7 @@ import FileSelect, { FileItemType, Props as FileSelectProps } from './FileSelect
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { postDatasetCollection } from '@/web/core/dataset/api';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
-import { splitText2Chunks } from '@/global/common/string/tools';
+import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { useToast } from '@/web/common/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
@@ -22,7 +22,7 @@ import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
 import MyIcon from '@/components/Icon';
 import { chunksUpload } from '@/web/core/dataset/utils';
 import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { ImportTypeEnum } from './ImportModal';
 
 const filenameStyles = {
@@ -39,7 +39,7 @@ type useImportStoreType = {
   setSuccessChunks: Dispatch<SetStateAction<number>>;
   isUnselectedFile: boolean;
   totalChunks: number;
-  onclickUpload: (e: { files: FileItemType[] }) => void;
+  onclickUpload: (e: { prompt?: string }) => void;
   onReSplitChunks: () => void;
   price: number;
   uploading: boolean;
@@ -49,7 +49,7 @@ type useImportStoreType = {
   setReShowRePreview: Dispatch<SetStateAction<boolean>>;
 };
 const StateContext = createContext<useImportStoreType>({
-  onclickUpload: function (e: { files: FileItemType[] }): void {
+  onclickUpload: function (e: { prompt?: string }): void {
     throw new Error('Function not implemented.');
   },
   uploading: false,
@@ -125,7 +125,8 @@ const Provider = ({
 
   /* start upload data */
   const { mutate: onclickUpload, isLoading: uploading } = useRequest({
-    mutationFn: async () => {
+    mutationFn: async (props?: { prompt?: string }) => {
+      const { prompt } = props || {};
       let totalInsertion = 0;
       for await (const file of files) {
         const chunks = file.chunks;
@@ -150,7 +151,8 @@ const Provider = ({
           mode,
           onUploading: (insertLen) => {
             setSuccessChunks((state) => state + insertLen);
-          }
+          },
+          prompt
         });
         totalInsertion += insertLen;
       }

projects/app/src/pages/dataset/detail/components/Import/QA.tsx

@@ -5,7 +5,7 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons';
 import { Prompt_AgentQA } from '@/global/core/prompt/agent';
-import { replaceVariable } from '@/global/common/string/tools';
+import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 
@@ -81,7 +81,10 @@ const QAImport = () => {
               重新生成预览
             </Button>
           )}
-          <Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
+          <Button
+            isDisabled={uploading}
+            onClick={openConfirm(() => onclickUpload({ prompt: previewQAPrompt }))}
+          >
             {uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
           </Button>
         </Flex>