v4.6-3 (#471)

2025-08-05 22:55:27 +00:00 · 2023-11-15 11:36:25 +08:00
parent 592e1a93a2
commit bfd8be5df0
181 changed files with 2499 additions and 1552 deletions
--- a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx
+++ b/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx
@@ -11,7 +11,6 @@ import {
  Tbody,
  Image,
  MenuButton,
-  useTheme,
  useDisclosure
 } from '@chakra-ui/react';
 import {
@@ -24,7 +23,7 @@ import {
 import { useQuery } from '@tanstack/react-query';
 import { debounce } from 'lodash';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import MyIcon from '@/components/Icon';
 import MyInput from '@/components/MyInput';
 import dayjs from 'dayjs';
@@ -35,13 +34,10 @@ import { usePagination } from '@/web/common/hooks/usePagination';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyMenu from '@/components/MyMenu';
 import { useEditTitle } from '@/web/common/hooks/useEditTitle';
-import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
+import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
 import EmptyTip from '@/components/EmptyTip';
-import { AddIcon } from '@chakra-ui/icons';
 import { FolderAvatarSrc, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
-
 import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
-
 import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
 import { TabEnum } from '..';
 import ParentPath from '@/components/common/ParentPaths';
@@ -423,7 +419,11 @@ const CollectionCard = () => {
                    </MyTooltip>
                  </Flex>
                </Td>
-                <Td fontSize={'md'}>{collection.dataAmount ?? '-'}</Td>
+                <Td fontSize={'md'}>
+                  {collection.type === DatasetCollectionTypeEnum.folder
+                    ? '-'
+                    : collection.dataAmount}
+                </Td>
                <Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
                <Td>
                  <Flex
--- a/projects/app/src/pages/dataset/detail/components/DataCard.tsx
+++ b/projects/app/src/pages/dataset/detail/components/DataCard.tsx
@@ -12,16 +12,17 @@ import { useToast } from '@/web/common/hooks/useToast';
 import { debounce } from 'lodash';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { useRouter } from 'next/router';
 import MyIcon from '@/components/Icon';
 import MyInput from '@/components/MyInput';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import InputDataModal, { RawSourceText, type InputDataType } from '../components/InputDataModal';
-import type { DatasetDataListItemType } from '@/global/core/dataset/response.d';
+import type { DatasetDataListItemType } from '@/global/core/dataset/type.d';
 import { TabEnum } from '..';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
+import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';

 const DataCard = () => {
  const BoxRef = useRef<HTMLDivElement>(null);
@@ -127,9 +128,8 @@ const DataCard = () => {
              onClick={() => {
                if (!collection) return;
                setEditInputData({
-                  collectionId: collection._id,
-                  sourceId: collection.metadata?.fileId || collection.metadata?.rawLink,
-                  sourceName: collection.name
+                  q: '',
+                  indexes: [getDefaultIndex({ dataId: `${Date.now()}` })]
                });
              }}
            >
@@ -175,7 +175,7 @@ const DataCard = () => {
      >
        {datasetDataList.map((item) => (
          <Card
-            key={item.id}
+            key={item._id}
            cursor={'pointer'}
            pt={3}
            userSelect={'none'}
@@ -186,12 +186,10 @@ const DataCard = () => {
            onClick={() => {
              if (!collection) return;
              setEditInputData({
-                id: item.id,
-                collectionId: collection._id,
+                id: item._id,
                q: item.q,
                a: item.a,
-                sourceId: collection.metadata?.fileId || collection.metadata?.rawLink,
-                sourceName: collection.name
+                indexes: item.indexes
              });
            }}
          >
@@ -210,7 +208,7 @@ const DataCard = () => {
            </Box>
            <Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
              <Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
-                ID:{item.id}
+                ID:{item._id}
              </Box>
              {canWrite && (
                <IconButton
@@ -228,7 +226,7 @@ const DataCard = () => {
                    openConfirm(async () => {
                      try {
                        setIsLoading(true);
-                        await delOneDatasetDataById(item.id);
+                        await delOneDatasetDataById(item._id);
                        getData(pageNum);
                      } catch (error) {
                        toast({
@@ -262,11 +260,11 @@ const DataCard = () => {

      {editInputData !== undefined && collection && (
        <InputDataModal
-          datasetId={collection?.datasetId}
-          defaultValues={editInputData}
+          collectionId={collection._id}
+          defaultValue={editInputData}
          onClose={() => setEditInputData(undefined)}
          onSuccess={() => getData(pageNum)}
-          canWrite={canWrite}
+          onDelete={() => getData(pageNum)}
        />
      )}
      <ConfirmModal />
--- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
@@ -2,7 +2,7 @@ import MyIcon from '@/components/Icon';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
 import { useToast } from '@/web/common/hooks/useToast';
-import { splitText2Chunks } from '@/global/common/string/tools';
+import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { simpleText } from '@fastgpt/global/common/string/tools';
 import {
  fileDownload,
@@ -19,15 +19,13 @@ import { customAlphabet } from 'nanoid';
 import dynamic from 'next/dynamic';
 import MyTooltip from '@/components/MyTooltip';
 import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
-import type {
-  DatasetChunkItemType,
-  DatasetCollectionSchemaType
-} from '@fastgpt/global/core/dataset/type';
+import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
-import { countPromptTokens } from '@/global/common/tiktoken';
+import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';

 const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
 const CreateFileModal = dynamic(() => import('./CreateFileModal'));
@@ -37,7 +35,7 @@ const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
 export type FileItemType = {
  id: string; // fileId / raw Link
  filename: string;
-  chunks: DatasetChunkItemType[];
+  chunks: PushDatasetDataChunkProps[];
  text: string; // raw text
  icon: string;
  tokens: number; // total tokens
@@ -152,7 +150,6 @@ const FileSelect = ({
                fileId
              }
            };
-            console.log(fileItem);

            onPushFiles([fileItem]);
            continue;
--- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
@@ -3,7 +3,7 @@ import { Box, type BoxProps, Flex, useTheme, ModalCloseButton } from '@chakra-ui
 import MyRadio from '@/components/Radio/index';
 import dynamic from 'next/dynamic';
 import ChunkImport from './Chunk';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';

 const QAImport = dynamic(() => import('./QA'), {});
 const CsvImport = dynamic(() => import('./Csv'), {});
@@ -14,7 +14,7 @@ import { qaModelList } from '@/web/common/system/staticData';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';

 export enum ImportTypeEnum {
-  index = 'index',
+  chunk = 'chunk',
  qa = 'qa',
  csv = 'csv'
 }
@@ -33,16 +33,16 @@ const ImportData = ({
  const { t } = useTranslation();
  const theme = useTheme();
  const { datasetDetail } = useDatasetStore();
-  const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.index);
+  const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.chunk);

  const typeMap = useMemo(() => {
    const vectorModel = datasetDetail.vectorModel;
    const qaModel = qaModelList[0];
    const map = {
-      [ImportTypeEnum.index]: {
+      [ImportTypeEnum.chunk]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
        unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.index
+        mode: TrainingModeEnum.chunk
      },
      [ImportTypeEnum.qa]: {
        defaultChunkLen: qaModel?.maxContext * 0.5 || 8000,
@@ -52,7 +52,7 @@ const ImportData = ({
      [ImportTypeEnum.csv]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
        unitPrice: vectorModel?.price || 0.2,
-        mode: TrainingModeEnum.index
+        mode: TrainingModeEnum.chunk
      }
    };
    return map[importType];
@@ -82,7 +82,7 @@ const ImportData = ({
                icon: 'indexImport',
                title: '直接分段',
                desc: '选择文本文件，直接将其按分段进行处理',
-                value: ImportTypeEnum.index
+                value: ImportTypeEnum.chunk
              },
              {
                icon: 'qaImport',
@@ -110,7 +110,7 @@ const ImportData = ({
          onUploadSuccess={uploadSuccess}
        >
          <Box flex={'1 0 0'} h={0}>
-            {importType === ImportTypeEnum.index && <ChunkImport />}
+            {importType === ImportTypeEnum.chunk && <ChunkImport />}
            {importType === ImportTypeEnum.qa && <QAImport />}
            {importType === ImportTypeEnum.csv && <CsvImport />}
          </Box>
--- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
@@ -12,7 +12,7 @@ import FileSelect, { FileItemType, Props as FileSelectProps } from './FileSelect
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { postDatasetCollection } from '@/web/core/dataset/api';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
-import { splitText2Chunks } from '@/global/common/string/tools';
+import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { useToast } from '@/web/common/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
@@ -22,7 +22,7 @@ import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
 import MyIcon from '@/components/Icon';
 import { chunksUpload } from '@/web/core/dataset/utils';
 import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { ImportTypeEnum } from './ImportModal';

 const filenameStyles = {
@@ -39,7 +39,7 @@ type useImportStoreType = {
  setSuccessChunks: Dispatch<SetStateAction<number>>;
  isUnselectedFile: boolean;
  totalChunks: number;
-  onclickUpload: (e: { files: FileItemType[] }) => void;
+  onclickUpload: (e: { prompt?: string }) => void;
  onReSplitChunks: () => void;
  price: number;
  uploading: boolean;
@@ -49,7 +49,7 @@ type useImportStoreType = {
  setReShowRePreview: Dispatch<SetStateAction<boolean>>;
 };
 const StateContext = createContext<useImportStoreType>({
-  onclickUpload: function (e: { files: FileItemType[] }): void {
+  onclickUpload: function (e: { prompt?: string }): void {
    throw new Error('Function not implemented.');
  },
  uploading: false,
@@ -125,7 +125,8 @@ const Provider = ({

  /* start upload data */
  const { mutate: onclickUpload, isLoading: uploading } = useRequest({
-    mutationFn: async () => {
+    mutationFn: async (props?: { prompt?: string }) => {
+      const { prompt } = props || {};
      let totalInsertion = 0;
      for await (const file of files) {
        const chunks = file.chunks;
@@ -150,7 +151,8 @@ const Provider = ({
          mode,
          onUploading: (insertLen) => {
            setSuccessChunks((state) => state + insertLen);
-          }
+          },
+          prompt
        });
        totalInsertion += insertLen;
      }
--- a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
@@ -5,7 +5,7 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons';
 import { Prompt_AgentQA } from '@/global/core/prompt/agent';
-import { replaceVariable } from '@/global/common/string/tools';
+import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';

@@ -81,7 +81,10 @@ const QAImport = () => {
              重新生成预览
            </Button>
          )}
-          <Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
+          <Button
+            isDisabled={uploading}
+            onClick={openConfirm(() => onclickUpload({ prompt: previewQAPrompt }))}
+          >
            {uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
          </Button>
        </Flex>
--- a/projects/app/src/pages/dataset/detail/components/Info.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Info.tsx
@@ -16,11 +16,11 @@ import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { UseFormReturn } from 'react-hook-form';
 import { compressImgAndUpload } from '@/web/common/file/controller';
-import type { DatasetItemType } from '@/types/core/dataset';
+import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
 import Avatar from '@/components/Avatar';
 import Tag from '@/components/Tag';
 import MyTooltip from '@/components/MyTooltip';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import PermissionRadio from '@/components/support/permission/Radio';

 export interface ComponentRef {
@@ -208,13 +208,15 @@ const Info = (
          placeholder={'标签,使用空格分割。'}
          maxLength={30}
          onChange={(e) => {
-            setValue('tags', e.target.value);
+            setValue(
+              'tags',
+              e.target.value.split(' ').filter((item) => item)
+            );
            setRefresh(!refresh);
          }}
        />
        <Flex w={'100%'} pl={['90px', '160px']} mt={2}>
          {getValues('tags')
-            .split(' ')
            .filter((item) => item)
            .map((item, i) => (
              <Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
--- a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
@@ -1,10 +1,11 @@
-import React, { useMemo } from 'react';
-import { Box, Flex, Button, Textarea, IconButton, BoxProps, Image, Link } from '@chakra-ui/react';
-import { useForm } from 'react-hook-form';
+import React, { useMemo, useState } from 'react';
+import { Box, Flex, Button, Textarea, BoxProps, Image, useTheme, Grid } from '@chakra-ui/react';
+import { useFieldArray, useForm } from 'react-hook-form';
 import {
-  postData2Dataset,
+  postInsertData2Dataset,
  putDatasetDataById,
-  delOneDatasetDataById
+  delOneDatasetDataById,
+  getDatasetCollectionById
 } from '@/web/core/dataset/api';
 import { useToast } from '@/web/common/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
@@ -13,62 +14,106 @@ import MyModal from '@/components/MyModal';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useQuery } from '@tanstack/react-query';
-import { useTranslation } from 'react-i18next';
-import { useDatasetStore } from '@/web/core/dataset/store/dataset';
+import { useTranslation } from 'next-i18next';
 import { getFileAndOpen } from '@/web/core/dataset/utils';
-import { strIsLink } from '@fastgpt/global/common/string/tools';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq';
 import { useRequest } from '@/web/common/hooks/useRequest';
-import { countPromptTokens } from '@/global/common/tiktoken';
+import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
-import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
-import { feConfigs } from '@/web/common/system/staticData';
+import { getDefaultIndex, getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
+import { feConfigs, vectorModelList } from '@/web/common/system/staticData';
+import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
+import SideTabs from '@/components/SideTabs';
+import { useLoading } from '@/web/common/hooks/useLoading';
+import DeleteIcon from '@/components/Icon/delete';
+import { defaultCollectionDetail } from '@/constants/dataset';

-export type RawSourceType = {
+export type RawSourceTextProps = BoxProps & {
  sourceName?: string;
  sourceId?: string;
-  addr?: boolean;
+  canView?: boolean;
 };
-export type RawSourceTextProps = BoxProps & RawSourceType;
-export type InputDataType = SetOneDatasetDataProps & RawSourceType;
+export type InputDataType = {
+  id?: string;
+  q: string;
+  a?: string;
+  indexes: (Omit<DatasetDataIndexItemType, 'dataId'> & {
+    dataId?: string; // pg data id
+  })[];
+};
+
+enum TabEnum {
+  content = 'content',
+  index = 'index',
+  delete = 'delete',
+  doc = 'doc'
+}

 const InputDataModal = ({
+  collectionId,
+  defaultValue,
  onClose,
  onSuccess,
-  onDelete,
-  datasetId,
-  defaultValues = {
-    collectionId: '',
-    sourceId: '',
-    sourceName: ''
-  },
-  canWrite
+  onDelete
 }: {
+  collectionId: string;
+  defaultValue: InputDataType;
  onClose: () => void;
-  onSuccess: (data: SetOneDatasetDataProps) => void;
+  onSuccess: (data: InputDataType) => void;
  onDelete?: () => void;
-  datasetId: string;
-  defaultValues: InputDataType;
-  canWrite: boolean;
 }) => {
  const { t } = useTranslation();
+  const theme = useTheme();
  const { toast } = useToast();
-  const { datasetDetail, loadDatasetDetail } = useDatasetStore();
+  const { Loading } = useLoading();
+  const [currentTab, setCurrentTab] = useState(TabEnum.content);

-  const { register, handleSubmit, reset } = useForm<InputDataType>({
-    defaultValues
+  const { register, handleSubmit, reset, control } = useForm<InputDataType>({
+    defaultValues: defaultValue
  });
+  const {
+    fields: indexes,
+    append: appendIndexes,
+    remove: removeIndexes
+  } = useFieldArray({
+    control,
+    name: 'indexes'
+  });
+
+  const tabList = [
+    { label: t('dataset.data.edit.Content'), id: TabEnum.content, icon: 'overviewLight' },
+    {
+      label: t('dataset.data.edit.Index', { amount: indexes.length }),
+      id: TabEnum.index,
+      icon: 'kbTest'
+    },
+    ...(defaultValue.id
+      ? [{ label: t('dataset.data.edit.Delete'), id: TabEnum.delete, icon: 'delete' }]
+      : []),
+    { label: t('dataset.data.edit.Course'), id: TabEnum.doc, icon: 'courseLight' }
+  ];

  const { ConfirmModal, openConfirm } = useConfirm({
    content: t('dataset.data.Delete Tip')
  });

-  const maxToken = datasetDetail.vectorModel?.maxToken || 2000;
+  const { data: collection = defaultCollectionDetail } = useQuery(
+    ['loadCollectionId', collectionId],
+    () => {
+      return getDatasetCollectionById(collectionId);
+    }
+  );

-  /**
-   * 确认导入新数据
-   */
+  const maxToken = useMemo(() => {
+    const vectorModel =
+      vectorModelList.find((item) => item.model === collection.datasetId.vectorModel) ||
+      vectorModelList[0];
+
+    return vectorModel?.maxToken || 3000;
+  }, [collection.datasetId.vectorModel]);
+
+  // import new data
  const { mutate: sureImportData, isLoading: isImporting } = useRequest({
    mutationFn: async (e: InputDataType) => {
      if (!e.q) {
@@ -85,10 +130,16 @@ const InputDataModal = ({
      }

      const data = { ...e };
-      delete data.sourceName;
-      delete data.sourceId;

-      data.id = await postData2Dataset(data);
+      data.id = await postInsertData2Dataset({
+        collectionId: collection._id,
+        q: e.q,
+        a: e.a,
+        // remove dataId
+        indexes: e.indexes.map((index) =>
+          index.defaultIndex ? getDefaultIndex({ q: e.q, a: e.a }) : index
+        )
+      });

      return data;
    },
@@ -97,26 +148,26 @@ const InputDataModal = ({
      reset({
        ...e,
        q: '',
-        a: ''
+        a: '',
+        indexes: [getDefaultIndex({ q: e.q, a: e.a, dataId: `${Date.now()}` })]
      });

      onSuccess(e);
    },
    errorToast: t('common.error.unKnow')
  });
-
+  // update
  const { mutate: onUpdateData, isLoading: isUpdating } = useRequest({
-    mutationFn: async (e: SetOneDatasetDataProps) => {
+    mutationFn: async (e: InputDataType) => {
      if (!e.id) return e;

      // not exactly same
-      if (e.q !== defaultValues.q || e.a !== defaultValues.a) {
-        await putDatasetDataById({
-          ...e,
-          q: e.q === defaultValues.q ? '' : e.q
-        });
-        return e;
-      }
+      await putDatasetDataById({
+        id: e.id,
+        q: e.q,
+        a: e.a,
+        indexes: e.indexes
+      });

      return e;
    },
@@ -127,153 +178,207 @@ const InputDataModal = ({
      onClose();
    }
  });
+  // delete
+  const { mutate: onDeleteData, isLoading: isDeleting } = useRequest({
+    mutationFn: () => {
+      if (!onDelete || !defaultValue.id) return Promise.resolve(null);
+      return delOneDatasetDataById(defaultValue.id);
+    },
+    onSuccess() {
+      if (!onDelete) return;
+      onDelete();
+      onClose();
+    },
+    successToast: t('common.Delete Success'),
+    errorToast: t('common.error.unKnow')
+  });

  const loading = useMemo(() => isImporting || isUpdating, [isImporting, isUpdating]);

-  useQuery(['loadDatasetDetail'], () => {
-    if (datasetDetail._id === datasetId) return null;
-    return loadDatasetDetail(datasetId);
-  });
-
  return (
-    <MyModal
-      isOpen={true}
-      isCentered
-      title={
-        <Flex alignItems={'flex-end'}>
-          <Box>
-            {defaultValues.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')}
-          </Box>
-          <Link
-            href={`${feConfigs.docUrl}/docs/use-cases/datasetengine`}
-            target={'_blank'}
-            fontSize={'sm'}
-            color={'myGray.600'}
-            textDecor={'underline'}
-            ml={2}
-          >
-            结构详解
-          </Link>
-        </Flex>
-      }
-      w={'90vw'}
-      maxW={'90vw'}
-      h={'90vh'}
-    >
-      <Flex flexDirection={'column'} h={'100%'}>
-        <Box
-          display={'flex'}
-          flexDirection={['column', 'row']}
-          flex={'1 0 0'}
-          h={['100%', 0]}
-          overflow={'overlay'}
-          px={6}
-          pb={2}
-        >
-          <Box flex={1} mr={[0, 4]} mb={[4, 0]} h={['50%', '100%']}>
-            <Flex>
-              <Box h={'25px'}>{'被搜索的内容'}</Box>
-              <MyTooltip
-                label={
-                  '被向量化的部分，该部分的质量决定了对话时，能否高效的查找到合适的知识点。\n该内容通常是问题，或是一段陈述描述介绍'
-                }
-              >
-                <QuestionOutlineIcon ml={1} />
-              </MyTooltip>
-            </Flex>
-            <Textarea
-              placeholder={`被向量化的部分，该部分的质量决定了对话时，能否高效的查找到合适的知识点。\n该内容通常是问题，或是一段陈述描述介绍，最多 ${maxToken} 字。`}
-              maxLength={maxToken}
-              resize={'none'}
-              h={'calc(100% - 30px)'}
-              {...register(`q`, {
-                required: true
-              })}
-            />
-          </Box>
-          <Box flex={1} h={['50%', '100%']}>
-            <Flex>
-              <Box h={'25px'}>{'补充内容(可选)'}</Box>
-              <MyTooltip
-                label={
-                  '该部分内容不影响搜索质量。当“被搜索的内容”被搜索到后，“补充内容”可以选择性被填入提示词，从而实现更加丰富的提示词组合。'
-                }
-              >
-                <QuestionOutlineIcon ml={1} />
-              </MyTooltip>
-            </Flex>
-            <Textarea
-              placeholder={
-                '该部分内容不影响搜索质量。当“被搜索的内容”被搜索到后，“补充内容”可以选择性被填入提示词，从而实现更加丰富的提示词组合。可以是问题的答案、代码、图片、表格等。'
-              }
-              resize={'none'}
-              h={'calc(100% - 30px)'}
-              {...register('a')}
-            />
-          </Box>
-        </Box>
-
-        <Flex px={6} pt={['34px', 2]} pb={4} alignItems={'center'} position={'relative'}>
+    <MyModal isOpen={true} isCentered w={'90vw'} maxW={'90vw'} h={'90vh'}>
+      <Flex h={'100%'}>
+        <Box p={5} borderRight={theme.borders.base}>
          <RawSourceText
-            sourceName={defaultValues.sourceName}
-            sourceId={defaultValues.sourceId}
-            position={'absolute'}
-            left={'50%'}
-            top={['16px', '50%']}
-            transform={'translate(-50%,-50%)'}
+            w={'200px'}
+            className=""
+            whiteSpace={'pre-wrap'}
+            sourceName={collection.sourceName}
+            sourceId={collection.sourceId}
+            mb={6}
+            fontSize={['14px', '16px']}
          />
-
-          <Box flex={1}>
-            {defaultValues.id && onDelete && canWrite && (
-              <IconButton
-                variant={'outline'}
-                icon={<MyIcon name={'delete'} w={'16px'} h={'16px'} />}
-                aria-label={''}
-                isLoading={loading}
-                size={'sm'}
-                _hover={{
-                  color: 'red.600',
-                  borderColor: 'red.600'
-                }}
-                onClick={openConfirm(async () => {
-                  if (!onDelete || !defaultValues.id) return;
-                  try {
-                    await delOneDatasetDataById(defaultValues.id);
-                    onDelete();
-                    onClose();
-                    toast({
-                      status: 'success',
-                      title: '记录已删除'
-                    });
-                  } catch (error) {
-                    toast({
-                      status: 'warning',
-                      title: getErrText(error)
-                    });
-                    console.log(error);
+          <SideTabs
+            list={tabList}
+            activeId={currentTab}
+            onChange={async (e: any) => {
+              if (e === TabEnum.delete) {
+                return openConfirm(onDeleteData)();
+              }
+              if (e === TabEnum.doc) {
+                return window.open(`${feConfigs.docUrl}/docs/use-cases/datasetengine`, '_blank');
+              }
+              setCurrentTab(e);
+            }}
+          />
+        </Box>
+        <Flex flexDirection={'column'} px={5} py={3} flex={1} h={'100%'}>
+          <Box fontSize={'lg'} fontWeight={'bold'} mb={4}>
+            {currentTab === TabEnum.content && (
+              <>{defaultValue.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')}</>
+            )}
+            {currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}</>}
+          </Box>
+          <Box flex={1} overflow={'auto'}>
+            {currentTab === TabEnum.content && (
+              <>
+                <Box>
+                  <Flex alignItems={'center'}>
+                    <Box>
+                      <Box as="span" color={'red.600'}>
+                        *
+                      </Box>
+                      {'相关数据内容'}
+                    </Box>
+                    <MyTooltip
+                      label={'该输入框是必填项\n该内容通常是对于知识点的描述，也可以是用户的问题。'}
+                    >
+                      <QuestionOutlineIcon ml={1} />
+                    </MyTooltip>
+                  </Flex>
+                  <Textarea
+                    mt={1}
+                    placeholder={`该输入框是必填项，该内容通常是对于知识点的描述，也可以是用户的问题，最多 ${maxToken} 字。`}
+                    maxLength={maxToken}
+                    rows={10}
+                    bg={'myWhite.400'}
+                    {...register(`q`, {
+                      required: true
+                    })}
+                  />
+                </Box>
+                <Box mt={5}>
+                  <Flex>
+                    <Box>{'辅助数据'}</Box>
+                    <MyTooltip
+                      label={
+                        '该部分为可选填项\n该内容通常是为了与前面的数据内容配合，构建结构化提示词，用于特殊场景'
+                      }
+                    >
+                      <QuestionOutlineIcon ml={1} />
+                    </MyTooltip>
+                  </Flex>
+                  <Textarea
+                    mt={1}
+                    placeholder={`该部分为可选填项, 通常是为了与前面的【数据内容】配合，构建结构化提示词，用于特殊场景，最多 ${
+                      maxToken * 1.5
+                    } 字。`}
+                    bg={'myWhite.400'}
+                    rows={10}
+                    maxLength={maxToken * 1.5}
+                    {...register('a')}
+                  />
+                </Box>
+              </>
+            )}
+            {currentTab === TabEnum.index && (
+              <Grid gridTemplateColumns={['1fr', '1fr 1fr']} gridGap={4}>
+                {indexes.map((index, i) => (
+                  <Box
+                    key={index.dataId || i}
+                    p={3}
+                    borderRadius={'md'}
+                    border={theme.borders.base}
+                    bg={i % 2 !== 0 ? 'myWhite.400' : ''}
+                    _hover={{
+                      '& .delete': {
+                        display: index.defaultIndex && indexes.length === 1 ? 'none' : 'block'
+                      }
+                    }}
+                  >
+                    <Flex mb={1}>
+                      <Box flex={1}>
+                        {index.defaultIndex
+                          ? t('dataset.data.Default Index')
+                          : t('dataset.data.Custom Index Number', { number: i })}
+                      </Box>
+                      <DeleteIcon
+                        onClick={() => {
+                          if (indexes.length <= 1) {
+                            appendIndexes(getDefaultIndex({ dataId: `${Date.now()}` }));
+                          }
+                          removeIndexes(i);
+                        }}
+                      />
+                    </Flex>
+                    {index.defaultIndex ? (
+                      <Box>
+                        无法编辑，默认索引会使用【相关数据内容】与【辅助数据】的文本直接生成索引，如不需要默认索引，可删除。
+                        每条数据必须保证有一个以上索引，所有索引被删除后，会自动生成默认索引。
+                      </Box>
+                    ) : (
+                      <Textarea
+                        maxLength={maxToken}
+                        rows={10}
+                        borderColor={'transparent'}
+                        px={0}
+                        _focus={{
+                          borderColor: 'myBlue.400',
+                          px: 3
+                        }}
+                        placeholder={t('dataset.data.Index Placeholder')}
+                        {...register(`indexes.${i}.text`, {
+                          required: true
+                        })}
+                      />
+                    )}
+                  </Box>
+                ))}
+                <Flex
+                  flexDirection={'column'}
+                  alignItems={'center'}
+                  justifyContent={'center'}
+                  borderRadius={'md'}
+                  border={theme.borders.base}
+                  cursor={'pointer'}
+                  _hover={{
+                    bg: 'myBlue.100'
+                  }}
+                  minH={'100px'}
+                  onClick={() =>
+                    appendIndexes({
+                      defaultIndex: false,
+                      type: DatasetDataIndexTypeEnum.chunk,
+                      text: '',
+                      dataId: `${Date.now()}`
+                    })
                  }
-                })}
-              />
+                >
+                  <MyIcon name={'addCircle'} w={'16px'} />
+                  <Box>{t('dataset.data.Add Index')}</Box>
+                </Flex>
+              </Grid>
            )}
          </Box>
-          <Box>
+          <Flex justifyContent={'flex-end'} mt={4}>
            <Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
              {t('common.Close')}
            </Button>
-            <MyTooltip label={canWrite ? '' : t('dataset.data.Can not edit')}>
+            <MyTooltip label={collection.canWrite ? '' : t('dataset.data.Can not edit')}>
              <Button
-                isDisabled={!canWrite}
+                isDisabled={!collection.canWrite}
                isLoading={loading}
                // @ts-ignore
-                onClick={handleSubmit(defaultValues.id ? onUpdateData : sureImportData)}
+                onClick={handleSubmit(defaultValue.id ? onUpdateData : sureImportData)}
              >
-                {defaultValues.id ? '确认变更' : '确认导入'}
+                {defaultValue.id ? '确认变更' : '确认导入'}
              </Button>
            </MyTooltip>
-          </Box>
+          </Flex>
        </Flex>
      </Flex>
      <ConfirmModal />
+      <Loading fixed={false} loading={isDeleting} />
    </MyModal>
  );
 };
@@ -283,14 +388,14 @@ export default InputDataModal;
 export function RawSourceText({
  sourceId,
  sourceName = '',
-  addr = true,
+  canView = true,
  ...props
 }: RawSourceTextProps) {
  const { t } = useTranslation();
  const { toast } = useToast();
  const { setLoading } = useSystemStore();

-  const canPreview = useMemo(() => !!sourceId && addr, [addr, sourceId]);
+  const canPreview = useMemo(() => !!sourceId && canView, [canView, sourceId]);

  const icon = useMemo(() => getSourceNameIcon({ sourceId, sourceName }), [sourceId, sourceName]);

@@ -302,16 +407,12 @@ export function RawSourceText({
      <Box
        color={'myGray.600'}
        display={'inline-flex'}
-        alignItems={'center'}
        whiteSpace={'nowrap'}
        {...(canPreview
          ? {
              cursor: 'pointer',
              textDecoration: 'underline',
              onClick: async () => {
-                if (strIsLink(sourceId)) {
-                  return window.open(sourceId, '_blank');
-                }
                setLoading(true);
                try {
                  await getFileAndOpen(sourceId as string);
@@ -327,8 +428,8 @@ export function RawSourceText({
          : {})}
        {...props}
      >
-        <Image src={icon} alt="" w={'14px'} mr={2} />
-        <Box maxW={['200px', '300px']} className={'textEllipsis'}>
+        <Image src={icon} alt="" w={['14px', '16px']} mr={2} />
+        <Box maxW={['200px', '300px']} className={props.className ?? 'textEllipsis'}>
          {sourceName || t('common.UnKnow Source')}
        </Box>
      </Box>
--- a/projects/app/src/pages/dataset/detail/components/Test.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Test.tsx
@@ -13,9 +13,12 @@ import { useToast } from '@/web/common/hooks/useToast';
 import { customAlphabet } from 'nanoid';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
+import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
+import { useTranslation } from 'next-i18next';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);

 const Test = ({ datasetId }: { datasetId: string }) => {
+  const { t } = useTranslation();
  const theme = useTheme();
  const { toast } = useToast();
  const { setLoading } = useSystemStore();
@@ -24,7 +27,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
    useSearchTestStore();
  const [inputText, setInputText] = useState('');
  const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
-  const [editInputData, setEditInputData] = useState<InputDataType>();
+  const [editInputData, setEditInputData] = useState<InputDataType & { collectionId: string }>();

  const kbTestHistory = useMemo(
    () => datasetTestList.filter((item) => item.datasetId === datasetId),
@@ -33,7 +36,13 @@ const Test = ({ datasetId }: { datasetId: string }) => {

  const { mutate, isLoading } = useRequest({
    mutationFn: () => postSearchText({ datasetId, text: inputText.trim() }),
-    onSuccess(res) {
+    onSuccess(res: SearchDataResponseItemType[]) {
+      if (!res || res.length === 0) {
+        return toast({
+          status: 'warning',
+          title: t('dataset.test.noResult')
+        });
+      }
      const testItem = {
        id: nanoid(),
        datasetId,
@@ -209,8 +218,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                        collectionId: data.collectionId,
                        q: data.q,
                        a: data.a,
-                        sourceName: data.sourceName,
-                        sourceId: data.sourceId
+                        indexes: data.indexes
                      });
                    } catch (err) {
                      toast({
@@ -255,9 +263,8 @@ const Test = ({ datasetId }: { datasetId: string }) => {

      {!!editInputData && (
        <InputDataModal
-          datasetId={datasetDetail._id}
-          canWrite={datasetDetail.canWrite}
-          defaultValues={editInputData}
+          collectionId={editInputData.collectionId}
+          defaultValue={editInputData}
          onClose={() => setEditInputData(undefined)}
          onSuccess={(data) => {
            if (datasetTestItem && editInputData.id) {
--- a/projects/app/src/pages/dataset/detail/index.tsx
+++ b/projects/app/src/pages/dataset/detail/index.tsx
@@ -1,10 +1,10 @@
-import React, { useCallback, useEffect, useRef } from 'react';
+import React, { useCallback, useRef } from 'react';
 import { useRouter } from 'next/router';
 import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useForm } from 'react-hook-form';
 import { useQuery } from '@tanstack/react-query';
-import { DatasetItemType } from '@/types/core/dataset';
+import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { type ComponentRef } from './components/Info';
@@ -16,7 +16,7 @@ import PageContainer from '@/components/PageContainer';
 import Avatar from '@/components/Avatar';
 import Info from './components/Info';
 import { serviceSideProps } from '@/web/common/utils/i18n';
-import { useTranslation } from 'react-i18next';
+import { useTranslation } from 'next-i18next';
 import { getTrainingQueueLen } from '@/web/core/dataset/api';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
@@ -77,7 +77,7 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
  useQuery([datasetId], () => loadDatasetDetail(datasetId), {
    onSuccess(res) {
      form.reset(res);
-      InfoRef.current?.initInput(res.tags);
+      InfoRef.current?.initInput(res.tags?.join(' '));
    },
    onError(err: any) {
      router.replace(`/dataset/list`);