V4.6.6-2 (#673)

2025-08-02 20:58:12 +00:00 · 2023-12-31 14:12:51 +08:00
parent ccca0468da
commit 9ccfda47b7
270 changed files with 8182 additions and 1295 deletions
--- a/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
@@ -1,4 +1,4 @@
-import React, { useState } from 'react';
+import React from 'react';
 import {
  Box,
  Flex,
@@ -8,10 +8,10 @@ import {
  NumberInputStepper,
  NumberIncrementStepper,
  NumberDecrementStepper,
-  Input
+  Input,
+  Grid
 } from '@chakra-ui/react';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
-import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
@@ -25,7 +25,7 @@ const ChunkImport = () => {
  const { t } = useTranslation();
  const { datasetDetail } = useDatasetStore();
  const vectorModel = datasetDetail.vectorModel;
-  const unitPrice = vectorModel?.price || 0.2;
+  const unitPrice = vectorModel?.inputPrice || 0.002;

  const {
    chunkLen,
@@ -33,6 +33,7 @@ const ChunkImport = () => {
    setCustomSplitChar,
    successChunks,
    totalChunks,
+    totalTokens,
    isUnselectedFile,
    price,
    onclickUpload,
@@ -108,21 +109,27 @@ const ChunkImport = () => {
            />
          </Box>
        </Box>
-        {/* price */}
-        <Flex mt={4} alignItems={'center'}>
-          <Box>
-            {t('core.dataset.import.Estimated Price')}
-            <MyTooltip
-              label={t('core.dataset.import.Estimated Price Tips', {
-                price: formatPrice(unitPrice, 1000)
-              })}
-              forceShow
-            >
-              <QuestionOutlineIcon ml={1} />
-            </MyTooltip>
-          </Box>
-          <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
-        </Flex>
+        <Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
+          <Flex alignItems={'center'}>
+            <Box>{t('core.dataset.import.Total tokens')}：</Box>
+            <Box>{totalTokens}</Box>
+          </Flex>
+          {/* price */}
+          <Flex alignItems={'center'}>
+            <Box>
+              {t('core.dataset.import.Estimated Price')}
+              <MyTooltip
+                label={t('core.dataset.import.Embedding Estimated Price Tips', {
+                  price: unitPrice
+                })}
+                forceShow
+              >
+                <QuestionOutlineIcon ml={1} />
+              </MyTooltip>
+            </Box>
+            <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
+          </Flex>
+        </Grid>
        <Flex mt={3}>
          {showRePreview && (
            <Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
--- a/projects/app/src/pages/dataset/detail/components/Import/Csv.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Csv.tsx
@@ -1,8 +1,11 @@
 import React from 'react';
-import { Box, Flex, Button } from '@chakra-ui/react';
+import { Box, Flex, Button, Grid } from '@chakra-ui/react';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
 import { useTranslation } from 'next-i18next';
+import { useDatasetStore } from '@/web/core/dataset/store/dataset';
+import MyTooltip from '@/components/MyTooltip';
+import { QuestionOutlineIcon } from '@chakra-ui/icons';

 const fileExtension = '.csv';
 const csvTemplate = `index,content
@@ -12,8 +15,19 @@ const csvTemplate = `index,content

 const CsvImport = () => {
  const { t } = useTranslation();
-  const { successChunks, totalChunks, isUnselectedFile, onclickUpload, uploading } =
-    useImportStore();
+  const {
+    successChunks,
+    totalChunks,
+    isUnselectedFile,
+    onclickUpload,
+    uploading,
+    totalTokens,
+    price
+  } = useImportStore();
+  const { datasetDetail } = useDatasetStore();
+
+  const vectorModel = datasetDetail.vectorModel;
+  const unitPrice = vectorModel?.inputPrice || 0.002;

  const { openConfirm, ConfirmModal } = useConfirm({
    content: t('core.dataset.import.Import Tip')
@@ -31,6 +45,27 @@ const CsvImport = () => {
        }}
        tip={t('dataset.import csv tip')}
      >
+        <Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
+          <Flex alignItems={'center'}>
+            <Box>{t('core.dataset.import.Total tokens')}：</Box>
+            <Box>{totalTokens}</Box>
+          </Flex>
+          {/* price */}
+          <Flex alignItems={'center'}>
+            <Box>
+              {t('core.dataset.import.Estimated Price')}
+              <MyTooltip
+                label={t('core.dataset.import.Embedding Estimated Price Tips', {
+                  price: unitPrice
+                })}
+                forceShow
+              >
+                <QuestionOutlineIcon ml={1} />
+              </MyTooltip>
+            </Box>
+            <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
+          </Flex>
+        </Grid>
        <Flex mt={3}>
          <Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
            {uploading ? (
--- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
@@ -44,32 +44,36 @@ const ImportData = ({
      [ImportTypeEnum.chunk]: {
        defaultChunkLen: vectorModel?.defaultToken || 500,
        chunkOverlapRatio: 0.2,
-        unitPrice: vectorModel?.price || 0.2,
+        inputPrice: vectorModel?.inputPrice || 0,
+        outputPrice: 0,
        mode: TrainingModeEnum.chunk,
        collectionTrainingType: DatasetCollectionTrainingModeEnum.chunk
      },
      [ImportTypeEnum.qa]: {
        defaultChunkLen: agentModel?.maxContext * 0.55 || 8000,
        chunkOverlapRatio: 0,
-        unitPrice: agentModel?.price || 3,
+        inputPrice: agentModel?.inputPrice || 0,
+        outputPrice: agentModel?.outputPrice || 0,
        mode: TrainingModeEnum.qa,
        collectionTrainingType: DatasetCollectionTrainingModeEnum.qa
      },
      [ImportTypeEnum.csv]: {
        defaultChunkLen: 0,
        chunkOverlapRatio: 0,
-        unitPrice: vectorModel?.price || 0.2,
+        inputPrice: vectorModel?.inputPrice || 0,
+        outputPrice: 0,
        mode: TrainingModeEnum.chunk,
        collectionTrainingType: DatasetCollectionTrainingModeEnum.manual
      }
    };
    return map[importType];
  }, [
+    agentModel?.inputPrice,
    agentModel?.maxContext,
-    agentModel?.price,
+    agentModel?.outputPrice,
    importType,
    vectorModel?.defaultToken,
-    vectorModel?.price
+    vectorModel?.inputPrice
  ]);

  const TitleStyle: BoxProps = {
--- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
@@ -11,7 +11,7 @@ import React, {
 import FileSelect, { FileItemType, Props as FileSelectProps } from './FileSelect';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { postDatasetCollection } from '@/web/core/dataset/api';
-import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
+import { formatModelPrice2Read } from '@fastgpt/global/support/wallet/bill/tools';
 import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { useToast } from '@/web/common/hooks/useToast';
@@ -43,6 +43,7 @@ type useImportStoreType = {
  setSuccessChunks: Dispatch<SetStateAction<number>>;
  isUnselectedFile: boolean;
  totalChunks: number;
+  totalTokens: number;
  onclickUpload: (e?: { prompt?: string }) => void;
  onReSplitChunks: () => void;
  price: number;
@@ -68,6 +69,7 @@ const StateContext = createContext<useImportStoreType>({

  isUnselectedFile: false,
  totalChunks: 0,
+  totalTokens: 0,
  onReSplitChunks: function (): void {
    throw new Error('Function not implemented.');
  },
@@ -100,7 +102,8 @@ export const useImportStore = () => useContext(StateContext);
 const Provider = ({
  datasetId,
  parentId,
-  unitPrice,
+  inputPrice,
+  outputPrice,
  mode,
  collectionTrainingType,
  vectorModel,
@@ -113,7 +116,8 @@ const Provider = ({
 }: {
  datasetId: string;
  parentId: string;
-  unitPrice: number;
+  inputPrice: number;
+  outputPrice: number;
  mode: `${TrainingModeEnum}`;
  collectionTrainingType: `${DatasetCollectionTrainingModeEnum}`;
  vectorModel: string;
@@ -140,9 +144,17 @@ const Provider = ({
    [files]
  );

+  const totalTokens = useMemo(() => files.reduce((sum, file) => sum + file.tokens, 0), [files]);
+
  const price = useMemo(() => {
-    return formatPrice(files.reduce((sum, file) => sum + file.tokens, 0) * unitPrice);
-  }, [files, unitPrice]);
+    if (mode === TrainingModeEnum.qa) {
+      const inputTotal = totalTokens * inputPrice;
+      const outputTotal = totalTokens * 0.5 * outputPrice;
+
+      return formatModelPrice2Read(inputTotal + outputTotal);
+    }
+    return formatModelPrice2Read(totalTokens * inputPrice);
+  }, [inputPrice, mode, outputPrice, totalTokens]);

  /* start upload data */
  const { mutate: onclickUpload, isLoading: uploading } = useRequest({
@@ -249,6 +261,7 @@ const Provider = ({
    setSuccessChunks,
    isUnselectedFile,
    totalChunks,
+    totalTokens,
    price,
    onReSplitChunks,
    onclickUpload,
--- a/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Import/QA.tsx
@@ -1,7 +1,6 @@
 import React, { useState } from 'react';
-import { Box, Flex, Button, Textarea } from '@chakra-ui/react';
+import { Box, Flex, Button, Textarea, Grid } from '@chakra-ui/react';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
-import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import MyTooltip from '@/components/MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { Prompt_AgentQA } from '@/global/core/prompt/agent';
@@ -15,11 +14,11 @@ const QAImport = () => {
  const { t } = useTranslation();
  const { datasetDetail } = useDatasetStore();
  const agentModel = datasetDetail.agentModel;
-  const unitPrice = agentModel?.price || 3;

  const {
    successChunks,
    totalChunks,
+    totalTokens,
    isUnselectedFile,
    price,
    onclickUpload,
@@ -55,20 +54,28 @@ const QAImport = () => {
          </Box>
        </Box>
        {/* price */}
-        <Flex py={5} alignItems={'center'}>
-          <Box>
-            {t('core.dataset.import.Estimated Price')}
-            <MyTooltip
-              label={t('core.dataset.import.Estimated Price Tips', {
-                price: formatPrice(unitPrice, 1000)
-              })}
-              forceShow
-            >
-              <QuestionOutlineIcon ml={1} />
-            </MyTooltip>
-          </Box>
-          <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
-        </Flex>
+        <Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
+          <Flex alignItems={'center'}>
+            <Box>{t('core.dataset.import.Total tokens')}：</Box>
+            <Box>{totalTokens}</Box>
+          </Flex>
+          {/* price */}
+          <Flex alignItems={'center'}>
+            <Box>
+              {t('core.dataset.import.Estimated Price')}
+              <MyTooltip
+                label={t('core.dataset.import.QA Estimated Price Tips', {
+                  inputPrice: agentModel?.inputPrice,
+                  outputPrice: agentModel?.outputPrice
+                })}
+                forceShow
+              >
+                <QuestionOutlineIcon ml={1} />
+              </MyTooltip>
+            </Box>
+            <Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
+          </Flex>
+        </Grid>
        <Flex mt={3}>
          {showRePreview && (
            <Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
--- a/projects/app/src/pages/dataset/detail/components/Test.tsx
+++ b/projects/app/src/pages/dataset/detail/components/Test.tsx
@@ -1,22 +1,12 @@
 import React, { useEffect, useMemo, useState } from 'react';
-import {
-  Box,
-  Textarea,
-  Button,
-  Flex,
-  useTheme,
-  Grid,
-  Progress,
-  Switch,
-  useDisclosure
-} from '@chakra-ui/react';
+import { Box, Textarea, Button, Flex, useTheme, Grid, useDisclosure } from '@chakra-ui/react';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { useSearchTestStore, SearchTestStoreItemType } from '@/web/core/dataset/store/searchTest';
 import { getDatasetDataItemById, postSearchText } from '@/web/core/dataset/api';
 import MyIcon from '@/components/Icon';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { formatTimeToChatTime } from '@/utils/tools';
-import InputDataModal, { type InputDataType } from './InputDataModal';
+import InputDataModal, { RawSourceText, type InputDataType } from './InputDataModal';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useToast } from '@/web/common/hooks/useToast';
@@ -45,6 +35,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
  const [searchMode, setSearchMode] = useState<`${DatasetSearchModeEnum}`>(
    DatasetSearchModeEnum.embedding
  );
+  const [usingReRank, setUsingReRank] = useState(false);
  const searchModeData = DatasetSearchModeMap[searchMode];

  const {
@@ -59,7 +50,8 @@ const Test = ({ datasetId }: { datasetId: string }) => {
  );

  const { mutate, isLoading } = useRequest({
-    mutationFn: () => postSearchText({ datasetId, text: inputText.trim(), searchMode, limit: 30 }),
+    mutationFn: () =>
+      postSearchText({ datasetId, text: inputText.trim(), searchMode, usingReRank, limit: 20 }),
    onSuccess(res: SearchTestResponse) {
      if (!res || res.list.length === 0) {
        return toast({
@@ -73,7 +65,8 @@ const Test = ({ datasetId }: { datasetId: string }) => {
        text: inputText.trim(),
        time: new Date(),
        results: res.list,
-        duration: res.duration
+        duration: res.duration,
+        searchMode
      };
      pushDatasetTestItem(testItem);
      setDatasetTestItem(testItem);
@@ -123,8 +116,8 @@ const Test = ({ datasetId }: { datasetId: string }) => {
            variant={'unstyled'}
            maxLength={datasetDetail.vectorModel.maxToken}
            placeholder={t('core.dataset.test.Test Text Placeholder')}
-            value={inputText}
-            onChange={(e) => setInputText(e.target.value)}
+            defaultValue={inputText}
+            onBlur={(e) => setInputText(e.target.value)}
          />
          <Flex alignItems={'center'} justifyContent={'flex-end'}>
            <Box mx={3} color={'myGray.500'}>
@@ -142,8 +135,9 @@ const Test = ({ datasetId }: { datasetId: string }) => {
          </Flex>
          <Box mt={2}>
            <Flex py={2} fontWeight={'bold'} borderBottom={theme.borders.sm}>
+              <Box w={'80px'}>{t('core.dataset.search.search mode')}</Box>
              <Box flex={1}>{t('core.dataset.test.Test Text')}</Box>
-              <Box w={'80px'}>{t('common.Time')}</Box>
+              <Box w={'70px'}>{t('common.Time')}</Box>
              <Box w={'14px'}></Box>
            </Flex>
            {testHistories.map((item) => (
@@ -159,12 +153,27 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                  }
                }}
                cursor={'pointer'}
+                fontSize={'sm'}
                onClick={() => setDatasetTestItem(item)}
              >
+                <Box w={'80px'}>
+                  {DatasetSearchModeMap[item.searchMode] ? (
+                    <Flex alignItems={'center'}>
+                      <MyIcon
+                        name={DatasetSearchModeMap[item.searchMode].icon as any}
+                        w={'12px'}
+                        mr={'1px'}
+                      />
+                      {t(DatasetSearchModeMap[item.searchMode].title)}
+                    </Flex>
+                  ) : (
+                    '-'
+                  )}
+                </Box>
                <Box flex={1} mr={2}>
                  {item.text}
                </Box>
-                <Box w={'80px'}>{formatTimeToChatTime(item.time)}</Box>
+                <Box w={'70px'}>{formatTimeToChatTime(item.time)}</Box>
                <MyTooltip label={t('core.dataset.test.delete test history')}>
                  <Box w={'14px'} h={'14px'}>
                    <MyIcon
@@ -232,7 +241,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                <Box
                  key={item.id}
                  pb={2}
-                  borderRadius={'sm'}
+                  borderRadius={'lg'}
                  border={theme.borders.base}
                  _notLast={{ mb: 2 }}
                  cursor={'pointer'}
@@ -267,12 +276,19 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                      border={theme.borders.base}
                      px={2}
                      fontSize={'sm'}
-                      mr={1}
+                      mr={3}
                      borderRadius={'md'}
                    >
                      # {index + 1}
                    </Box>
-                    <MyIcon name={'kbTest'} w={'14px'} />
+                    <RawSourceText
+                      fontWeight={'bold'}
+                      color={'black'}
+                      sourceName={item.sourceName}
+                      sourceId={item.sourceId}
+                      canView
+                    />
+                    {/* <MyIcon name={'kbTest'} w={'14px'} />
                    <Progress
                      mx={2}
                      flex={'1 0 0'}
@@ -281,7 +297,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                      borderRadius={'20px'}
                      colorScheme="gray"
                    />
-                    <Box>{item.score.toFixed(4)}</Box>
+                    <Box>{item.score.toFixed(4)}</Box> */}
                  </Flex>
                  <Box px={2} fontSize={'xs'} color={'myGray.600'} wordBreak={'break-word'}>
                    <Box>{item.q}</Box>
@@ -335,9 +351,11 @@ const Test = ({ datasetId }: { datasetId: string }) => {
      {isOpenSelectMode && (
        <DatasetParamsModal
          searchMode={searchMode}
+          usingReRank={usingReRank}
          onClose={onCloseSelectMode}
          onSuccess={(e) => {
            setSearchMode(e.searchMode);
+            e.usingReRank !== undefined && setUsingReRank(e.usingReRank);
          }}
        />
      )}
--- a/projects/app/src/pages/dataset/detail/index.tsx
+++ b/projects/app/src/pages/dataset/detail/index.tsx
@@ -1,6 +1,6 @@
-import React, { useCallback } from 'react';
+import React, { useCallback, useMemo } from 'react';
 import { useRouter } from 'next/router';
-import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
+import { Box, Flex, IconButton, useTheme, Progress } from '@chakra-ui/react';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useQuery } from '@tanstack/react-query';
 import { getErrText } from '@fastgpt/global/common/error/utils';
@@ -92,9 +92,55 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
    }
  });

-  const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, {
-    refetchInterval: 10000
-  });
+  const { data: { vectorTrainingCount = 0, agentTrainingCount = 0 } = {} } = useQuery(
+    ['getTrainingQueueLen'],
+    () =>
+      getTrainingQueueLen({
+        vectorModel: datasetDetail.vectorModel.model,
+        agentModel: datasetDetail.agentModel.model
+      }),
+    {
+      refetchInterval: 10000
+    }
+  );
+  const { vectorTrainingMap, agentTrainingMap } = useMemo(() => {
+    const vectorTrainingMap = (() => {
+      if (vectorTrainingCount < 1000)
+        return {
+          colorSchema: 'green',
+          tip: t('core.dataset.training.Leisure')
+        };
+      if (vectorTrainingCount < 10000)
+        return {
+          colorSchema: 'yellow',
+          tip: t('core.dataset.training.Waiting')
+        };
+      return {
+        colorSchema: 'red',
+        tip: t('core.dataset.training.Full')
+      };
+    })();
+    const agentTrainingMap = (() => {
+      if (agentTrainingCount < 100)
+        return {
+          colorSchema: 'green',
+          tip: t('core.dataset.training.Leisure')
+        };
+      if (agentTrainingCount < 1000)
+        return {
+          colorSchema: 'yellow',
+          tip: t('core.dataset.training.Waiting')
+        };
+      return {
+        colorSchema: 'red',
+        tip: t('core.dataset.training.Full')
+      };
+    })();
+    return {
+      vectorTrainingMap,
+      agentTrainingMap
+    };
+  }, [agentTrainingCount, t, vectorTrainingCount]);

  return (
    <>
@@ -155,19 +201,32 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
                  setCurrentTab(e);
                }}
              />
-              <Box textAlign={'center'}>
-                <Flex justifyContent={'center'} alignItems={'center'}>
-                  <MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
-                  <Box>{t('dataset.System Data Queue')}</Box>
-                  <MyTooltip
-                    label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
-                    placement={'top'}
-                  >
-                    <QuestionOutlineIcon ml={1} w={'16px'} />
-                  </MyTooltip>
-                </Flex>
-                <Box mt={1} fontWeight={'bold'}>
-                  {trainingQueueLen}
+              <Box>
+                <Box mb={3}>
+                  <Box fontSize={'sm'}>
+                    {t('core.dataset.training.Agent queue')}({agentTrainingMap.tip})
+                  </Box>
+                  <Progress
+                    value={100}
+                    size={'xs'}
+                    colorScheme={agentTrainingMap.colorSchema}
+                    borderRadius={'10px'}
+                    isAnimated
+                    hasStripe
+                  />
+                </Box>
+                <Box mb={3}>
+                  <Box fontSize={'sm'}>
+                    {t('core.dataset.training.Vector queue')}({vectorTrainingMap.tip})
+                  </Box>
+                  <Progress
+                    value={100}
+                    size={'xs'}
+                    colorScheme={vectorTrainingMap.colorSchema}
+                    borderRadius={'10px'}
+                    isAnimated
+                    hasStripe
+                  />
                </Box>
              </Box>
              <Flex