This commit is contained in:
Archer
2023-11-09 09:46:57 +08:00
committed by GitHub
parent 661ee79943
commit 8bb5588305
402 changed files with 9899 additions and 5967 deletions

View File

@@ -10,7 +10,7 @@ import {
NumberDecrementStepper
} from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';

View File

@@ -5,13 +5,13 @@ import { useToast } from '@/web/common/hooks/useToast';
import { splitText2Chunks } from '@/global/common/string/tools';
import { simpleText } from '@fastgpt/global/common/string/tools';
import {
uploadFiles,
fileDownload,
readCsvContent,
readTxtContent,
readPdfContent,
readDocContent
} from '@/web/common/file/utils';
import { uploadFiles } from '@/web/common/file/controller';
import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
import React, { DragEvent, useCallback, useState } from 'react';
import { useTranslation } from 'next-i18next';
@@ -108,13 +108,18 @@ const FileSelect = ({
if (!icon) continue;
// upload file
const filesId = await uploadFiles([file], { datasetId: datasetDetail._id }, (percent) => {
if (percent < 100) {
setSelectingText(
t('file.Uploading', { name: file.name.slice(0, 30), percent }) || ''
);
} else {
setSelectingText(t('file.Parse', { name: file.name.slice(0, 30) }) || '');
const filesId = await uploadFiles({
files: [file],
bucketName: 'dataset',
metadata: { datasetId: datasetDetail._id },
percentListen: (percent) => {
if (percent < 100) {
setSelectingText(
t('file.Uploading', { name: file.name.slice(0, 30), percent }) || ''
);
} else {
setSelectingText(t('file.Parse', { name: file.name.slice(0, 30) }) || '');
}
}
});
const fileId = filesId[0];
@@ -243,7 +248,11 @@ const FileSelect = ({
type: txtBlob.type,
lastModified: new Date().getTime()
});
const fileIds = await uploadFiles([txtFile], { datasetId: datasetDetail._id });
const fileIds = await uploadFiles({
files: [txtFile],
bucketName: 'dataset',
metadata: { datasetId: datasetDetail._id }
});
const splitRes = splitText2Chunks({
text: content,

View File

@@ -45,7 +45,7 @@ const ImportData = ({
mode: TrainingModeEnum.index
},
[ImportTypeEnum.qa]: {
defaultChunkLen: qaModel?.maxToken * 0.5 || 8000,
defaultChunkLen: qaModel?.maxContext * 0.5 || 8000,
unitPrice: qaModel?.price || 3,
mode: TrainingModeEnum.qa
},

View File

@@ -11,7 +11,7 @@ import React, {
import FileSelect, { FileItemType, Props as FileSelectProps } from './FileSelect';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postDatasetCollection } from '@/web/core/dataset/api';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import { splitText2Chunks } from '@/global/common/string/tools';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
@@ -21,7 +21,7 @@ import { CloseIcon } from '@chakra-ui/icons';
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
import MyIcon from '@/components/Icon';
import { chunksUpload } from '@/web/core/dataset/utils';
import { postCreateTrainingBill } from '@/web/common/bill/api';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
import { useTranslation } from 'react-i18next';
import { ImportTypeEnum } from './ImportModal';
@@ -129,19 +129,19 @@ const Provider = ({
let totalInsertion = 0;
for await (const file of files) {
const chunks = file.chunks;
// create training bill
const billId = await postCreateTrainingBill({
name: t('dataset.collections.Create Training Data', { filename: file.filename })
});
// create a file collection and training bill
const [collectionId, billId] = await Promise.all([
postDatasetCollection({
datasetId,
parentId,
name: file.filename,
type: file.type,
metadata: file.metadata
}),
postCreateTrainingBill({
name: t('dataset.collections.Create Training Data', { filename: file.filename })
})
]);
const collectionId = await postDatasetCollection({
datasetId,
parentId,
name: file.filename,
type: file.type,
metadata: file.metadata
});
// upload data
const { insertLen } = await chunksUpload({
collectionId,

View File

@@ -1,7 +1,7 @@
import React, { useState, useMemo } from 'react';
import { Box, Flex, Button, Input } from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon, InfoOutlineIcon } from '@chakra-ui/icons';
import { Prompt_AgentQA } from '@/global/core/prompt/agent';