New dpcs structure and dataset i18n (#551)

* perf: check balance

* md

* lock way

* i18n

* docs

* doc

* i18n

* update doc

* feat: one link sync

* feat: one link sync

* feat: one link sync

* feat: one link sync

* feat: one link sync

* feat: one link sync

* feat: one link sync
This commit is contained in:
Archer
2023-12-04 21:37:07 +08:00
committed by GitHub
parent c3ae38df8b
commit 62e87551ac
141 changed files with 961 additions and 469 deletions

View File

@@ -12,6 +12,7 @@ import MyIcon from '../Icon';
import { useTranslation } from 'next-i18next';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyTooltip from '../MyTooltip';
import { getDocPath } from '@/web/common/system/doc';
export enum NavbarTypeEnum {
normal = 'normal',
@@ -181,7 +182,7 @@ const Navbar = ({ unread }: { unread: number }) => {
mb={0}
color={'#9096a5'}
onClick={() => {
window.open(`${feConfigs.docUrl}/docs/intro`);
window.open(getDocPath('/docs/intro'));
}}
>
<MyIcon name={'common/courseLight'} width={'26px'} height={'26px'} />

View File

@@ -25,6 +25,7 @@ import { PromptTemplateItem } from '@fastgpt/global/core/ai/type.d';
import type { AIChatModuleProps } from '@fastgpt/global/core/module/node/type.d';
import type { AppSimpleEditConfigTemplateType } from '@fastgpt/global/core/app/type.d';
import { SimpleModeTemplate_FastGPT_Universal } from '@/global/core/app/constants';
import { getDocPath } from '@/web/common/system/doc';
const PromptTemplate = dynamic(() => import('@/components/PromptTemplate'));
@@ -77,7 +78,7 @@ const AIChatSettingsModal = ({
{t('app.AI Advanced Settings')}
{feConfigs?.docUrl && (
<Link
href={`${feConfigs.docUrl}/docs/use-cases/ai_settings/`}
href={getDocPath('/docs/use-cases/ai_settings/')}
target={'_blank'}
ml={1}
textDecoration={'underline'}

View File

@@ -39,6 +39,7 @@ import MyModal from '@/components/MyModal';
import { useForm } from 'react-hook-form';
import { useRequest } from '@/web/common/hooks/useRequest';
import MyTooltip from '@/components/MyTooltip';
import { getDocPath } from '@/web/common/system/doc';
type EditProps = EditApiKeyProps & { _id?: string };
const defaultEditData: EditProps = {
@@ -82,9 +83,9 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => {
<Box fontSize={['md', 'xl']} fontWeight={'bold'}>
API
</Box>
{feConfigs.docUrl && (
{feConfigs?.docUrl && (
<Link
href={feConfigs.openAPIDocUrl || `${feConfigs.docUrl}/docs/development/openapi`}
href={feConfigs.openAPIDocUrl || getDocPath('/docs/development/openapi')}
target={'_blank'}
ml={1}
color={'myBlue.600'}

View File

@@ -30,6 +30,7 @@ import { useRouter } from 'next/router';
import MySelect from '@/components/Select';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import { putUpdateMemberName } from '@/web/support/user/team/api';
import { getDocPath } from '@/web/common/system/doc';
const TeamMenu = dynamic(() => import('@/components/support/user/team/TeamMenu'));
const PayModal = dynamic(() => import('./PayModal'), {
@@ -261,7 +262,7 @@ const UserInfo = () => {
cursor={'pointer'}
userSelect={'none'}
onClick={() => {
window.open(`${feConfigs.docUrl}/docs/intro`);
window.open(getDocPath('/docs/intro'));
}}
>
<MyIcon name={'common/courseLight'} w={'18px'} />

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';

View File

@@ -30,7 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// delete
await delCollectionRelevantData({
collectionIds: delIdList,
fileIds: collections.map((item) => String(item.metadata?.fileId)).filter(Boolean)
fileIds: collections.map((item) => item.metadata?.fileId).filter(Boolean)
});
// delete collection

View File

@@ -0,0 +1,87 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { loadingOneChunkCollection } from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { collectionId } = req.body as { collectionId: string };
if (!collectionId) {
throw new Error('CollectionIdId is required');
}
const { collection, tmbId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
per: 'w'
});
if (collection.type !== DatasetCollectionTypeEnum.link || !collection.rawLink) {
return Promise.reject(DatasetErrEnum.unLinkCollection);
}
const vectorModelData = getVectorModel(collection.datasetId.vectorModel);
const agentModelData = getQAModel(collection.datasetId.agentModel);
// create training bill
const { billId } = await createTrainingBill({
teamId: collection.teamId,
tmbId,
appName: 'core.dataset.collection.Sync Collection',
billSource: BillSourceEnum.training,
vectorModel: vectorModelData.name,
agentModel: agentModelData.name
});
// create a collection and delete old
const { _id } = await MongoDatasetCollection.create({
parentId: collection.parentId,
teamId: collection.teamId,
tmbId: collection.tmbId,
datasetId: collection.datasetId._id,
type: collection.type,
name: collection.name,
createTime: collection.createTime,
trainingType: collection.trainingType,
chunkSize: collection.chunkSize,
fileId: collection.fileId,
rawLink: collection.rawLink,
metadata: collection.metadata
});
// start load
await loadingOneChunkCollection({
collectionId: _id,
tmbId,
billId
});
// delete old collection
await Promise.all([
delCollectionRelevantData({
collectionIds: [collection._id],
fileIds: collection.fileId ? [collection.fileId] : []
}),
MongoDatasetCollection.findByIdAndRemove(collection._id)
]);
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';

View File

@@ -1,11 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { CreateTrainingBillProps } from '@fastgpt/global/support/wallet/bill/api.d';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -17,30 +17,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const vectorModelData = getVectorModel(vectorModel);
const agentModelData = getQAModel(agentModel);
const { _id } = await MongoBill.create({
const { billId } = await createTrainingBill({
teamId,
tmbId,
appName: name,
source: BillSourceEnum.training,
list: [
{
moduleName: 'wallet.moduleName.index',
model: vectorModelData.name,
amount: 0,
tokenLen: 0
},
{
moduleName: 'wallet.moduleName.qa',
model: agentModelData.name,
amount: 0,
tokenLen: 0
}
],
total: 0
billSource: BillSourceEnum.training,
vectorModel: vectorModelData.name,
agentModel: agentModelData.name
});
jsonRes<string>(res, {
data: _id
data: billId
});
} catch (err) {
jsonRes(res, {

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { MongoUser } from '@fastgpt/service/support/user/schema';

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { clearCookie } from '@fastgpt/service/support/permission/controller';

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { MongoUser } from '@fastgpt/service/support/user/schema';

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';

View File

@@ -1,4 +1,3 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';

View File

@@ -44,6 +44,7 @@ import { feConfigs } from '@/web/common/system/staticData';
import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal';
import dayjs from 'dayjs';
import { getDocPath } from '@/web/common/system/doc';
const Share = ({ appId }: { appId: string }) => {
const { t } = useTranslation();
@@ -357,7 +358,7 @@ function EditLinkModal({
/>
</Flex>
<Link
href={`${feConfigs.docUrl}/docs/development/openapi/share`}
href={getDocPath('/docs/development/openapi/share')}
target={'_blank'}
fontSize={'sm'}
color={'myGray.500'}

View File

@@ -5,6 +5,7 @@ import { useTranslation } from 'next-i18next';
import Avatar from '@/components/Avatar';
import { useRouter } from 'next/router';
import CommunityModal from '@/components/CommunityModal';
import { getDocPath } from '@/web/common/system/doc';
const Footer = () => {
const { t } = useTranslation();
@@ -47,7 +48,7 @@ const Footer = () => {
{
label: t('home.Footer Docs'),
onClick: () => {
window.open(`${feConfigs.docUrl}/docs/intro`, '_blank');
window.open(getDocPath('/docs/intro'), '_blank');
}
}
]

View File

@@ -7,6 +7,7 @@ import Avatar from '@/components/Avatar';
import CommunityModal from '@/components/CommunityModal';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyIcon from '@/components/Icon';
import { getDocPath } from '@/web/common/system/doc';
const Navbar = () => {
const router = useRouter();
@@ -47,7 +48,7 @@ const Navbar = () => {
label: t('home.Docs'),
key: 'docs',
onClick: () => {
window.open(`${feConfigs.docUrl}/docs/intro`);
window.open(getDocPath('/docs/intro'));
}
}
]

View File

@@ -22,7 +22,7 @@ import {
putDatasetCollectionById,
postDatasetCollection,
getDatasetCollectionPathById,
postWebsiteSync
postLinkCollectionSync
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { debounce } from 'lodash';
@@ -61,7 +61,6 @@ import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {});
@@ -78,11 +77,15 @@ const CollectionCard = () => {
const { isPc } = useSystemStore();
const { userInfo } = useUserStore();
const [searchText, setSearchText] = useState('');
const { datasetDetail, updateDataset, loadDatasetDetail } = useDatasetStore();
const { datasetDetail, updateDataset, startWebsiteSync, loadDatasetDetail } = useDatasetStore();
const { openConfirm, ConfirmModal } = useConfirm({
const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
content: t('dataset.Confirm to delete the file')
});
const { openConfirm: openSyncConfirm, ConfirmModal: ConfirmSyncModal } = useConfirm({
content: t('core.dataset.collection.Start Sync Tip')
});
const {
isOpen: isOpenFileImportModal,
onOpen: onOpenFileImportModal,
@@ -112,7 +115,7 @@ const CollectionCard = () => {
Pagination,
total,
getData,
isLoading,
isLoading: isGetting,
pageNum,
pageSize
} = usePagination<DatasetCollectionsListItemType>({
@@ -231,27 +234,23 @@ const CollectionCard = () => {
const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({
mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
onCloseWebsiteModal();
const [_, billId] = await Promise.all([
updateDataset({
id: datasetDetail._id,
websiteConfig,
status: DatasetStatusEnum.syncing
}),
postCreateTrainingBill({
name: 'core.dataset.training.Website Sync',
vectorModel: datasetDetail.vectorModel.model,
agentModel: datasetDetail.agentModel.model
})
]);
return billId;
},
onSuccess(billId: string) {
try {
postWebsiteSync({ datasetId: datasetDetail._id, billId });
} catch (error) {}
await updateDataset({
id: datasetDetail._id,
websiteConfig
});
return startWebsiteSync();
},
errorToast: t('common.Update Failed')
});
const { mutate: onclickStartSync, isLoading: isSyncing } = useRequest({
mutationFn: (collectionId: string) => {
return postLinkCollectionSync(collectionId);
},
onSuccess() {
getData(pageNum);
},
errorToast: t('core.dataset.error.Start Sync Failed')
});
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
getDatasetCollectionPathById(parentId)
@@ -261,6 +260,16 @@ const CollectionCard = () => {
() => !!formatCollections.find((item) => item.trainingAmount > 0),
[formatCollections]
);
const isLoading = useMemo(
() =>
isCreating ||
isDeleting ||
isUpdating ||
isSyncing ||
(isGetting && collections.length === 0),
[collections.length, isCreating, isDeleting, isGetting, isSyncing, isUpdating]
);
useQuery(
['refreshCollection'],
() => {
@@ -589,6 +598,23 @@ const CollectionCard = () => {
</MenuButton>
}
menuList={[
...(collection.type === DatasetCollectionTypeEnum.link
? [
{
child: (
<Flex alignItems={'center'}>
<MyIcon name={'common/refreshLight'} w={'14px'} mr={2} />
{t('core.dataset.collection.Sync')}
</Flex>
),
onClick: () =>
openSyncConfirm(() => {
console.log(collection._id);
onclickStartSync(collection._id);
})()
}
]
: []),
{
child: (
<Flex alignItems={'center'}>
@@ -629,7 +655,7 @@ const CollectionCard = () => {
</Flex>
),
onClick: () =>
openConfirm(
openDeleteConfirm(
() => {
onDelCollection(collection._id);
},
@@ -659,21 +685,38 @@ const CollectionCard = () => {
t('core.dataset.collection.Empty Tip')
) : (
<Flex>
{t('core.dataset.collection.Website Empty Tip')}
<Box textDecoration={'underline'} cursor={'pointer'} onClick={onOpenWebsiteModal}>
{t('core.dataset.collection.Click top config website')}
</Box>
{datasetDetail.status === DatasetStatusEnum.syncing && (
<>{t('core.dataset.status.syncing')}</>
)}
{datasetDetail.status === DatasetStatusEnum.active && (
<>
{!datasetDetail?.websiteConfig?.url ? (
<>
{t('core.dataset.collection.Website Empty Tip')}
{', '}
<Box
textDecoration={'underline'}
cursor={'pointer'}
onClick={onOpenWebsiteModal}
>
{t('core.dataset.collection.Click top config website')}
</Box>
</>
) : (
<>{t('core.dataset.website.UnValid Website Tip')}</>
)}
</>
)}
</Flex>
)
}
/>
)}
</TableContainer>
<Loading
loading={isCreating || isDeleting || isUpdating || (isLoading && collections.length === 0)}
/>
<Loading loading={isLoading} />
<ConfirmModal />
<ConfirmDeleteModal />
<ConfirmSyncModal />
<EditTitleModal />
<EditCreateVirtualFileModal />
{isOpenFileImportModal && (

View File

@@ -189,7 +189,7 @@ const DataCard = () => {
textDecoration={'none'}
/>
<Box fontSize={'sm'} color={'myGray.500'}>
ID:{' '}
{t('core.dataset.collection.id')}:{' '}
<Box as={'span'} userSelect={'all'}>
{collection?._id}
</Box>
@@ -229,7 +229,7 @@ const DataCard = () => {
<Flex my={3} alignItems={'center'}>
<Box>
<Box as={'span'} fontSize={['md', 'lg']}>
{total}
{t('core.dataset.data.Total Amount', { total })}
</Box>
</Box>
<Box flex={1} mr={1} />
@@ -243,7 +243,7 @@ const DataCard = () => {
/>
}
w={['200px', '300px']}
placeholder="根据匹配知识,预期答案和来源进行搜索"
placeholder={t('core.dataset.data.Search data placeholder')}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
@@ -410,7 +410,7 @@ const DataCard = () => {
<Flex flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
{t('core.dataset.data.Empty Tip')}
</Box>
</Flex>
)}

View File

@@ -41,7 +41,7 @@ const ChunkImport = () => {
} = useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: `该任务无法终止,需要一定时间生成索引,请确认导入。如果余额不足,未完成的任务会被暂停,充值后可继续进行。`
content: t('core.dataset.import.Import Tip')
});
return (
@@ -63,7 +63,11 @@ const ChunkImport = () => {
}
}}
>
<MyTooltip label={`范围: 100~${datasetDetail.vectorModel.maxToken}`}>
<MyTooltip
label={t('core.dataset.import.Chunk Range', {
max: datasetDetail.vectorModel.maxToken
})}
>
<NumberInput
ml={4}
defaultValue={chunkLen}
@@ -87,20 +91,22 @@ const ChunkImport = () => {
{/* price */}
<Flex py={4} alignItems={'center'}>
<Box>
{t('core.dataset.import.Estimated Price')}
<MyTooltip
label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}
label={t('core.dataset.import.Estimated Price Tips', {
price: formatPrice(unitPrice, 1000)
})}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box ml={4}>{price}</Box>
<Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
</Flex>
<Flex mt={3}>
{showRePreview && (
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
{t('core.dataset.import.Re Preview')}
</Button>
)}
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>

View File

@@ -37,20 +37,20 @@ const CreateFileModal = ({
>
<ModalBody>
<Box mb={1} fontSize={'sm'}>
{t('common.file.File Name')}
</Box>
<Input
mb={5}
{...register('filename', {
required: '文件名不能为空'
required: t('common.file.Filename Can not Be Empty')
})}
/>
<Box mb={1} fontSize={'sm'}>
{t('common.file.File Content')}
</Box>
<Textarea
{...register('content', {
required: '文件内容不能为空'
required: t('common.file.File content can not be empty')
})}
rows={12}
whiteSpace={'nowrap'}
@@ -59,10 +59,10 @@ const CreateFileModal = ({
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
{t('common.Close')}
</Button>
<Button isLoading={isLoading} onClick={mutate}>
{t('common.Confirm Create')}
</Button>
</ModalFooter>
</MyModal>

View File

@@ -16,7 +16,7 @@ const CsvImport = () => {
useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: `该任务无法终止,需要一定时间生成索引,请确认导入。如果余额不足,未完成的任务会被暂停,充值后可继续进行。`
content: t('core.dataset.import.Import Tip')
});
return (
@@ -25,7 +25,7 @@ const CsvImport = () => {
fileExtension={fileExtension}
showUrlFetch={false}
fileTemplate={{
filename: 'csv 模板.csv',
filename: 'csv templates.csv',
value: csvTemplate,
type: 'text/csv'
}}
@@ -33,7 +33,11 @@ const CsvImport = () => {
>
<Flex mt={3}>
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
{uploading ? (
<Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
) : (
t('common.Confirm Import')
)}
</Button>
</Flex>
</SelectorContainer>

View File

@@ -137,7 +137,7 @@ const FileSelect = ({
if (extension === 'csv') {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'index' || header[1] !== 'content') {
throw new Error('csv 文件格式有误,请确保 index 和 content 两列');
throw new Error(t('core.dataset.import.Csv format error'));
}
const filterData = data
@@ -205,7 +205,7 @@ const FileSelect = ({
} catch (error: any) {
console.log(error);
toast({
title: getErrText(error, '解析文件失败'),
title: getErrText(error, t('common.file.Read File Error')),
status: 'error'
});
}

View File

@@ -10,7 +10,6 @@ const CsvImport = dynamic(() => import('./Csv'), {});
import MyModal from '@/components/MyModal';
import Provider from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { qaModelList } from '@/web/common/system/staticData';
import {
DatasetCollectionTrainingModeEnum,
TrainingModeEnum
@@ -96,20 +95,20 @@ const ImportData = ({
list={[
{
icon: 'indexImport',
title: '直接分段',
desc: '选择文本文件,直接将其按分段进行处理',
title: t('core.dataset.import.Chunk Split'),
desc: t('core.dataset.import.Chunk Split Tip'),
value: ImportTypeEnum.chunk
},
{
icon: 'qaImport',
title: 'QA拆分',
desc: '选择文本文件,让大模型自动生成问答对',
title: t('core.dataset.import.QA Import'),
desc: t('core.dataset.import.QA Import Tip'),
value: ImportTypeEnum.qa
},
{
icon: 'csvImport',
title: 'CSV 导入',
desc: '批量导入问答对,是最精准的数据',
title: t('core.dataset.import.CSV Import'),
desc: t('core.dataset.import.CSV Import Tip'),
value: ImportTypeEnum.csv
}
]}

View File

@@ -178,12 +178,12 @@ const Provider = ({
},
onSuccess(num) {
toast({
title: `共成功导入 ${num} 组数据,请耐心等待训练.`,
title: t('core.dataset.import.Import Success Tip', { num }),
status: 'success'
});
onUploadSuccess();
},
errorToast: '导入文件失败'
errorToast: t('core.dataset.import.Import Failed')
});
const onReSplitChunks = useCallback(async () => {
@@ -212,10 +212,10 @@ const Provider = ({
} catch (error) {
toast({
status: 'warning',
title: getErrText(error, '文本分段异常')
title: getErrText(error, t('core.dataset.import.Set Chunk Error'))
});
}
}, [chunkLen, toast]);
}, [chunkLen, chunkOverlapRatio, t, toast]);
const reset = useCallback(() => {
setFiles([]);
@@ -255,6 +255,7 @@ export default React.memo(Provider);
export const PreviewFileOrChunk = () => {
const theme = useTheme();
const { t } = useTranslation();
const { setFiles, previewFile, setPreviewFile, setReShowRePreview, totalChunks, files } =
useImportStore();
@@ -312,11 +313,11 @@ export const PreviewFileOrChunk = () => {
<Box pt={[3, 6]}>
<Flex px={[4, 8]} alignItems={'center'}>
<Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
({totalChunks})
{t('core.dataset.import.Total Chunk Preview', { totalChunks })}
</Box>
{totalChunks > 50 && (
<Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
{t('core.dataset.import.Only Show First 50 Chunk')}
</Box>
)}
</Flex>

View File

@@ -7,10 +7,12 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { Prompt_AgentQA } from '@/global/core/prompt/agent';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useTranslation } from 'next-i18next';
const fileExtension = '.txt, .docx, .pdf, .md';
const QAImport = () => {
const { t } = useTranslation();
const { datasetDetail } = useDatasetStore();
const agentModel = datasetDetail.agentModel;
const unitPrice = agentModel?.price || 3;
@@ -27,7 +29,7 @@ const QAImport = () => {
} = useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: `该任务无法终止!导入后会自动调用大模型生成问答对,会有一些细节丢失,请确认!如果余额不足,未完成的任务会被暂停。`
content: t('core.dataset.import.Import Tip')
});
const [prompt, setPrompt] = useState(Prompt_AgentQA.description);
@@ -38,7 +40,7 @@ const QAImport = () => {
{/* prompt */}
<Box p={3} bg={'myWhite.600'} borderRadius={'md'}>
<Box mb={1} fontWeight={'bold'}>
QA
{t('core.dataset.collection.QA Prompt')}
</Box>
<Box whiteSpace={'pre-wrap'} fontSize={'sm'}>
<Textarea
@@ -55,24 +57,30 @@ const QAImport = () => {
{/* price */}
<Flex py={5} alignItems={'center'}>
<Box>
{t('core.dataset.import.Estimated Price')}
<MyTooltip
label={`索引生成计费为: ${formatPrice(unitPrice, 1000)}/1k tokens`}
label={t('core.dataset.import.Estimated Price Tips', {
price: formatPrice(unitPrice, 1000)
})}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box ml={4}>{price}</Box>
<Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
</Flex>
<Flex mt={3}>
{showRePreview && (
<Button variant={'base'} mr={4} onClick={onReSplitChunks}>
{t('core.dataset.import.Re Preview')}
</Button>
)}
<Button isDisabled={uploading} onClick={openConfirm(() => onclickUpload({ prompt }))}>
{uploading ? <Box>{Math.round((successChunks / totalChunks) * 100)}%</Box> : '确认导入'}
{uploading ? (
<Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
) : (
t('common.Confirm Import')
)}
</Button>
</Flex>
</SelectorContainer>

View File

@@ -1,20 +1,13 @@
import React, {
useCallback,
useState,
useRef,
forwardRef,
useImperativeHandle,
ForwardedRef
} from 'react';
import React, { useCallback, useState, useMemo } from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, Button, FormControl, IconButton, Input, Textarea } from '@chakra-ui/react';
import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { Box, Flex, Button, IconButton, Input, Textarea } from '@chakra-ui/react';
import { DeleteIcon } from '@chakra-ui/icons';
import { delDatasetById } from '@/web/core/dataset/api';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useToast } from '@/web/common/hooks/useToast';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { UseFormReturn } from 'react-hook-form';
import { useForm } from 'react-hook-form';
import { compressImgFileAndUpload } from '@/web/common/file/controller';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import Avatar from '@/components/Avatar';
@@ -23,26 +16,22 @@ import { useTranslation } from 'next-i18next';
import PermissionRadio from '@/components/support/permission/Radio';
import MySelect from '@/components/Select';
import { qaModelList } from '@/web/common/system/staticData';
import { useRequest } from '@/web/common/hooks/useRequest';
const Info = ({
datasetId,
form
}: {
datasetId: string;
form: UseFormReturn<DatasetItemType, any>;
}) => {
const Info = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
const { getValues, formState, setValue, register, handleSubmit } = form;
const InputRef = useRef<HTMLInputElement>(null);
const { datasetDetail, loadDatasets, updateDataset } = useDatasetStore();
const { getValues, setValue, register, handleSubmit } = useForm<DatasetItemType>({
defaultValues: datasetDetail
});
const { toast } = useToast();
const router = useRouter();
const [btnLoading, setBtnLoading] = useState(false);
const [refresh, setRefresh] = useState(false);
const { openConfirm, ConfirmModal } = useConfirm({
content: '确认删除该知识库?数据将无法恢复,请确认!'
content: t('core.dataset.Delete Confirm'),
type: 'delete'
});
const { File, onOpen: onOpenSelectFile } = useSelectFile({
@@ -50,107 +39,68 @@ const Info = ({
multiple: false
});
const { datasetDetail, loadDatasets, updateDataset } = useDatasetStore();
/* 点击删除 */
const onclickDelKb = useCallback(async () => {
setBtnLoading(true);
try {
await delDatasetById(datasetId);
toast({
title: '删除成功',
status: 'success'
});
const { mutate: onclickDelete, isLoading: isDeleting } = useRequest({
mutationFn: () => {
return delDatasetById(datasetId);
},
onSuccess() {
router.replace(`/dataset/list`);
await loadDatasets();
} catch (err: any) {
toast({
title: err?.message || '删除失败',
status: 'error'
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const { mutate: onclickSave, isLoading: isSaving } = useRequest({
mutationFn: (data: DatasetItemType) => {
return updateDataset({
id: datasetId,
...data
});
}
setBtnLoading(false);
}, [setBtnLoading, datasetId, toast, router, loadDatasets]);
const saveSubmitSuccess = useCallback(
async (data: DatasetItemType) => {
setBtnLoading(true);
try {
await updateDataset({
id: datasetId,
...data
});
toast({
title: '更新成功',
status: 'success'
});
loadDatasets();
} catch (err: any) {
toast({
title: err?.message || '更新失败',
status: 'error'
});
}
setBtnLoading(false);
},
[updateDataset, datasetId, toast, loadDatasets]
);
const saveSubmitError = useCallback(() => {
// deep search message
const deepSearch = (obj: any): string => {
if (!obj) return '提交表单错误';
if (!!obj.message) {
return obj.message;
}
return deepSearch(Object.values(obj)[0]);
};
toast({
title: deepSearch(formState.errors),
status: 'error',
duration: 4000,
isClosable: true
});
}, [formState.errors, toast]);
onSuccess() {
loadDatasets();
},
successToast: t('common.Update Success'),
errorToast: t('common.Update Failed')
});
const onSelectFile = useCallback(
async (e: File[]) => {
const { mutate: onSelectFile, isLoading: isSelecting } = useRequest({
mutationFn: (e: File[]) => {
const file = e[0];
if (!file) return;
try {
const src = await compressImgFileAndUpload({
file,
maxW: 300,
maxH: 300
});
if (!file) return Promise.resolve(null);
return compressImgFileAndUpload({
file,
maxW: 300,
maxH: 300
});
},
onSuccess(src: string | null) {
if (src) {
setValue('avatar', src);
setRefresh((state) => !state);
} catch (err: any) {
toast({
title: typeof err === 'string' ? err : '头像选择异常',
status: 'warning'
});
}
},
[setRefresh, setValue, toast]
);
errorToast: t('common.avatar.Select Failed')
});
const btnLoading = useMemo(() => isDeleting || isSaving, [isDeleting, isSaving]);
return (
<Box py={5} px={[5, 10]}>
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
ID
{t('core.dataset.Dataset ID')}
</Box>
<Box flex={1}>{datasetDetail._id}</Box>
</Flex>
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.dataset.Avatar')}
</Box>
<Box flex={[1, '0 0 300px']}>
<MyTooltip label={'点击切换头像'}>
<MyTooltip label={t('common.avatar.Select Avatar')}>
<Avatar
m={'auto'}
src={getValues('avatar')}
@@ -164,19 +114,13 @@ const Info = ({
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.dataset.Name')}
</Box>
<Input
flex={[1, '0 0 300px']}
maxLength={30}
{...register('name', {
required: '知识库名称不能为空'
})}
/>
<Input flex={[1, '0 0 300px']} maxLength={30} {...register('name')} />
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.ai.model.Vector Model')}
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
@@ -188,7 +132,7 @@ const Info = ({
</Flex>
<Flex mt={6} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('dataset.Agent Model')}
{t('core.ai.model.Dataset Agent Model')}
</Box>
<Box flex={[1, '0 0 300px']}>
<MySelect
@@ -234,9 +178,9 @@ const Info = ({
isLoading={btnLoading}
mr={4}
w={'100px'}
onClick={handleSubmit(saveSubmitSuccess, saveSubmitError)}
onClick={handleSubmit((data) => onclickSave(data))}
>
{t('common.Save')}
</Button>
{datasetDetail.isOwner && (
<IconButton
@@ -249,7 +193,7 @@ const Info = ({
color: 'red.600',
borderColor: 'red.600'
}}
onClick={openConfirm(onclickDelKb)}
onClick={openConfirm(onclickDelete)}
/>
)}
</Flex>

View File

@@ -28,6 +28,7 @@ import SideTabs from '@/components/SideTabs';
import { useLoading } from '@/web/common/hooks/useLoading';
import DeleteIcon from '@/components/Icon/delete';
import { defaultCollectionDetail } from '@/constants/dataset';
import { getDocPath } from '@/web/common/system/doc';
export type RawSourceTextProps = BoxProps & {
sourceName?: string;
@@ -122,7 +123,7 @@ const InputDataModal = ({
}
if (countPromptTokens(e.q) >= maxToken) {
return toast({
title: '总长度超长了',
title: t('core.dataset.data.Too Long'),
status: 'warning'
});
}
@@ -214,7 +215,7 @@ const InputDataModal = ({
return openConfirm(onDeleteData)();
}
if (e === TabEnum.doc) {
return window.open(`${feConfigs.docUrl}/docs/use-cases/datasetengine`, '_blank');
return window.open(getDocPath('/docs/use-cases/datasetengine'), '_blank');
}
setCurrentTab(e);
}}
@@ -236,17 +237,15 @@ const InputDataModal = ({
<Box as="span" color={'red.600'}>
*
</Box>
{'相关数据内容'}
{t('core.dataset.data.Data Content')}
</Box>
<MyTooltip
label={'该输入框是必填项\n该内容通常是对于知识点的描述也可以是用户的问题。'}
>
<MyTooltip label={t('core.dataset.data.Data Content Tip')}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
<Textarea
mt={1}
placeholder={`该输入框是必填项,该内容通常是对于知识点的描述,也可以是用户的问题,最多 ${maxToken} 字。`}
placeholder={t('core.dataset.data.Data Content Placeholder', { maxToken })}
maxLength={maxToken}
rows={12}
bg={'myWhite.400'}
@@ -257,20 +256,16 @@ const InputDataModal = ({
</Box>
<Box mt={5}>
<Flex>
<Box>{'辅助数据'}</Box>
<MyTooltip
label={
'该部分为可选填项\n该内容通常是为了与前面的数据内容配合构建结构化提示词用于特殊场景'
}
>
<Box>{t('core.dataset.data.Auxiliary Data')}</Box>
<MyTooltip label={t('core.dataset.data.Auxiliary Data Tip')}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
<Textarea
mt={1}
placeholder={`该部分为可选填项, 通常是为了与前面的【数据内容】配合,构建结构化提示词,用于特殊场景,最多 ${
maxToken * 1.5
} 字。`}
placeholder={t('core.dataset.data.Auxiliary Data Placeholder', {
maxToken: maxToken * 1.5
})}
bg={'myWhite.400'}
rows={12}
maxLength={maxToken * 1.5}
@@ -310,10 +305,7 @@ const InputDataModal = ({
/>
</Flex>
{index.defaultIndex ? (
<Box>
使
</Box>
<Box>{t('core.dataset.data.Default Index Tip')}</Box>
) : (
<Textarea
maxLength={maxToken}
@@ -369,7 +361,7 @@ const InputDataModal = ({
// @ts-ignore
onClick={handleSubmit(defaultValue.id ? onUpdateData : sureImportData)}
>
{defaultValue.id ? '确认变更' : '确认导入'}
{defaultValue.id ? t('common.Confirm Update') : t('common.Confirm Import')}
</Button>
</MyTooltip>
</Flex>

View File

@@ -236,7 +236,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
border={theme.borders.base}
_notLast={{ mb: 2 }}
cursor={'pointer'}
title={'编辑'}
title={t('common.Edit')}
onClick={async () => {
try {
setLoading(true);

View File

@@ -1,10 +1,8 @@
import React, { useCallback, useRef } from 'react';
import React, { useCallback } from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
import { useToast } from '@/web/common/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useQuery } from '@tanstack/react-query';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import Tabs from '@/components/Tabs';
@@ -24,7 +22,13 @@ import Script from 'next/script';
import CollectionCard from './components/CollectionCard';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useUserStore } from '@/web/support/user/useUserStore';
import { DatasetTypeMap } from '../../../../../../packages/global/core/dataset/constant';
import {
DatasetStatusEnum,
DatasetTypeEnum,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constant';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { useRequest } from '@/web/common/hooks/useRequest';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
@@ -46,17 +50,26 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
const { toast } = useToast();
const router = useRouter();
const { isPc } = useSystemStore();
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
const { datasetDetail, loadDatasetDetail, startWebsiteSync } = useDatasetStore();
const { userInfo } = useUserStore();
const tabList = [
{ label: '数据集', id: TabEnum.collectionCard, icon: 'overviewLight' },
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
{ label: t('core.dataset.Dataset'), id: TabEnum.collectionCard, icon: 'overviewLight' },
{ label: t('core.dataset.test.Search Test'), id: TabEnum.test, icon: 'kbTest' },
...(userInfo?.team.canWrite && datasetDetail.isOwner
? [{ label: '配置', id: TabEnum.info, icon: 'settingLight' }]
? [{ label: t('common.Config'), id: TabEnum.info, icon: 'settingLight' }]
: [])
];
const { ConfirmModal: ConfirmSyncModal, openConfirm: openConfirmSync } = useConfirm({
type: 'common'
});
const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({
mutationFn: () => startWebsiteSync(),
errorToast: t('common.Update Failed')
});
const setCurrentTab = useCallback(
(tab: `${TabEnum}`) => {
router.replace({
@@ -69,18 +82,11 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
[datasetId, router]
);
const form = useForm<DatasetItemType>({
defaultValues: datasetDetail
});
useQuery([datasetId], () => loadDatasetDetail(datasetId), {
onSuccess(res) {
form.reset(res);
},
onError(err: any) {
router.replace(`/dataset/list`);
toast({
title: getErrText(err, '获取知识库异常'),
title: getErrText(err, t('common.Load Failed')),
status: 'error'
});
}
@@ -116,7 +122,26 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
mr={1}
w={'16px'}
/>
<Box>{t(DatasetTypeMap[datasetDetail.type]?.label)}</Box>
<Box flex={1}>{t(DatasetTypeMap[datasetDetail.type]?.label)}</Box>
{datasetDetail.type === DatasetTypeEnum.websiteDataset &&
datasetDetail.status === DatasetStatusEnum.active && (
<MyTooltip label={t('core.dataset.website.Start Sync')}>
<MyIcon
mt={1}
name={'common/refreshLight'}
w={'12px'}
color={'myGray.500'}
cursor={'pointer'}
onClick={() =>
openConfirmSync(
onUpdateDatasetWebsiteConfig,
undefined,
t('core.dataset.website.Confirm Create Tips')
)()
}
/>
</MyTooltip>
)}
</Flex>
)}
<SideTabs
@@ -188,11 +213,12 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
{currentTab === TabEnum.collectionCard && <CollectionCard />}
{currentTab === TabEnum.dataCard && <DataCard />}
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
{currentTab === TabEnum.info && <Info datasetId={datasetId} form={form} />}
{currentTab === TabEnum.info && <Info datasetId={datasetId} />}
</Box>
)}
</Flex>
</PageContainer>
<ConfirmSyncModal isLoading={isUpdating} />
</>
);
};

View File

@@ -57,7 +57,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
setRefresh((state) => !state);
} catch (err: any) {
toast({
title: getErrText(err, '头像选择异常'),
title: getErrText(err, t('common.avatar.Select Failed')),
status: 'warning'
});
}
@@ -71,8 +71,8 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
const id = await postCreateDataset(data);
return id;
},
successToast: '创建成功',
errorToast: '创建知识库出现意外',
successToast: t('common.Create Success'),
errorToast: t('common.Create Failed'),
onSuccess(id) {
router.push(`/dataset/detail?datasetId=${id}`);
}
@@ -122,10 +122,10 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
</>
<Box mt={5}>
<Box color={'myGray.800'} fontWeight={'bold'}>
{t('common.Set Name')}
</Box>
<Flex mt={1} alignItems={'center'}>
<MyTooltip label={'点击设置头像'}>
<MyTooltip label={t('common.avatar.Select Avatar')}>
<Avatar
flexShrink={0}
src={getValues('avatar')}
@@ -143,14 +143,12 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
bg={'myWhite.600'}
placeholder={t('common.Name')}
maxLength={30}
{...register('name', {
required: '知识库名称不能为空~'
})}
{...register('name')}
/>
</Flex>
</Box>
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}></Box>
<Box flex={'0 0 100px'}>{t('core.ai.model.Vector Model')}</Box>
<Box flex={1}>
<MySelect
w={'100%'}
@@ -167,7 +165,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
</Box>
</Flex>
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}>{t('dataset.Agent Model')}</Box>
<Box flex={'0 0 100px'}>{t('core.ai.model.Dataset Agent Model')}</Box>
<Box flex={1}>
<MySelect
w={'100%'}

View File

@@ -41,7 +41,7 @@ const MoveModal = ({
() => [
{
parentId: '',
parentName: t('dataset.My Dataset')
parentName: t('core.dataset.My Dataset')
},
...(data?.[1] || [])
],
@@ -95,7 +95,7 @@ const MoveModal = ({
))}
</Flex>
) : (
<Box>我的知识库</Box>
<Box>{t('core.dataset.My Dataset')}</Box>
)}
</>
}

View File

@@ -57,8 +57,8 @@ const Kb = () => {
const DeleteTipsMap = useRef({
[DatasetTypeEnum.folder]: t('dataset.deleteFolderTips'),
[DatasetTypeEnum.dataset]: t('dataset.deleteDatasetTips'),
[DatasetTypeEnum.websiteDataset]: t('core.dataset.Delete Website Tips')
[DatasetTypeEnum.dataset]: t('core.dataset.Delete Confirm'),
[DatasetTypeEnum.websiteDataset]: t('core.dataset.Delete Confirm')
});
const { openConfirm, ConfirmModal } = useConfirm({
@@ -143,7 +143,7 @@ const Kb = () => {
<Flex flex={1} alignItems={'center'}>
<Image src={'/imgs/module/db.png'} alt={''} mr={2} h={'24px'} />
<Box className="textlg" letterSpacing={1} fontSize={'24px'} fontWeight={'bold'}>
{t('dataset.My Dataset')}
{t('core.dataset.My Dataset')}
</Box>
</Flex>
}

View File

@@ -11,6 +11,7 @@ import { feConfigs } from '@/web/common/system/staticData';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyIcon from '@/components/Icon';
import { customAlphabet } from 'nanoid';
import { getDocPath } from '@/web/common/system/doc';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 8);
interface Props {
@@ -146,7 +147,7 @@ const LoginForm = ({ setPageType, loginSuccess }: Props) => {
<Box textAlign={'center'} mt={2} fontSize={'sm'}>
使{' '}
<Link
href={`${feConfigs.docUrl}/docs/intro/#%e5%85%8d%e8%b4%a3%e5%a3%b0%e6%98%8e`}
href={getDocPath('/docs/agreement/disclaimer/')}
target={'_blank'}
color={'myBlue.600'}
>

View File

@@ -6,6 +6,7 @@ import { useRouter } from 'next/router';
import { feConfigs } from '@/web/common/system/staticData';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useTranslation } from 'next-i18next';
import { getDocPath } from '@/web/common/system/doc';
const Tools = () => {
const { t } = useTranslation();
@@ -44,7 +45,7 @@ const Tools = () => {
{
icon: 'common/courseLight',
label: '使用文档',
link: `${feConfigs.docUrl}/docs/intro`
link: getDocPath('/docs/intro')
}
]
: [])

View File

@@ -12,6 +12,8 @@ import { pushDataToDatasetCollection } from '@/pages/api/core/dataset/data/pushD
import { getErrText } from '@fastgpt/global/common/error/utils';
import { authTeamBalance } from '../support/permission/auth/bill';
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
import { UserErrEnum } from '@fastgpt/global/common/error/code/user';
import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
const reduceQueue = (retry = false) => {
global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -91,26 +93,22 @@ export async function generateQA(): Promise<any> {
// auth balance
try {
await authTeamBalance(data.teamId);
} catch (error) {
// send inform and lock data
try {
sendOneInform({
type: 'system',
title: '文本训练任务中止',
content:
'该团队账号余额不足,文本训练任务中止,重新充值后将会继续。暂停的任务将在 7 天后被删除。',
tmbId: data.tmbId
});
console.log('余额不足暂停【QA】生成任务');
await MongoDatasetTraining.updateMany(
{
teamId: data.teamId
},
{
lockTime: new Date('2999/5/5')
}
);
} catch (error) {}
} catch (error: any) {
if (error?.statusText === UserErrEnum.balanceNotEnough) {
// send inform and lock data
try {
sendOneInform({
type: 'system',
title: '文本训练任务中止',
content:
'该团队账号余额不足,文本训练任务中止,重新充值后将会继续。暂停的任务将在 7 天后被删除。',
tmbId: data.tmbId
});
console.log('余额不足暂停【QA】生成任务');
lockTrainingDataByTeamId(data.teamId);
} catch (error) {}
}
reduceQueue();
return generateQA();
}

View File

@@ -6,6 +6,8 @@ import { addLog } from '@fastgpt/service/common/mongo/controller';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
import { UserErrEnum } from '@fastgpt/global/common/error/code/user';
import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
const reduceQueue = (retry = false) => {
global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0;
@@ -93,26 +95,22 @@ export async function generateVector(): Promise<any> {
// auth balance
try {
await authTeamBalance(data.teamId);
} catch (error) {
// send inform and lock data
try {
sendOneInform({
type: 'system',
title: '文本训练任务中止',
content:
'该团队账号余额不足,文本训练任务中止,重新充值后将会继续。暂停的任务将在 7 天后被删除。',
tmbId: data.tmbId
});
console.log('余额不足,暂停【向量】生成任务');
await MongoDatasetTraining.updateMany(
{
teamId: data.teamId
},
{
lockTime: new Date('2999/5/5')
}
);
} catch (error) {}
} catch (error: any) {
if (error?.statusText === UserErrEnum.balanceNotEnough) {
// send inform and lock data
try {
sendOneInform({
type: 'system',
title: '文本训练任务中止',
content:
'该团队账号余额不足,文本训练任务中止,重新充值后将会继续。暂停的任务将在 7 天后被删除。',
tmbId: data.tmbId
});
console.log('余额不足,暂停【向量】生成任务');
lockTrainingDataByTeamId(data.teamId);
} catch (error) {}
}
reduceQueue();
return generateVector();
}

View File

@@ -76,7 +76,8 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
quoteTemplate
});
if (modelConstantsData.censor) {
// censor model and system key
if (modelConstantsData.censor && !user.openaiAccount?.key) {
await postTextCensor({
text: `${systemPrompt}
${quoteText}

View File

@@ -57,7 +57,7 @@ export const useConfirm = (props?: {
[onOpen]
),
ConfirmModal: useCallback(
() => (
({ isLoading }: { isLoading?: boolean }) => (
<MyModal
isOpen={isOpen}
onClose={onClose}
@@ -82,6 +82,7 @@ export const useConfirm = (props?: {
<Button
{...(bg && { bg: `${bg} !important` })}
ml={4}
isLoading={isLoading}
onClick={() => {
onClose();
typeof confirmCb.current === 'function' && confirmCb.current();

View File

@@ -0,0 +1,7 @@
import { feConfigs } from './staticData';
export const getDocPath = (path: string) => {
if (!feConfigs?.docUrl) return '';
if (feConfigs.docUrl.endsWith('/')) return feConfigs.docUrl;
return feConfigs.docUrl + path;
};

View File

@@ -46,6 +46,11 @@ export const putDatasetById = (data: DatasetUpdateBody) => PUT<void>(`/core/data
export const delDatasetById = (id: string) => DELETE(`/core/dataset/delete?id=${id}`);
export const postWebsiteSync = (data: PostWebsiteSyncParams) =>
POST(`/plusApi/core/dataset/websiteSync`, data, {
timeout: 600000
}).catch();
export const getCheckExportLimit = (datasetId: string) =>
GET(`/core/dataset/checkExportLimit`, { datasetId });
@@ -66,10 +71,8 @@ export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
POST(`/core/dataset/collection/update`, data);
export const delDatasetCollectionById = (params: { collectionId: string }) =>
DELETE(`/core/dataset/collection/delete`, params);
export const postWebsiteSync = (data: PostWebsiteSyncParams) =>
POST(`/plusApi/core/dataset/websiteSync`, data, {
timeout: 600000
}).catch();
export const postLinkCollectionSync = (collectionId: string) =>
POST(`/core/dataset/collection/sync/link`, { collectionId });
/* =============================== data ==================================== */
/* get dataset list */

View File

@@ -2,9 +2,17 @@ import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { DatasetItemType, DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { getAllDataset, getDatasets, getDatasetById, putDatasetById } from '@/web/core/dataset/api';
import {
getAllDataset,
getDatasets,
getDatasetById,
putDatasetById,
postWebsiteSync
} from '@/web/core/dataset/api';
import { defaultDatasetDetail } from '@/constants/dataset';
import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constant';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
type State = {
allDatasets: DatasetListItemType[];
@@ -15,6 +23,7 @@ type State = {
datasetDetail: DatasetItemType;
loadDatasetDetail: (id: string, init?: boolean) => Promise<DatasetItemType>;
updateDataset: (data: DatasetUpdateBody) => Promise<any>;
startWebsiteSync: () => Promise<any>;
};
export const useDatasetStore = create<State>()(
@@ -76,6 +85,22 @@ export const useDatasetStore = create<State>()(
: item
);
});
},
async startWebsiteSync() {
const [_, billId] = await Promise.all([
get().updateDataset({
id: get().datasetDetail._id,
status: DatasetStatusEnum.syncing
}),
postCreateTrainingBill({
name: 'core.dataset.training.Website Sync',
vectorModel: get().datasetDetail.vectorModel.model,
agentModel: get().datasetDetail.agentModel.model
})
]);
try {
postWebsiteSync({ datasetId: get().datasetDetail._id, billId });
} catch (error) {}
}
})),
{