mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 21:13:50 +00:00
feat: callback training data
This commit is contained in:
@@ -52,7 +52,10 @@ export const getExportDataList = (modelId: string) =>
|
||||
* 获取模型正在拆分数据的数量
|
||||
*/
|
||||
export const getModelSplitDataListLen = (modelId: string) =>
|
||||
GET<number>(`/model/data/getSplitData?modelId=${modelId}`);
|
||||
GET<{
|
||||
splitDataQueue: number;
|
||||
embeddingQueue: number;
|
||||
}>(`/model/data/getTrainingData?modelId=${modelId}`);
|
||||
|
||||
/**
|
||||
* 获取 web 页面内容
|
||||
|
@@ -2,6 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@/service/response';
|
||||
import { connectToDatabase, SplitData, Model } from '@/service/mongo';
|
||||
import { authToken } from '@/service/utils/auth';
|
||||
import { ModelDataStatusEnum } from '@/constants/model';
|
||||
import { PgClient } from '@/service/pg';
|
||||
|
||||
/* 拆分数据成QA */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
@@ -14,15 +16,29 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
|
||||
|
||||
const userId = await authToken(req);
|
||||
|
||||
// 找到长度大于0的数据
|
||||
// split queue data
|
||||
const data = await SplitData.find({
|
||||
userId,
|
||||
modelId,
|
||||
textList: { $exists: true, $not: { $size: 0 } }
|
||||
});
|
||||
|
||||
// embedding queue data
|
||||
const where: any = [
|
||||
['user_id', userId],
|
||||
'AND',
|
||||
['model_id', modelId],
|
||||
'AND',
|
||||
['status', ModelDataStatusEnum.waiting]
|
||||
];
|
||||
|
||||
jsonRes(res, {
|
||||
data: data.map((item) => item.textList).flat().length
|
||||
data: {
|
||||
splitDataQueue: data.map((item) => item.textList).flat().length,
|
||||
embeddingQueue: await PgClient.count('modelData', {
|
||||
where
|
||||
})
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
jsonRes(res, {
|
@@ -1,6 +1,6 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@/service/response';
|
||||
import { connectToDatabase, SplitData, Model } from '@/service/mongo';
|
||||
import { connectToDatabase, SplitData } from '@/service/mongo';
|
||||
import { authModel, authToken } from '@/service/utils/auth';
|
||||
import { generateVector } from '@/service/events/generateVector';
|
||||
import { generateQA } from '@/service/events/generateQA';
|
||||
@@ -65,7 +65,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
|
||||
export const config = {
|
||||
api: {
|
||||
bodyParser: {
|
||||
sizeLimit: '10mb'
|
||||
sizeLimit: '100mb'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
@@ -88,7 +88,7 @@ const ModelDataCard = ({ modelId, isOwner }: { modelId: string; isOwner: boolean
|
||||
onClose: onCloseSelectCsvModal
|
||||
} = useDisclosure();
|
||||
|
||||
const { data: splitDataLen = 0, refetch } = useQuery(
|
||||
const { data: { splitDataQueue = 0, embeddingQueue = 0 } = {}, refetch } = useQuery(
|
||||
['getModelSplitDataList'],
|
||||
() => getModelSplitDataListLen(modelId),
|
||||
{
|
||||
@@ -109,7 +109,7 @@ const ModelDataCard = ({ modelId, isOwner }: { modelId: string; isOwner: boolean
|
||||
|
||||
useQuery(['refetchData'], () => refetchData(pageNum), {
|
||||
refetchInterval: 5000,
|
||||
enabled: splitDataLen > 0
|
||||
enabled: splitDataQueue > 0 || embeddingQueue > 0
|
||||
});
|
||||
|
||||
// 获取所有的数据,并导出 json
|
||||
@@ -186,8 +186,12 @@ const ModelDataCard = ({ modelId, isOwner }: { modelId: string; isOwner: boolean
|
||||
)}
|
||||
</Flex>
|
||||
<Flex mt={4}>
|
||||
{isOwner && splitDataLen > 0 && (
|
||||
<Box fontSize={'xs'}>{splitDataLen}条数据正在拆分,请耐心等待...</Box>
|
||||
{isOwner && (splitDataQueue > 0 || embeddingQueue > 0) && (
|
||||
<Box fontSize={'xs'}>
|
||||
{splitDataQueue > 0 ? `${splitDataQueue}条数据正在拆分,` : ''}
|
||||
{embeddingQueue > 0 ? `${embeddingQueue}条数据正在生成索引,` : ''}
|
||||
请耐心等待...
|
||||
</Box>
|
||||
)}
|
||||
<Box flex={1} />
|
||||
<Input
|
||||
|
Reference in New Issue
Block a user