feat: callback training data

This commit is contained in:
archer
2023-05-09 13:20:52 +08:00
parent fac53923dd
commit f52f514f5f
4 changed files with 32 additions and 9 deletions

View File

@@ -52,7 +52,10 @@ export const getExportDataList = (modelId: string) =>
* 获取模型正在拆分数据的数量
*/
export const getModelSplitDataListLen = (modelId: string) =>
GET<number>(`/model/data/getSplitData?modelId=${modelId}`);
GET<{
splitDataQueue: number;
embeddingQueue: number;
}>(`/model/data/getTrainingData?modelId=${modelId}`);
/**
* 获取 web 页面内容

View File

@@ -2,6 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, SplitData, Model } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { ModelDataStatusEnum } from '@/constants/model';
import { PgClient } from '@/service/pg';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -14,15 +16,29 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const userId = await authToken(req);
// 找到长度大于0的数据
// split queue data
const data = await SplitData.find({
userId,
modelId,
textList: { $exists: true, $not: { $size: 0 } }
});
// embedding queue data
const where: any = [
['user_id', userId],
'AND',
['model_id', modelId],
'AND',
['status', ModelDataStatusEnum.waiting]
];
jsonRes(res, {
data: data.map((item) => item.textList).flat().length
data: {
splitDataQueue: data.map((item) => item.textList).flat().length,
embeddingQueue: await PgClient.count('modelData', {
where
})
}
});
} catch (err) {
jsonRes(res, {

View File

@@ -1,6 +1,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, SplitData, Model } from '@/service/mongo';
import { connectToDatabase, SplitData } from '@/service/mongo';
import { authModel, authToken } from '@/service/utils/auth';
import { generateVector } from '@/service/events/generateVector';
import { generateQA } from '@/service/events/generateQA';
@@ -65,7 +65,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
export const config = {
api: {
bodyParser: {
sizeLimit: '10mb'
sizeLimit: '100mb'
}
}
};

View File

@@ -88,7 +88,7 @@ const ModelDataCard = ({ modelId, isOwner }: { modelId: string; isOwner: boolean
onClose: onCloseSelectCsvModal
} = useDisclosure();
const { data: splitDataLen = 0, refetch } = useQuery(
const { data: { splitDataQueue = 0, embeddingQueue = 0 } = {}, refetch } = useQuery(
['getModelSplitDataList'],
() => getModelSplitDataListLen(modelId),
{
@@ -109,7 +109,7 @@ const ModelDataCard = ({ modelId, isOwner }: { modelId: string; isOwner: boolean
useQuery(['refetchData'], () => refetchData(pageNum), {
refetchInterval: 5000,
enabled: splitDataLen > 0
enabled: splitDataQueue > 0 || embeddingQueue > 0
});
// 获取所有的数据,并导出 json
@@ -186,8 +186,12 @@ const ModelDataCard = ({ modelId, isOwner }: { modelId: string; isOwner: boolean
)}
</Flex>
<Flex mt={4}>
{isOwner && splitDataLen > 0 && (
<Box fontSize={'xs'}>{splitDataLen}...</Box>
{isOwner && (splitDataQueue > 0 || embeddingQueue > 0) && (
<Box fontSize={'xs'}>
{splitDataQueue > 0 ? `${splitDataQueue}条数据正在拆分,` : ''}
{embeddingQueue > 0 ? `${embeddingQueue}条数据正在生成索引,` : ''}
...
</Box>
)}
<Box flex={1} />
<Input