feat: 数据集合管理

This commit is contained in:
archer
2023-03-25 12:55:32 +08:00
parent 8a9f1ed29b
commit 3db690773f
14 changed files with 244 additions and 13 deletions

View File

@@ -11,3 +11,8 @@ export const postData = (name: string) => POST<string>(`/data/postData?name=${na
export const postSplitData = (dataId: string, text: string) =>
POST(`/data/splitData`, { dataId, text });
export const updateDataName = (dataId: string, name: string) =>
PUT(`/data/putDataName?dataId=${dataId}&name=${name}`);
export const delData = (dataId: string) => DELETE(`/data/delData?dataId=${dataId}`);

33
src/hooks/useRequest.tsx Normal file
View File

@@ -0,0 +1,33 @@
import { useToast } from '@/hooks/useToast';
import { useMutation } from '@tanstack/react-query';
import type { UseMutationOptions } from '@tanstack/react-query';
interface Props extends UseMutationOptions<any, any, any, any> {
successToast?: string;
errorToast?: string;
}
export const useRequest = ({ successToast, errorToast, onSuccess, onError, ...props }: Props) => {
const { toast } = useToast();
const mutation = useMutation<unknown, unknown, any, unknown>({
...props,
onSuccess(res, variables: void, context: unknown) {
onSuccess?.(res, variables, context);
successToast &&
toast({
title: successToast,
status: 'success'
});
},
onError(err: any, variables: void, context: unknown) {
onError?.(err, variables, context);
errorToast &&
toast({
title: typeof err === 'string' ? err : err?.message || errorToast,
status: 'error'
});
}
});
return mutation;
};

View File

@@ -0,0 +1,37 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '@/types';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
if (!authorization) {
throw new Error('缺少登录凭证');
}
await authToken(authorization);
const { dataId } = req.query as { dataId: string };
if (!dataId) {
throw new Error('缺少参数');
}
await connectToDatabase();
await Data.findByIdAndUpdate(dataId, {
isDeleted: true
});
jsonRes<PagingData<DataListItem>>(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -28,7 +28,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const datalist = await Data.aggregate<DataListItem>([
{
$match: {
userId: new mongoose.Types.ObjectId(userId)
userId: new mongoose.Types.ObjectId(userId),
isDeleted: false
}
},
{

View File

@@ -0,0 +1,37 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '@/types';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
if (!authorization) {
throw new Error('缺少登录凭证');
}
await authToken(authorization);
const { dataId, name } = req.query as { dataId: string; name: string };
if (!dataId || !name) {
throw new Error('缺少参数');
}
await connectToDatabase();
await Data.findByIdAndUpdate(dataId, {
name
});
jsonRes<PagingData<DataListItem>>(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -5,6 +5,9 @@ import { AuthCode } from '@/service/models/authCode';
import { connectToDatabase } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
try {
await connectToDatabase();

View File

@@ -4,6 +4,9 @@ import { connectToDatabase, Chat } from '@/service/mongo';
/* 定时删除那些不活跃的内容 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
try {
await connectToDatabase();

View File

@@ -7,6 +7,9 @@ import type { BillSchema } from '@/types/mongoSchema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
await connectToDatabase();
await Bill.updateMany(

View File

@@ -0,0 +1,36 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, DataItem, Data } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
await connectToDatabase();
// await DataItem.updateMany(
// {},
// {
// times: 2
// }
// );
await Data.updateMany(
{},
{
isDeleted: false
}
);
jsonRes(res, {
data: {}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -11,6 +11,9 @@ import { sendTrainSucceed } from '@/service/utils/sendEmail';
import { httpsAgent } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
try {
await connectToDatabase();

15
src/pages/data/defail.tsx Normal file
View File

@@ -0,0 +1,15 @@
import React from 'react';
const DataDetail = ({ dataId }: { dataId: string }) => {
return <div>DataDetail</div>;
};
export default DataDetail;
export async function getServerSideProps(context: any) {
const dataId = context.query?.dataId || '';
return {
props: { dataId }
};
}

View File

@@ -11,14 +11,17 @@ import {
Th,
Td,
TableContainer,
useDisclosure
useDisclosure,
Input
} from '@chakra-ui/react';
import { getDataList } from '@/api/data';
import { getDataList, updateDataName, delData } from '@/api/data';
import { usePaging } from '@/hooks/usePaging';
import type { DataListItem } from '@/types/data';
import ScrollData from '@/components/ScrollData';
import dayjs from 'dayjs';
import dynamic from 'next/dynamic';
import { useConfirm } from '@/hooks/useConfirm';
import { useRequest } from '@/hooks/useRequest';
const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
@@ -34,6 +37,9 @@ const DataList = () => {
pageSize: 20
});
const [ImportDataId, setImportDataId] = useState<string>();
const { openConfirm, ConfirmChild } = useConfirm({
content: '删除数据集,将删除里面的所有内容,请确认!'
});
const {
isOpen: isOpenCreateDataModal,
@@ -41,6 +47,15 @@ const DataList = () => {
onClose: onCloseCreateDataModal
} = useDisclosure();
const { mutate: handleDelData, isLoading: isDeleting } = useRequest({
mutationFn: (dataId: string) => delData(dataId),
successToast: '删除数据集成功',
errorToast: '删除数据集异常',
onSuccess() {
getData(1, true);
}
});
return (
<Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}>
<Card px={6} py={4}>
@@ -74,12 +89,31 @@ const DataList = () => {
<Tbody>
{dataList.map((item, i) => (
<Tr key={item._id}>
<Td>{item.name}</Td>
<Td>
<Input
placeholder="请输入数据集名称"
defaultValue={item.name}
size={'sm'}
onBlur={(e) => {
if (!e.target.value) return;
updateDataName(item._id, e.target.value);
}}
/>
</Td>
<Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>
{item.trainingData} / {item.totalData}
</Td>
<Td>
<Button
size={'sm'}
variant={'outline'}
colorScheme={'gray'}
mr={2}
onClick={() => setImportDataId(item._id)}
>
</Button>
<Button
size={'sm'}
variant={'outline'}
@@ -88,7 +122,17 @@ const DataList = () => {
>
</Button>
<Button size={'sm'}></Button>
<Button mr={2} size={'sm'}>
</Button>
<Button
size={'sm'}
colorScheme={'red'}
isLoading={isDeleting}
onClick={openConfirm(() => handleDelData(item._id))}
>
</Button>
</Td>
</Tr>
))}
@@ -104,6 +148,7 @@ const DataList = () => {
{isOpenCreateDataModal && (
<CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
)}
<ConfirmChild />
</Box>
);
};

View File

@@ -35,10 +35,7 @@ export async function generateQA(next = false): Promise<any> {
// 减少一次重试次数, 并更新状态为生成中
await DataItem.findByIdAndUpdate(dataItem._id, {
status: 2,
$inc: {
time: -1
}
status: 2
});
// 获取 openapi Key
@@ -86,22 +83,31 @@ export async function generateQA(next = false): Promise<any> {
const splitResponse = splitText(content || '');
// 插入数据库,并修改状态
await DataItem.findByIdAndUpdate(dataItem._id, {
status: dataItem.temperature >= 100 ? 0 : 1,
temperature: dataItem.temperature >= 100 ? dataItem.temperature : dataItem.temperature + 25,
status: dataItem.temperature >= 80 ? 0 : 1, // 需要生成 5 组内容。0,0.2,0.4,0.6,0.8
temperature: dataItem.temperature >= 80 ? dataItem.temperature : dataItem.temperature + 20,
$push: {
result: {
$each: splitResponse
}
}
});
console.log('生成成功time:', `${(Date.now() - startTime) / 1000}s`);
console.log(
'生成成功time:',
`${(Date.now() - startTime) / 1000}s`,
'result length: ',
splitResponse.length
);
} catch (error: any) {
console.log('error: 生成QA错误', dataItem?._id);
console.log('response:', error?.response);
// 重置状态
if (dataItem?._id) {
await DataItem.findByIdAndUpdate(dataItem._id, {
status: dataItem.times > 0 ? 1 : 0 // 还有重试次数则可以继续进行
status: dataItem.times > 0 ? 1 : 0, // 还有重试次数则可以继续进行
$inc: {
// 剩余尝试次数-1
times: -1
}
});
}
}

View File

@@ -13,6 +13,10 @@ const DataSchema = new Schema({
createTime: {
type: Date,
default: () => new Date()
},
isDeleted: {
type: Boolean,
default: false
}
});