mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-25 06:14:06 +00:00
feat: 数据集合管理
This commit is contained in:
@@ -11,3 +11,8 @@ export const postData = (name: string) => POST<string>(`/data/postData?name=${na
|
||||
|
||||
export const postSplitData = (dataId: string, text: string) =>
|
||||
POST(`/data/splitData`, { dataId, text });
|
||||
|
||||
export const updateDataName = (dataId: string, name: string) =>
|
||||
PUT(`/data/putDataName?dataId=${dataId}&name=${name}`);
|
||||
|
||||
export const delData = (dataId: string) => DELETE(`/data/delData?dataId=${dataId}`);
|
||||
|
33
src/hooks/useRequest.tsx
Normal file
33
src/hooks/useRequest.tsx
Normal file
@@ -0,0 +1,33 @@
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import type { UseMutationOptions } from '@tanstack/react-query';
|
||||
|
||||
interface Props extends UseMutationOptions<any, any, any, any> {
|
||||
successToast?: string;
|
||||
errorToast?: string;
|
||||
}
|
||||
|
||||
export const useRequest = ({ successToast, errorToast, onSuccess, onError, ...props }: Props) => {
|
||||
const { toast } = useToast();
|
||||
const mutation = useMutation<unknown, unknown, any, unknown>({
|
||||
...props,
|
||||
onSuccess(res, variables: void, context: unknown) {
|
||||
onSuccess?.(res, variables, context);
|
||||
successToast &&
|
||||
toast({
|
||||
title: successToast,
|
||||
status: 'success'
|
||||
});
|
||||
},
|
||||
onError(err: any, variables: void, context: unknown) {
|
||||
onError?.(err, variables, context);
|
||||
errorToast &&
|
||||
toast({
|
||||
title: typeof err === 'string' ? err : err?.message || errorToast,
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return mutation;
|
||||
};
|
37
src/pages/api/data/delData.ts
Normal file
37
src/pages/api/data/delData.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@/service/response';
|
||||
import { connectToDatabase, Data } from '@/service/mongo';
|
||||
import { authToken } from '@/service/utils/tools';
|
||||
import type { DataListItem } from '@/types/data';
|
||||
import type { PagingData } from '@/types';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { authorization } = req.headers;
|
||||
|
||||
if (!authorization) {
|
||||
throw new Error('缺少登录凭证');
|
||||
}
|
||||
|
||||
await authToken(authorization);
|
||||
|
||||
const { dataId } = req.query as { dataId: string };
|
||||
if (!dataId) {
|
||||
throw new Error('缺少参数');
|
||||
}
|
||||
|
||||
await connectToDatabase();
|
||||
|
||||
await Data.findByIdAndUpdate(dataId, {
|
||||
isDeleted: true
|
||||
});
|
||||
|
||||
jsonRes<PagingData<DataListItem>>(res);
|
||||
} catch (err) {
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error: err
|
||||
});
|
||||
}
|
||||
}
|
@@ -28,7 +28,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
|
||||
const datalist = await Data.aggregate<DataListItem>([
|
||||
{
|
||||
$match: {
|
||||
userId: new mongoose.Types.ObjectId(userId)
|
||||
userId: new mongoose.Types.ObjectId(userId),
|
||||
isDeleted: false
|
||||
}
|
||||
},
|
||||
{
|
||||
|
37
src/pages/api/data/putDataName.ts
Normal file
37
src/pages/api/data/putDataName.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@/service/response';
|
||||
import { connectToDatabase, Data } from '@/service/mongo';
|
||||
import { authToken } from '@/service/utils/tools';
|
||||
import type { DataListItem } from '@/types/data';
|
||||
import type { PagingData } from '@/types';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { authorization } = req.headers;
|
||||
|
||||
if (!authorization) {
|
||||
throw new Error('缺少登录凭证');
|
||||
}
|
||||
|
||||
await authToken(authorization);
|
||||
|
||||
const { dataId, name } = req.query as { dataId: string; name: string };
|
||||
if (!dataId || !name) {
|
||||
throw new Error('缺少参数');
|
||||
}
|
||||
|
||||
await connectToDatabase();
|
||||
|
||||
await Data.findByIdAndUpdate(dataId, {
|
||||
name
|
||||
});
|
||||
|
||||
jsonRes<PagingData<DataListItem>>(res);
|
||||
} catch (err) {
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error: err
|
||||
});
|
||||
}
|
||||
}
|
@@ -5,6 +5,9 @@ import { AuthCode } from '@/service/models/authCode';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
if (req.headers.auth !== 'archer') {
|
||||
throw new Error('凭证错误');
|
||||
}
|
||||
try {
|
||||
await connectToDatabase();
|
||||
|
||||
|
@@ -4,6 +4,9 @@ import { connectToDatabase, Chat } from '@/service/mongo';
|
||||
|
||||
/* 定时删除那些不活跃的内容 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
if (req.headers.auth !== 'archer') {
|
||||
throw new Error('凭证错误');
|
||||
}
|
||||
try {
|
||||
await connectToDatabase();
|
||||
|
||||
|
@@ -7,6 +7,9 @@ import type { BillSchema } from '@/types/mongoSchema';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
if (req.headers.auth !== 'archer') {
|
||||
throw new Error('凭证错误');
|
||||
}
|
||||
await connectToDatabase();
|
||||
|
||||
await Bill.updateMany(
|
||||
|
36
src/pages/api/timer/initDataItemTime.ts
Normal file
36
src/pages/api/timer/initDataItemTime.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@/service/response';
|
||||
import { connectToDatabase, DataItem, Data } from '@/service/mongo';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
if (req.headers.auth !== 'archer') {
|
||||
throw new Error('凭证错误');
|
||||
}
|
||||
await connectToDatabase();
|
||||
|
||||
// await DataItem.updateMany(
|
||||
// {},
|
||||
// {
|
||||
// times: 2
|
||||
// }
|
||||
// );
|
||||
|
||||
await Data.updateMany(
|
||||
{},
|
||||
{
|
||||
isDeleted: false
|
||||
}
|
||||
);
|
||||
|
||||
jsonRes(res, {
|
||||
data: {}
|
||||
});
|
||||
} catch (err) {
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error: err
|
||||
});
|
||||
}
|
||||
}
|
@@ -11,6 +11,9 @@ import { sendTrainSucceed } from '@/service/utils/sendEmail';
|
||||
import { httpsAgent } from '@/service/utils/tools';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
if (req.headers.auth !== 'archer') {
|
||||
throw new Error('凭证错误');
|
||||
}
|
||||
try {
|
||||
await connectToDatabase();
|
||||
|
||||
|
15
src/pages/data/defail.tsx
Normal file
15
src/pages/data/defail.tsx
Normal file
@@ -0,0 +1,15 @@
|
||||
import React from 'react';
|
||||
|
||||
const DataDetail = ({ dataId }: { dataId: string }) => {
|
||||
return <div>DataDetail</div>;
|
||||
};
|
||||
|
||||
export default DataDetail;
|
||||
|
||||
export async function getServerSideProps(context: any) {
|
||||
const dataId = context.query?.dataId || '';
|
||||
|
||||
return {
|
||||
props: { dataId }
|
||||
};
|
||||
}
|
@@ -11,14 +11,17 @@ import {
|
||||
Th,
|
||||
Td,
|
||||
TableContainer,
|
||||
useDisclosure
|
||||
useDisclosure,
|
||||
Input
|
||||
} from '@chakra-ui/react';
|
||||
import { getDataList } from '@/api/data';
|
||||
import { getDataList, updateDataName, delData } from '@/api/data';
|
||||
import { usePaging } from '@/hooks/usePaging';
|
||||
import type { DataListItem } from '@/types/data';
|
||||
import ScrollData from '@/components/ScrollData';
|
||||
import dayjs from 'dayjs';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { useRequest } from '@/hooks/useRequest';
|
||||
|
||||
const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
|
||||
const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
|
||||
@@ -34,6 +37,9 @@ const DataList = () => {
|
||||
pageSize: 20
|
||||
});
|
||||
const [ImportDataId, setImportDataId] = useState<string>();
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: '删除数据集,将删除里面的所有内容,请确认!'
|
||||
});
|
||||
|
||||
const {
|
||||
isOpen: isOpenCreateDataModal,
|
||||
@@ -41,6 +47,15 @@ const DataList = () => {
|
||||
onClose: onCloseCreateDataModal
|
||||
} = useDisclosure();
|
||||
|
||||
const { mutate: handleDelData, isLoading: isDeleting } = useRequest({
|
||||
mutationFn: (dataId: string) => delData(dataId),
|
||||
successToast: '删除数据集成功',
|
||||
errorToast: '删除数据集异常',
|
||||
onSuccess() {
|
||||
getData(1, true);
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}>
|
||||
<Card px={6} py={4}>
|
||||
@@ -74,12 +89,31 @@ const DataList = () => {
|
||||
<Tbody>
|
||||
{dataList.map((item, i) => (
|
||||
<Tr key={item._id}>
|
||||
<Td>{item.name}</Td>
|
||||
<Td>
|
||||
<Input
|
||||
placeholder="请输入数据集名称"
|
||||
defaultValue={item.name}
|
||||
size={'sm'}
|
||||
onBlur={(e) => {
|
||||
if (!e.target.value) return;
|
||||
updateDataName(item._id, e.target.value);
|
||||
}}
|
||||
/>
|
||||
</Td>
|
||||
<Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
|
||||
<Td>
|
||||
{item.trainingData} / {item.totalData}
|
||||
</Td>
|
||||
<Td>
|
||||
<Button
|
||||
size={'sm'}
|
||||
variant={'outline'}
|
||||
colorScheme={'gray'}
|
||||
mr={2}
|
||||
onClick={() => setImportDataId(item._id)}
|
||||
>
|
||||
详细
|
||||
</Button>
|
||||
<Button
|
||||
size={'sm'}
|
||||
variant={'outline'}
|
||||
@@ -88,7 +122,17 @@ const DataList = () => {
|
||||
>
|
||||
导入
|
||||
</Button>
|
||||
<Button size={'sm'}>导出</Button>
|
||||
<Button mr={2} size={'sm'}>
|
||||
导出
|
||||
</Button>
|
||||
<Button
|
||||
size={'sm'}
|
||||
colorScheme={'red'}
|
||||
isLoading={isDeleting}
|
||||
onClick={openConfirm(() => handleDelData(item._id))}
|
||||
>
|
||||
删除
|
||||
</Button>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
@@ -104,6 +148,7 @@ const DataList = () => {
|
||||
{isOpenCreateDataModal && (
|
||||
<CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
|
||||
)}
|
||||
<ConfirmChild />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
@@ -35,10 +35,7 @@ export async function generateQA(next = false): Promise<any> {
|
||||
|
||||
// 减少一次重试次数, 并更新状态为生成中
|
||||
await DataItem.findByIdAndUpdate(dataItem._id, {
|
||||
status: 2,
|
||||
$inc: {
|
||||
time: -1
|
||||
}
|
||||
status: 2
|
||||
});
|
||||
|
||||
// 获取 openapi Key
|
||||
@@ -86,22 +83,31 @@ export async function generateQA(next = false): Promise<any> {
|
||||
const splitResponse = splitText(content || '');
|
||||
// 插入数据库,并修改状态
|
||||
await DataItem.findByIdAndUpdate(dataItem._id, {
|
||||
status: dataItem.temperature >= 100 ? 0 : 1,
|
||||
temperature: dataItem.temperature >= 100 ? dataItem.temperature : dataItem.temperature + 25,
|
||||
status: dataItem.temperature >= 80 ? 0 : 1, // 需要生成 5 组内容。0,0.2,0.4,0.6,0.8
|
||||
temperature: dataItem.temperature >= 80 ? dataItem.temperature : dataItem.temperature + 20,
|
||||
$push: {
|
||||
result: {
|
||||
$each: splitResponse
|
||||
}
|
||||
}
|
||||
});
|
||||
console.log('生成成功,time:', `${(Date.now() - startTime) / 1000}s`);
|
||||
console.log(
|
||||
'生成成功,time:',
|
||||
`${(Date.now() - startTime) / 1000}s`,
|
||||
'result length: ',
|
||||
splitResponse.length
|
||||
);
|
||||
} catch (error: any) {
|
||||
console.log('error: 生成QA错误', dataItem?._id);
|
||||
console.log('response:', error?.response);
|
||||
// 重置状态
|
||||
if (dataItem?._id) {
|
||||
await DataItem.findByIdAndUpdate(dataItem._id, {
|
||||
status: dataItem.times > 0 ? 1 : 0 // 还有重试次数则可以继续进行
|
||||
status: dataItem.times > 0 ? 1 : 0, // 还有重试次数则可以继续进行
|
||||
$inc: {
|
||||
// 剩余尝试次数-1
|
||||
times: -1
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@@ -13,6 +13,10 @@ const DataSchema = new Schema({
|
||||
createTime: {
|
||||
type: Date,
|
||||
default: () => new Date()
|
||||
},
|
||||
isDeleted: {
|
||||
type: Boolean,
|
||||
default: false
|
||||
}
|
||||
});
|
||||
|
||||
|
Reference in New Issue
Block a user