mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 13:03:50 +00:00
feat: 一次性获取data集合
This commit is contained in:
@@ -6,8 +6,7 @@ import type { PagingData } from '../types/index';
|
||||
import type { DataItemSchema } from '@/types/mongoSchema';
|
||||
import type { CreateDataProps } from '@/pages/data/components/CreateDataModal';
|
||||
|
||||
export const getDataList = (data: RequestPaging) =>
|
||||
GET<PagingData<DataListItem>>(`/data/getDataList?${Obj2Query(data)}`);
|
||||
export const getDataList = () => GET<DataListItem[]>(`/data/getDataList`);
|
||||
|
||||
export const postData = (data: CreateDataProps) => POST<string>(`/data/postData`, data);
|
||||
|
||||
|
@@ -3,18 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@/service/response';
|
||||
import { connectToDatabase, Data, DataItem } from '@/service/mongo';
|
||||
import { authToken } from '@/service/utils/tools';
|
||||
import type { DataSchema } from '@/types/mongoSchema';
|
||||
import type { DataListItem } from '@/types/data';
|
||||
import type { PagingData } from '@/types';
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const { authorization } = req.headers;
|
||||
let { pageNum = 1, pageSize = 10 } = req.query as { pageNum: string; pageSize: string };
|
||||
|
||||
pageNum = +pageNum;
|
||||
pageSize = +pageSize;
|
||||
|
||||
if (!authorization) {
|
||||
throw new Error('缺少登录凭证');
|
||||
@@ -34,12 +28,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
|
||||
{
|
||||
$sort: { createTime: -1 } // 按照创建时间倒序排列
|
||||
},
|
||||
{
|
||||
$skip: (pageNum - 1) * pageSize // 跳过前面的数据
|
||||
},
|
||||
{
|
||||
$limit: pageSize // 取出指定数量的数据
|
||||
},
|
||||
{
|
||||
$lookup: {
|
||||
from: 'dataitems',
|
||||
@@ -71,13 +59,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
|
||||
}
|
||||
]);
|
||||
|
||||
jsonRes<PagingData<DataListItem>>(res, {
|
||||
data: {
|
||||
pageNum,
|
||||
pageSize,
|
||||
data: datalist,
|
||||
total: 1
|
||||
}
|
||||
jsonRes(res, {
|
||||
data: datalist
|
||||
});
|
||||
} catch (err) {
|
||||
jsonRes(res, {
|
||||
|
@@ -19,9 +19,7 @@ import {
|
||||
MenuItem
|
||||
} from '@chakra-ui/react';
|
||||
import { getDataList, updateDataName, delData, getDataItems } from '@/api/data';
|
||||
import { usePaging } from '@/hooks/usePaging';
|
||||
import type { DataListItem } from '@/types/data';
|
||||
import ScrollData from '@/components/ScrollData';
|
||||
import dayjs from 'dayjs';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useRouter } from 'next/router';
|
||||
@@ -30,6 +28,7 @@ import { useRequest } from '@/hooks/useRequest';
|
||||
import { DataItemSchema } from '@/types/mongoSchema';
|
||||
import { DataTypeTextMap } from '@/constants/data';
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
const nanoid = customAlphabet('.,', 1);
|
||||
|
||||
const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
|
||||
@@ -39,17 +38,6 @@ export type ExportDataType = 'jsonl' | 'txt';
|
||||
|
||||
const DataList = () => {
|
||||
const router = useRouter();
|
||||
const {
|
||||
nextPage,
|
||||
isLoadAll,
|
||||
requesting,
|
||||
data: dataList,
|
||||
getData,
|
||||
initRequesting
|
||||
} = usePaging<DataListItem>({
|
||||
api: getDataList,
|
||||
pageSize: 20
|
||||
});
|
||||
const [ImportDataId, setImportDataId] = useState<string>();
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: '删除数据集,将删除里面的所有内容,请确认!'
|
||||
@@ -61,12 +49,16 @@ const DataList = () => {
|
||||
onClose: onCloseCreateDataModal
|
||||
} = useDisclosure();
|
||||
|
||||
const { data: dataList = [], refetch } = useQuery(['getDataList'], getDataList, {
|
||||
refetchInterval: 10000
|
||||
});
|
||||
|
||||
const { mutate: handleDelData, isLoading: isDeleting } = useRequest({
|
||||
mutationFn: (dataId: string) => delData(dataId),
|
||||
successToast: '删除数据集成功',
|
||||
errorToast: '删除数据集异常',
|
||||
onSuccess() {
|
||||
getData(1, true);
|
||||
refetch();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -131,111 +123,109 @@ const DataList = () => {
|
||||
</Flex>
|
||||
</Card>
|
||||
{/* 数据表 */}
|
||||
<Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
|
||||
<ScrollData
|
||||
h={'100%'}
|
||||
nextPage={nextPage}
|
||||
isLoadAll={isLoadAll}
|
||||
requesting={requesting}
|
||||
initRequesting={initRequesting}
|
||||
>
|
||||
<TableContainer>
|
||||
<Table>
|
||||
<Thead>
|
||||
<Tr>
|
||||
<Th>集合名</Th>
|
||||
<Th>类型</Th>
|
||||
<Th>创建时间</Th>
|
||||
<Th>训练中 / 总数据</Th>
|
||||
<Th></Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{dataList.map((item, i) => (
|
||||
<Tr key={item._id}>
|
||||
<Td>
|
||||
<Input
|
||||
minW={'150px'}
|
||||
placeholder="请输入数据集名称"
|
||||
defaultValue={item.name}
|
||||
size={'sm'}
|
||||
onBlur={(e) => {
|
||||
if (!e.target.value || e.target.value === item.name) return;
|
||||
updateDataName(item._id, e.target.value);
|
||||
}}
|
||||
/>
|
||||
</Td>
|
||||
<Td>{DataTypeTextMap[item.type || 'QA']}</Td>
|
||||
<Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
|
||||
<Td>
|
||||
{item.trainingData} / {item.totalData}
|
||||
</Td>
|
||||
<Td>
|
||||
<Button
|
||||
size={'sm'}
|
||||
variant={'outline'}
|
||||
colorScheme={'gray'}
|
||||
mr={2}
|
||||
onClick={() =>
|
||||
router.push(`/data/detail?dataId=${item._id}&dataName=${item.name}`)
|
||||
}
|
||||
>
|
||||
详细
|
||||
</Button>
|
||||
<Button
|
||||
size={'sm'}
|
||||
variant={'outline'}
|
||||
mr={2}
|
||||
onClick={() => setImportDataId(item._id)}
|
||||
>
|
||||
导入
|
||||
</Button>
|
||||
<Menu>
|
||||
<MenuButton as={Button} mr={2} size={'sm'} isLoading={isExporting}>
|
||||
导出
|
||||
</MenuButton>
|
||||
<MenuList>
|
||||
{item.type === 'QA' && (
|
||||
<MenuItem
|
||||
onClick={() => handleExportData({ data: item, type: 'jsonl' })}
|
||||
>
|
||||
jsonl
|
||||
</MenuItem>
|
||||
)}
|
||||
{item.type === 'abstract' && (
|
||||
<MenuItem onClick={() => handleExportData({ data: item, type: 'txt' })}>
|
||||
txt
|
||||
</MenuItem>
|
||||
)}
|
||||
</MenuList>
|
||||
</Menu>
|
||||
<TableContainer
|
||||
mt={3}
|
||||
flex={'1 0 0'}
|
||||
h={['auto', '0']}
|
||||
overflowY={'auto'}
|
||||
px={6}
|
||||
py={4}
|
||||
backgroundColor={'white'}
|
||||
borderRadius={'md'}
|
||||
boxShadow={'base'}
|
||||
>
|
||||
<Table>
|
||||
<Thead>
|
||||
<Tr>
|
||||
<Th>集合名</Th>
|
||||
<Th>类型</Th>
|
||||
<Th>创建时间</Th>
|
||||
<Th>训练中 / 总数据</Th>
|
||||
<Th></Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{dataList.map((item, i) => (
|
||||
<Tr key={item._id}>
|
||||
<Td>
|
||||
<Input
|
||||
minW={'150px'}
|
||||
placeholder="请输入数据集名称"
|
||||
defaultValue={item.name}
|
||||
size={'sm'}
|
||||
onBlur={(e) => {
|
||||
if (!e.target.value || e.target.value === item.name) return;
|
||||
updateDataName(item._id, e.target.value);
|
||||
}}
|
||||
/>
|
||||
</Td>
|
||||
<Td>{DataTypeTextMap[item.type || 'QA']}</Td>
|
||||
<Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
|
||||
<Td>
|
||||
{item.trainingData} / {item.totalData}
|
||||
</Td>
|
||||
<Td>
|
||||
<Button
|
||||
size={'sm'}
|
||||
variant={'outline'}
|
||||
colorScheme={'gray'}
|
||||
mr={2}
|
||||
onClick={() =>
|
||||
router.push(`/data/detail?dataId=${item._id}&dataName=${item.name}`)
|
||||
}
|
||||
>
|
||||
详细
|
||||
</Button>
|
||||
<Button
|
||||
size={'sm'}
|
||||
variant={'outline'}
|
||||
mr={2}
|
||||
onClick={() => setImportDataId(item._id)}
|
||||
>
|
||||
导入
|
||||
</Button>
|
||||
<Menu>
|
||||
<MenuButton as={Button} mr={2} size={'sm'} isLoading={isExporting}>
|
||||
导出
|
||||
</MenuButton>
|
||||
<MenuList>
|
||||
{item.type === 'QA' && (
|
||||
<MenuItem onClick={() => handleExportData({ data: item, type: 'jsonl' })}>
|
||||
jsonl
|
||||
</MenuItem>
|
||||
)}
|
||||
{item.type === 'abstract' && (
|
||||
<MenuItem onClick={() => handleExportData({ data: item, type: 'txt' })}>
|
||||
txt
|
||||
</MenuItem>
|
||||
)}
|
||||
</MenuList>
|
||||
</Menu>
|
||||
|
||||
<Button
|
||||
size={'sm'}
|
||||
colorScheme={'red'}
|
||||
isLoading={isDeleting}
|
||||
onClick={openConfirm(() => handleDelData(item._id))}
|
||||
>
|
||||
删除
|
||||
</Button>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
</ScrollData>
|
||||
</Card>
|
||||
<Button
|
||||
size={'sm'}
|
||||
colorScheme={'red'}
|
||||
isLoading={isDeleting}
|
||||
onClick={openConfirm(() => handleDelData(item._id))}
|
||||
>
|
||||
删除
|
||||
</Button>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
|
||||
{ImportDataId && (
|
||||
<ImportDataModal
|
||||
dataId={ImportDataId}
|
||||
onClose={() => setImportDataId(undefined)}
|
||||
onSuccess={() => getData(1, true)}
|
||||
onSuccess={refetch}
|
||||
/>
|
||||
)}
|
||||
{isOpenCreateDataModal && (
|
||||
<CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
|
||||
<CreateDataModal onClose={onCloseCreateDataModal} onSuccess={refetch} />
|
||||
)}
|
||||
<ConfirmChild />
|
||||
</Box>
|
||||
|
@@ -7,6 +7,8 @@ import { ChatModelNameEnum } from '@/constants/model';
|
||||
import { pushSplitDataBill } from '@/service/events/pushBill';
|
||||
|
||||
export async function generateAbstract(next = false): Promise<any> {
|
||||
if (process.env.NODE_ENV === 'development') return;
|
||||
|
||||
if (global.generatingAbstract && !next) return;
|
||||
global.generatingAbstract = true;
|
||||
|
||||
|
@@ -7,6 +7,8 @@ import { ChatModelNameEnum } from '@/constants/model';
|
||||
import { pushSplitDataBill } from '@/service/events/pushBill';
|
||||
|
||||
export async function generateQA(next = false): Promise<any> {
|
||||
if (process.env.NODE_ENV === 'development') return;
|
||||
|
||||
if (global.generatingQA && !next) return;
|
||||
global.generatingQA = true;
|
||||
|
||||
|
Reference in New Issue
Block a user