feat: 一次性获取data集合

src/api/data.ts

@@ -6,8 +6,7 @@ import type { PagingData } from '../types/index';
 import type { DataItemSchema } from '@/types/mongoSchema';
 import type { CreateDataProps } from '@/pages/data/components/CreateDataModal';
 
-export const getDataList = (data: RequestPaging) =>
-  GET<PagingData<DataListItem>>(`/data/getDataList?${Obj2Query(data)}`);
+export const getDataList = () => GET<DataListItem[]>(`/data/getDataList`);
 
 export const postData = (data: CreateDataProps) => POST<string>(`/data/postData`, data);
 

src/pages/api/data/getDataList.ts

@@ -3,18 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Data, DataItem } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
-import type { DataSchema } from '@/types/mongoSchema';
 import type { DataListItem } from '@/types/data';
-import type { PagingData } from '@/types';
 import mongoose from 'mongoose';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     const { authorization } = req.headers;
-    let { pageNum = 1, pageSize = 10 } = req.query as { pageNum: string; pageSize: string };
-
-    pageNum = +pageNum;
-    pageSize = +pageSize;
 
     if (!authorization) {
       throw new Error('缺少登录凭证');
@@ -34,12 +28,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       {
         $sort: { createTime: -1 } // 按照创建时间倒序排列
       },
-      {
-        $skip: (pageNum - 1) * pageSize // 跳过前面的数据
-      },
-      {
-        $limit: pageSize // 取出指定数量的数据
-      },
       {
         $lookup: {
           from: 'dataitems',
@@ -71,13 +59,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       }
     ]);
 
-    jsonRes<PagingData<DataListItem>>(res, {
-      data: {
-        pageNum,
-        pageSize,
-        data: datalist,
-        total: 1
-      }
+    jsonRes(res, {
+      data: datalist
     });
   } catch (err) {
     jsonRes(res, {

src/pages/data/list.tsx

@@ -19,9 +19,7 @@ import {
   MenuItem
 } from '@chakra-ui/react';
 import { getDataList, updateDataName, delData, getDataItems } from '@/api/data';
-import { usePaging } from '@/hooks/usePaging';
 import type { DataListItem } from '@/types/data';
-import ScrollData from '@/components/ScrollData';
 import dayjs from 'dayjs';
 import dynamic from 'next/dynamic';
 import { useRouter } from 'next/router';
@@ -30,6 +28,7 @@ import { useRequest } from '@/hooks/useRequest';
 import { DataItemSchema } from '@/types/mongoSchema';
 import { DataTypeTextMap } from '@/constants/data';
 import { customAlphabet } from 'nanoid';
+import { useQuery } from '@tanstack/react-query';
 const nanoid = customAlphabet('.,', 1);
 
 const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
@@ -39,17 +38,6 @@ export type ExportDataType = 'jsonl' | 'txt';
 
 const DataList = () => {
   const router = useRouter();
-  const {
-    nextPage,
-    isLoadAll,
-    requesting,
-    data: dataList,
-    getData,
-    initRequesting
-  } = usePaging<DataListItem>({
-    api: getDataList,
-    pageSize: 20
-  });
   const [ImportDataId, setImportDataId] = useState<string>();
   const { openConfirm, ConfirmChild } = useConfirm({
     content: '删除数据集，将删除里面的所有内容，请确认！'
@@ -61,12 +49,16 @@ const DataList = () => {
     onClose: onCloseCreateDataModal
   } = useDisclosure();
 
+  const { data: dataList = [], refetch } = useQuery(['getDataList'], getDataList, {
+    refetchInterval: 10000
+  });
+
   const { mutate: handleDelData, isLoading: isDeleting } = useRequest({
     mutationFn: (dataId: string) => delData(dataId),
     successToast: '删除数据集成功',
     errorToast: '删除数据集异常',
     onSuccess() {
-      getData(1, true);
+      refetch();
     }
   });
 
@@ -131,15 +123,17 @@ const DataList = () => {
         </Flex>
       </Card>
       {/* 数据表 */}
-      <Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
-        <ScrollData
-          h={'100%'}
-          nextPage={nextPage}
-          isLoadAll={isLoadAll}
-          requesting={requesting}
-          initRequesting={initRequesting}
+      <TableContainer
+        mt={3}
+        flex={'1 0 0'}
+        h={['auto', '0']}
+        overflowY={'auto'}
+        px={6}
+        py={4}
+        backgroundColor={'white'}
+        borderRadius={'md'}
+        boxShadow={'base'}
       >
-          <TableContainer>
         <Table>
           <Thead>
             <Tr>
@@ -196,9 +190,7 @@ const DataList = () => {
                     </MenuButton>
                     <MenuList>
                       {item.type === 'QA' && (
-                            <MenuItem
-                              onClick={() => handleExportData({ data: item, type: 'jsonl' })}
-                            >
+                        <MenuItem onClick={() => handleExportData({ data: item, type: 'jsonl' })}>
                           jsonl
                         </MenuItem>
                       )}
@@ -224,18 +216,16 @@ const DataList = () => {
           </Tbody>
         </Table>
       </TableContainer>
-        </ScrollData>
-      </Card>
 
       {ImportDataId && (
         <ImportDataModal
           dataId={ImportDataId}
           onClose={() => setImportDataId(undefined)}
-          onSuccess={() => getData(1, true)}
+          onSuccess={refetch}
         />
       )}
       {isOpenCreateDataModal && (
-        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
+        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={refetch} />
       )}
       <ConfirmChild />
     </Box>

src/service/events/generateAbstract.ts

@@ -7,6 +7,8 @@ import { ChatModelNameEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
 
 export async function generateAbstract(next = false): Promise<any> {
+  if (process.env.NODE_ENV === 'development') return;
+
   if (global.generatingAbstract && !next) return;
   global.generatingAbstract = true;
 

src/service/events/generateQA.ts

@@ -7,6 +7,8 @@ import { ChatModelNameEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
 
 export async function generateQA(next = false): Promise<any> {
+  if (process.env.NODE_ENV === 'development') return;
+
   if (global.generatingQA && !next) return;
   global.generatingQA = true;