feat: 一次性获取data集合

src/api/data.ts

@@ -6,8 +6,7 @@ import type { PagingData } from '../types/index';
 import type { DataItemSchema } from '@/types/mongoSchema';
 import type { CreateDataProps } from '@/pages/data/components/CreateDataModal';
 
-export const getDataList = (data: RequestPaging) =>
+export const getDataList = () => GET<DataListItem[]>(`/data/getDataList`);
-  GET<PagingData<DataListItem>>(`/data/getDataList?${Obj2Query(data)}`);
 
 export const postData = (data: CreateDataProps) => POST<string>(`/data/postData`, data);
 

src/pages/api/data/getDataList.ts

@@ -3,18 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Data, DataItem } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
-import type { DataSchema } from '@/types/mongoSchema';
 import type { DataListItem } from '@/types/data';
-import type { PagingData } from '@/types';
 import mongoose from 'mongoose';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     const { authorization } = req.headers;
-    let { pageNum = 1, pageSize = 10 } = req.query as { pageNum: string; pageSize: string };
-
-    pageNum = +pageNum;
-    pageSize = +pageSize;
 
     if (!authorization) {
       throw new Error('缺少登录凭证');
@@ -34,12 +28,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       {
         $sort: { createTime: -1 } // 按照创建时间倒序排列
       },
-      {
-        $skip: (pageNum - 1) * pageSize // 跳过前面的数据
-      },
-      {
-        $limit: pageSize // 取出指定数量的数据
-      },
       {
         $lookup: {
           from: 'dataitems',
@@ -71,13 +59,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       }
     ]);
 
-    jsonRes<PagingData<DataListItem>>(res, {
+    jsonRes(res, {
-      data: {
+      data: datalist
-        pageNum,
-        pageSize,
-        data: datalist,
-        total: 1
-      }
     });
   } catch (err) {
     jsonRes(res, {

src/pages/data/list.tsx

@@ -19,9 +19,7 @@ import {
   MenuItem
 } from '@chakra-ui/react';
 import { getDataList, updateDataName, delData, getDataItems } from '@/api/data';
-import { usePaging } from '@/hooks/usePaging';
 import type { DataListItem } from '@/types/data';
-import ScrollData from '@/components/ScrollData';
 import dayjs from 'dayjs';
 import dynamic from 'next/dynamic';
 import { useRouter } from 'next/router';
@@ -30,6 +28,7 @@ import { useRequest } from '@/hooks/useRequest';
 import { DataItemSchema } from '@/types/mongoSchema';
 import { DataTypeTextMap } from '@/constants/data';
 import { customAlphabet } from 'nanoid';
+import { useQuery } from '@tanstack/react-query';
 const nanoid = customAlphabet('.,', 1);
 
 const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
@@ -39,17 +38,6 @@ export type ExportDataType = 'jsonl' | 'txt';
 
 const DataList = () => {
   const router = useRouter();
-  const {
-    nextPage,
-    isLoadAll,
-    requesting,
-    data: dataList,
-    getData,
-    initRequesting
-  } = usePaging<DataListItem>({
-    api: getDataList,
-    pageSize: 20
-  });
   const [ImportDataId, setImportDataId] = useState<string>();
   const { openConfirm, ConfirmChild } = useConfirm({
     content: '删除数据集，将删除里面的所有内容，请确认！'
@@ -61,12 +49,16 @@ const DataList = () => {
     onClose: onCloseCreateDataModal
   } = useDisclosure();
 
+  const { data: dataList = [], refetch } = useQuery(['getDataList'], getDataList, {
+    refetchInterval: 10000
+  });
+
   const { mutate: handleDelData, isLoading: isDeleting } = useRequest({
     mutationFn: (dataId: string) => delData(dataId),
     successToast: '删除数据集成功',
     errorToast: '删除数据集异常',
     onSuccess() {
-      getData(1, true);
+      refetch();
     }
   });
 
@@ -131,111 +123,109 @@ const DataList = () => {
         </Flex>
       </Card>
       {/* 数据表 */}
-      <Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
+      <TableContainer
-        <ScrollData
+        mt={3}
-          h={'100%'}
+        flex={'1 0 0'}
-          nextPage={nextPage}
+        h={['auto', '0']}
-          isLoadAll={isLoadAll}
+        overflowY={'auto'}
-          requesting={requesting}
+        px={6}
-          initRequesting={initRequesting}
+        py={4}
-        >
+        backgroundColor={'white'}
-          <TableContainer>
+        borderRadius={'md'}
-            <Table>
+        boxShadow={'base'}
-              <Thead>
+      >
-                <Tr>
+        <Table>
-                  <Th>集合名</Th>
+          <Thead>
-                  <Th>类型</Th>
+            <Tr>
-                  <Th>创建时间</Th>
+              <Th>集合名</Th>
-                  <Th>训练中 / 总数据</Th>
+              <Th>类型</Th>
-                  <Th></Th>
+              <Th>创建时间</Th>
-                </Tr>
+              <Th>训练中 / 总数据</Th>
-              </Thead>
+              <Th></Th>
-              <Tbody>
+            </Tr>
-                {dataList.map((item, i) => (
+          </Thead>
-                  <Tr key={item._id}>
+          <Tbody>
-                    <Td>
+            {dataList.map((item, i) => (
-                      <Input
+              <Tr key={item._id}>
-                        minW={'150px'}
+                <Td>
-                        placeholder="请输入数据集名称"
+                  <Input
-                        defaultValue={item.name}
+                    minW={'150px'}
-                        size={'sm'}
+                    placeholder="请输入数据集名称"
-                        onBlur={(e) => {
+                    defaultValue={item.name}
-                          if (!e.target.value || e.target.value === item.name) return;
+                    size={'sm'}
-                          updateDataName(item._id, e.target.value);
+                    onBlur={(e) => {
-                        }}
+                      if (!e.target.value || e.target.value === item.name) return;
-                      />
+                      updateDataName(item._id, e.target.value);
-                    </Td>
+                    }}
-                    <Td>{DataTypeTextMap[item.type || 'QA']}</Td>
+                  />
-                    <Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
+                </Td>
-                    <Td>
+                <Td>{DataTypeTextMap[item.type || 'QA']}</Td>
-                      {item.trainingData} / {item.totalData}
+                <Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
-                    </Td>
+                <Td>
-                    <Td>
+                  {item.trainingData} / {item.totalData}
-                      <Button
+                </Td>
-                        size={'sm'}
+                <Td>
-                        variant={'outline'}
+                  <Button
-                        colorScheme={'gray'}
+                    size={'sm'}
-                        mr={2}
+                    variant={'outline'}
-                        onClick={() =>
+                    colorScheme={'gray'}
-                          router.push(`/data/detail?dataId=${item._id}&dataName=${item.name}`)
+                    mr={2}
-                        }
+                    onClick={() =>
-                      >
+                      router.push(`/data/detail?dataId=${item._id}&dataName=${item.name}`)
-                        详细
+                    }
-                      </Button>
+                  >
-                      <Button
+                    详细
-                        size={'sm'}
+                  </Button>
-                        variant={'outline'}
+                  <Button
-                        mr={2}
+                    size={'sm'}
-                        onClick={() => setImportDataId(item._id)}
+                    variant={'outline'}
-                      >
+                    mr={2}
-                        导入
+                    onClick={() => setImportDataId(item._id)}
-                      </Button>
+                  >
-                      <Menu>
+                    导入
-                        <MenuButton as={Button} mr={2} size={'sm'} isLoading={isExporting}>
+                  </Button>
-                          导出
+                  <Menu>
-                        </MenuButton>
+                    <MenuButton as={Button} mr={2} size={'sm'} isLoading={isExporting}>
-                        <MenuList>
+                      导出
-                          {item.type === 'QA' && (
+                    </MenuButton>
-                            <MenuItem
+                    <MenuList>
-                              onClick={() => handleExportData({ data: item, type: 'jsonl' })}
+                      {item.type === 'QA' && (
-                            >
+                        <MenuItem onClick={() => handleExportData({ data: item, type: 'jsonl' })}>
-                              jsonl
+                          jsonl
-                            </MenuItem>
+                        </MenuItem>
-                          )}
+                      )}
-                          {item.type === 'abstract' && (
+                      {item.type === 'abstract' && (
-                            <MenuItem onClick={() => handleExportData({ data: item, type: 'txt' })}>
+                        <MenuItem onClick={() => handleExportData({ data: item, type: 'txt' })}>
-                              txt
+                          txt
-                            </MenuItem>
+                        </MenuItem>
-                          )}
+                      )}
-                        </MenuList>
+                    </MenuList>
-                      </Menu>
+                  </Menu>
 
-                      <Button
+                  <Button
-                        size={'sm'}
+                    size={'sm'}
-                        colorScheme={'red'}
+                    colorScheme={'red'}
-                        isLoading={isDeleting}
+                    isLoading={isDeleting}
-                        onClick={openConfirm(() => handleDelData(item._id))}
+                    onClick={openConfirm(() => handleDelData(item._id))}
-                      >
+                  >
-                        删除
+                    删除
-                      </Button>
+                  </Button>
-                    </Td>
+                </Td>
-                  </Tr>
+              </Tr>
-                ))}
+            ))}
-              </Tbody>
+          </Tbody>
-            </Table>
+        </Table>
-          </TableContainer>
+      </TableContainer>
-        </ScrollData>
-      </Card>
 
       {ImportDataId && (
         <ImportDataModal
           dataId={ImportDataId}
           onClose={() => setImportDataId(undefined)}
-          onSuccess={() => getData(1, true)}
+          onSuccess={refetch}
         />
       )}
       {isOpenCreateDataModal && (
-        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
+        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={refetch} />
       )}
       <ConfirmChild />
     </Box>

src/service/events/generateAbstract.ts

@@ -7,6 +7,8 @@ import { ChatModelNameEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
 
 export async function generateAbstract(next = false): Promise<any> {
+  if (process.env.NODE_ENV === 'development') return;
+
   if (global.generatingAbstract && !next) return;
   global.generatingAbstract = true;
 

src/service/events/generateQA.ts

@@ -7,6 +7,8 @@ import { ChatModelNameEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
 
 export async function generateQA(next = false): Promise<any> {
+  if (process.env.NODE_ENV === 'development') return;
+
   if (global.generatingQA && !next) return;
   global.generatingQA = true;