ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-23 21:13:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: model data code

Browse Source
This commit is contained in:
archer
2023-04-24 09:50:56 +08:00
parent e0b1a78344
commit 29c95d24ae
6 changed files with 67 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
next.config.js
View File

@@ -1,11 +1,8 @@
/** @type {import('next').NextConfig} */
const path = require('path');
const isDev = process.env.NODE_ENV === 'development';
const nextConfig = {
output: 'standalone',
reactStrictMode: false,
reactStrictMode: true,
compress: true,
webpack(config) {
config.module.rules = config.module.rules.concat([

2
src/api/model.ts
View File

@@ -1,5 +1,5 @@
import { GET, POST, DELETE, PUT } from './request';
import type { ModelSchema, ModelDataSchema, ModelSplitDataSchema } from '@/types/mongoSchema';
import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema';
import { ModelUpdateParams } from '@/types/model';
import { TrainingItemType } from '../types/training';
import { RequestPaging } from '../types/index';

4
src/constants/model.ts
View File

@@ -1,4 +1,4 @@
import type { ModelDataType, ModelSchema } from '@/types/mongoSchema';
import type { ModelSchema } from '@/types/mongoSchema';
export enum ModelDataStatusEnum {
ready = 'ready',
@@ -120,7 +120,7 @@ export const ModelVectorSearchModeMap: Record<
export const defaultModel: ModelSchema = {
_id: '',
userId: '',
name: '',
name: 'modelName',
avatar: '',
status: ModelStatusEnum.pending,
updateTime: Date.now(),

15
src/pages/_error.tsx Normal file
View File

@@ -0,0 +1,15 @@
function Error({ statusCode }: { statusCode: number }) {
return (
<p>
{statusCode ? `An error ${statusCode} occurred on server` : 'An error occurred on client'}
</p>
);
}
Error.getInitialProps = ({ res, err }: { res: any; err: any }) => {
const statusCode = res ? res.statusCode : err ? err.statusCode : 404;
console.log(err);
return { statusCode };
};
export default Error;

98
src/pages/model/detail/components/ModelDataCard.tsx
View File

@@ -1,4 +1,4 @@
import React, { useCallback, useState } from 'react';
import React, { useCallback, useState, useRef } from 'react';
import {
Box,
TableContainer,
@@ -19,7 +19,6 @@ import {
Input
} from '@chakra-ui/react';
import type { BoxProps } from '@chakra-ui/react';
import type { ModelSchema } from '@/types/mongoSchema';
import type { ModelDataItemType } from '@/types/model';
import { ModelDataStatusMap } from '@/constants/model';
import { usePagination } from '@/hooks/usePagination';
@@ -34,19 +33,23 @@ import { useLoading } from '@/hooks/useLoading';
import { fileDownload } from '@/utils/file';
import dynamic from 'next/dynamic';
import { useMutation, useQuery } from '@tanstack/react-query';
import type { FormData as InputDataType } from './InputDataModal';
import Papa from 'papaparse';
import InputModal, { FormData as InputDataType } from './InputDataModal';
const InputModel = dynamic(() => import('./InputDataModal'));
const SelectFileModel = dynamic(() => import('./SelectFileModal'));
const SelectUrlModel = dynamic(() => import('./SelectUrlModal'));
const SelectFileModal = dynamic(() => import('./SelectFileModal'));
const SelectCsvModal = dynamic(() => import('./SelectCsvModal'));
let lastSearch = '';
const ModelDataCard = ({ model }: { model: ModelSchema }) => {
const ModelDataCard = ({ modelId }: { modelId: string }) => {
const { Loading, setIsLoading } = useLoading();
const lastSearch = useRef('');
const [searchText, setSearchText] = useState('');
const tdStyles = useRef<BoxProps>({
fontSize: 'xs',
maxW: '500px',
whiteSpace: 'pre-wrap',
maxH: '250px',
overflowY: 'auto'
});
const {
data: modelDataList,
isLoading,
@@ -58,7 +61,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
api: getModelDataList,
pageSize: 10,
params: {
modelId: model._id,
modelId,
searchText
}
});
@@ -70,19 +73,20 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
onOpen: onOpenSelectFileModal,
onClose: onCloseSelectFileModal
} = useDisclosure();
const {
isOpen: isOpenSelectUrlModal,
onOpen: onOpenSelectUrlModal,
onClose: onCloseSelectUrlModal
} = useDisclosure();
const {
isOpen: isOpenSelectCsvModal,
onOpen: onOpenSelectCsvModal,
onClose: onCloseSelectCsvModal
} = useDisclosure();
const { data: splitDataLen, refetch } = useQuery(['getModelSplitDataList'], () =>
getModelSplitDataListLen(model._id)
const { data: splitDataLen = 0, refetch } = useQuery(
['getModelSplitDataList'],
() => getModelSplitDataListLen(modelId),
{
onError(err) {
console.log(err);
}
}
);
const refetchData = useCallback(
@@ -94,8 +98,8 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
);
// 获取所有的数据，并导出 json
const { mutate: onclickExport, isLoading: isLoadingExport } = useMutation({
mutationFn: () => getExportDataList(model._id),
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({
mutationFn: () => getExportDataList(modelId),
onSuccess(res) {
try {
setIsLoading(true);
@@ -112,17 +116,12 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
error;
}
setIsLoading(false);
},
onError(err) {
console.log(err);
}
});
const tdStyles: BoxProps = {
fontSize: 'xs',
maxW: '500px',
whiteSpace: 'pre-wrap',
maxH: '250px',
overflowY: 'auto'
};
return (
<>
<Flex>
@@ -150,7 +149,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
>
导出
</Button>
<Menu>
<Menu autoSelect={false}>
<MenuButton as={Button} size={'sm'}>
导入
</MenuButton>
@@ -166,17 +165,13 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
手动输入
</MenuItem>
<MenuItem onClick={onOpenSelectFileModal}>文本/文件拆分</MenuItem>
{/* <MenuItem onClick={onOpenSelectUrlModal}>网站内容拆分</MenuItem> */}
<MenuItem onClick={onOpenSelectCsvModal}>csv 问答对导入</MenuItem>
</MenuList>
</Menu>
</Flex>
<Flex mt={4}>
{/* 拆分数据提示 */}
{!!(splitDataLen && splitDataLen > 0) && (
<Box fontSize={'xs'}>{splitDataLen}条数据正在拆分...</Box>
)}
<Box flex={1}></Box>
{splitDataLen > 0 && <Box fontSize={'xs'}>{splitDataLen}条数据正在拆分...</Box>}
<Box flex={1} />
<Input
maxW={'240px'}
size={'sm'}
@@ -184,15 +179,15 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
placeholder="搜索相关问题和答案，回车确认"
onChange={(e) => setSearchText(e.target.value)}
onBlur={() => {
if (searchText === lastSearch) return;
if (searchText === lastSearch.current) return;
getData(1);
lastSearch = searchText;
lastSearch.current = searchText;
}}
onKeyDown={(e) => {
if (searchText === lastSearch) return;
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
getData(1);
lastSearch = searchText;
lastSearch.current = searchText;
}
}}
/>
@@ -203,7 +198,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
<Table variant={'simple'} w={'100%'}>
<Thead>
<Tr>
<Th>匹配内容(问题)</Th>
<Th>{'匹配内容(问题)'}</Th>
<Th>对应答案</Th>
<Th>状态</Th>
<Th>操作</Th>
@@ -213,10 +208,10 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
{modelDataList.map((item) => (
<Tr key={item.id}>
<Td>
<Box {...tdStyles}>{item.q}</Box>
<Box {...tdStyles.current}>{item.q}</Box>
</Td>
<Td>
<Box {...tdStyles}>{item.a || '-'}</Box>
<Box {...tdStyles.current}>{item.a || '-'}</Box>
</Td>
<Td>{ModelDataStatusMap[item.status]}</Td>
<Td>
@@ -258,33 +253,22 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
<Loading loading={isLoading} fixed={false} />
{editInputData !== undefined && (
<InputModel
modelId={model._id}
<InputModal
modelId={modelId}
defaultValues={editInputData}
onClose={() => setEditInputData(undefined)}
onSuccess={refetchData}
/>
)}
{isOpenSelectFileModal && (
<SelectFileModel
modelId={model._id}
<SelectFileModal
modelId={modelId}
onClose={onCloseSelectFileModal}
onSuccess={refetchData}
/>
)}
{isOpenSelectUrlModal && (
<SelectUrlModel
modelId={model._id}
onClose={onCloseSelectUrlModal}
onSuccess={refetchData}
/>
)}
{isOpenSelectCsvModal && (
<SelectCsvModal
modelId={model._id}
onClose={onCloseSelectCsvModal}
onSuccess={refetchData}
/>
<SelectCsvModal modelId={modelId} onClose={onCloseSelectCsvModal} onSuccess={refetchData} />
)}
</>
);

53
src/pages/model/detail/index.tsx
View File

@@ -1,4 +1,4 @@
import React, { useCallback, useState, useRef, useMemo, useEffect } from 'react';
import React, { useCallback, useState, useMemo, useEffect } from 'react';
import { useRouter } from 'next/router';
import { getModelById, delModelById, putModelTrainingStatus, putModelById } from '@/api/model';
import type { ModelSchema } from '@/types/mongoSchema';
@@ -8,16 +8,16 @@ import { useForm } from 'react-hook-form';
import { formatModelStatus, ModelStatusEnum, modelList, defaultModel } from '@/constants/model';
import { useGlobalStore } from '@/store/global';
import { useScreen } from '@/hooks/useScreen';
import ModelEditForm from './components/ModelEditForm';
import { useQuery } from '@tanstack/react-query';
import dynamic from 'next/dynamic';
const ModelEditForm = dynamic(() => import('./components/ModelEditForm'));
const ModelDataCard = dynamic(() => import('./components/ModelDataCard'));
const ModelDetail = ({ modelId }: { modelId: string }) => {
const { toast } = useToast();
const router = useRouter();
const { isPc, media } = useScreen();
const { isPc } = useScreen();
const { setLoading } = useGlobalStore();
const [model, setModel] = useState<ModelSchema>(defaultModel);
@@ -76,36 +76,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
setLoading(false);
}, [setLoading, router, modelId]);
/* 上传数据集,触发微调 */
// const startTraining = useCallback(
// async (e: React.ChangeEvent<HTMLInputElement>) => {
// if (!modelId || !e.target.files || e.target.files?.length === 0) return;
// setLoading(true);
// try {
// const file = e.target.files[0];
// const formData = new FormData();
// formData.append('file', file);
// await postTrainModel(modelId, formData);
// toast({
// title: '开始训练...',
// status: 'success'
// });
// // 重新获取模型
// loadModel();
// } catch (err: any) {
// toast({
// title: err?.message || '上传文件失败',
// status: 'error'
// });
// console.log('error->', err);
// }
// setLoading(false);
// },
// [setLoading, loadModel, modelId, toast]
// );
/* 点击更新模型状态 */
const handleClickUpdateStatus = useCallback(async () => {
if (!model || model.status !== ModelStatusEnum.training) return;
@@ -236,21 +206,12 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
</>
)}
</Card>
<Grid mt={5} gridTemplateColumns={media('1fr 1fr', '1fr')} gridGap={5}>
<Grid mt={5} gridTemplateColumns={['1fr', '1fr 1fr']} gridGap={5}>
<ModelEditForm formHooks={formHooks} handleDelModel={handleDelModel} canTrain={canTrain} />
{canTrain && model._id && (
<Card
p={4}
{...media(
{
gridColumnStart: 1,
gridColumnEnd: 3
},
{}
)}
>
<ModelDataCard model={model} />
{canTrain && !!model._id && (
<Card p={4} gridColumnStart={[1, 1]} gridColumnEnd={[2, 3]}>
<ModelDataCard modelId={model._id} />
</Card>
)}
</Grid>