mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 21:13:50 +00:00
feat: 拆分测试环境
This commit is contained in:
@@ -38,6 +38,11 @@ type GetModelDataListProps = RequestPaging & {
|
|||||||
export const getModelDataList = (props: GetModelDataListProps) =>
|
export const getModelDataList = (props: GetModelDataListProps) =>
|
||||||
GET(`/model/data/getModelData?${Obj2Query(props)}`);
|
GET(`/model/data/getModelData?${Obj2Query(props)}`);
|
||||||
|
|
||||||
|
export const getExportDataList = (modelId: string) =>
|
||||||
|
GET<{ prompt: string; completion: string; vector: number[] }>(
|
||||||
|
`/model/data/exportModelData?modelId=${modelId}`
|
||||||
|
);
|
||||||
|
|
||||||
export const getModelSplitDataList = (modelId: string) =>
|
export const getModelSplitDataList = (modelId: string) =>
|
||||||
GET<ModelSplitDataSchema[]>(`/model/data/getSplitData?modelId=${modelId}`);
|
GET<ModelSplitDataSchema[]>(`/model/data/getSplitData?modelId=${modelId}`);
|
||||||
|
|
||||||
|
61
src/pages/api/model/data/exportModelData.ts
Normal file
61
src/pages/api/model/data/exportModelData.ts
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||||
|
import { jsonRes } from '@/service/response';
|
||||||
|
import { connectToDatabase } from '@/service/mongo';
|
||||||
|
import { authToken } from '@/service/utils/tools';
|
||||||
|
import { connectRedis } from '@/service/redis';
|
||||||
|
import { VecModelDataIdx } from '@/constants/redis';
|
||||||
|
import { BufferToVector } from '@/utils/tools';
|
||||||
|
|
||||||
|
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||||
|
try {
|
||||||
|
let { modelId } = req.query as {
|
||||||
|
modelId: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const { authorization } = req.headers;
|
||||||
|
|
||||||
|
if (!authorization) {
|
||||||
|
throw new Error('无权操作');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!modelId) {
|
||||||
|
throw new Error('缺少参数');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 凭证校验
|
||||||
|
const userId = await authToken(authorization);
|
||||||
|
|
||||||
|
await connectToDatabase();
|
||||||
|
const redis = await connectRedis();
|
||||||
|
|
||||||
|
// 从 redis 中获取数据
|
||||||
|
const searchRes = await redis.ft.search(
|
||||||
|
VecModelDataIdx,
|
||||||
|
`@modelId:{${modelId}} @userId:{${userId}}`,
|
||||||
|
{
|
||||||
|
RETURN: ['q', 'text', 'vector'],
|
||||||
|
LIMIT: {
|
||||||
|
from: 0,
|
||||||
|
size: 10000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
const data = searchRes.documents
|
||||||
|
.filter((item) => item?.value?.vector)
|
||||||
|
.map((item: any) => ({
|
||||||
|
prompt: item.value.q,
|
||||||
|
completion: item.value.text,
|
||||||
|
vector: BufferToVector(item.value.vector)
|
||||||
|
}));
|
||||||
|
|
||||||
|
jsonRes(res, {
|
||||||
|
data
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
jsonRes(res, {
|
||||||
|
code: 500,
|
||||||
|
error: err
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
@@ -71,7 +71,6 @@ const Login = () => {
|
|||||||
order={1}
|
order={1}
|
||||||
flex={`0 0 ${isPc ? '400px' : '100%'}`}
|
flex={`0 0 ${isPc ? '400px' : '100%'}`}
|
||||||
height={'100%'}
|
height={'100%'}
|
||||||
maxH={'450px'}
|
|
||||||
border="1px"
|
border="1px"
|
||||||
borderColor="gray.200"
|
borderColor="gray.200"
|
||||||
py={5}
|
py={5}
|
||||||
|
@@ -26,13 +26,14 @@ import {
|
|||||||
getModelDataList,
|
getModelDataList,
|
||||||
delOneModelData,
|
delOneModelData,
|
||||||
putModelDataById,
|
putModelDataById,
|
||||||
getModelSplitDataList
|
getModelSplitDataList,
|
||||||
|
getExportDataList
|
||||||
} from '@/api/model';
|
} from '@/api/model';
|
||||||
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
|
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
|
||||||
import { useToast } from '@/hooks/useToast';
|
import { useToast } from '@/hooks/useToast';
|
||||||
import { useLoading } from '@/hooks/useLoading';
|
import { useLoading } from '@/hooks/useLoading';
|
||||||
import dynamic from 'next/dynamic';
|
import dynamic from 'next/dynamic';
|
||||||
import { useQuery } from '@tanstack/react-query';
|
import { useMutation, useQuery } from '@tanstack/react-query';
|
||||||
|
|
||||||
const InputModel = dynamic(() => import('./InputDataModal'));
|
const InputModel = dynamic(() => import('./InputDataModal'));
|
||||||
const SelectFileModel = dynamic(() => import('./SelectFileModal'));
|
const SelectFileModel = dynamic(() => import('./SelectFileModal'));
|
||||||
@@ -99,10 +100,29 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
|
|||||||
[getData, refetch]
|
[getData, refetch]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// 获取所有的数据,并导出 json
|
||||||
|
const { mutate: onclickExport, isLoading: isLoadingExport } = useMutation({
|
||||||
|
mutationFn: () => getExportDataList(model._id),
|
||||||
|
onSuccess(res) {
|
||||||
|
// 导出为文件
|
||||||
|
const blob = new Blob([JSON.stringify(res)], { type: 'application/json;charset=utf-8' });
|
||||||
|
|
||||||
|
// 创建下载链接
|
||||||
|
const downloadLink = document.createElement('a');
|
||||||
|
downloadLink.href = window.URL.createObjectURL(blob);
|
||||||
|
downloadLink.download = `data.json`;
|
||||||
|
|
||||||
|
// 添加链接到页面并触发下载
|
||||||
|
document.body.appendChild(downloadLink);
|
||||||
|
downloadLink.click();
|
||||||
|
document.body.removeChild(downloadLink);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<Flex>
|
<Flex>
|
||||||
<Box fontWeight={'bold'} fontSize={'lg'} flex={1}>
|
<Box fontWeight={'bold'} fontSize={'lg'} flex={1} mr={2}>
|
||||||
模型数据: {total}组{' '}
|
模型数据: {total}组{' '}
|
||||||
<Box as={'span'} fontSize={'sm'}>
|
<Box as={'span'} fontSize={'sm'}>
|
||||||
(测试版本)
|
(测试版本)
|
||||||
@@ -113,10 +133,22 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
|
|||||||
aria-label={'refresh'}
|
aria-label={'refresh'}
|
||||||
variant={'outline'}
|
variant={'outline'}
|
||||||
mr={4}
|
mr={4}
|
||||||
|
size={'sm'}
|
||||||
onClick={() => refetchData(pageNum)}
|
onClick={() => refetchData(pageNum)}
|
||||||
/>
|
/>
|
||||||
|
{/* <Button
|
||||||
|
variant={'outline'}
|
||||||
|
mr={2}
|
||||||
|
size={'sm'}
|
||||||
|
isLoading={isLoadingExport}
|
||||||
|
onClick={() => onclickExport()}
|
||||||
|
>
|
||||||
|
导出
|
||||||
|
</Button> */}
|
||||||
<Menu>
|
<Menu>
|
||||||
<MenuButton as={Button}>导入</MenuButton>
|
<MenuButton as={Button} size={'sm'}>
|
||||||
|
导入
|
||||||
|
</MenuButton>
|
||||||
<MenuList>
|
<MenuList>
|
||||||
<MenuItem onClick={onOpenInputModal}>手动输入</MenuItem>
|
<MenuItem onClick={onOpenInputModal}>手动输入</MenuItem>
|
||||||
<MenuItem onClick={onOpenSelectFileModal}>文件导入</MenuItem>
|
<MenuItem onClick={onOpenSelectFileModal}>文件导入</MenuItem>
|
||||||
|
@@ -7,8 +7,6 @@ import { ChatModelNameEnum } from '@/constants/model';
|
|||||||
import { pushSplitDataBill } from '@/service/events/pushBill';
|
import { pushSplitDataBill } from '@/service/events/pushBill';
|
||||||
|
|
||||||
export async function generateAbstract(next = false): Promise<any> {
|
export async function generateAbstract(next = false): Promise<any> {
|
||||||
if (process.env.NODE_ENV === 'development') return;
|
|
||||||
|
|
||||||
if (global.generatingAbstract && !next) return;
|
if (global.generatingAbstract && !next) return;
|
||||||
global.generatingAbstract = true;
|
global.generatingAbstract = true;
|
||||||
|
|
||||||
|
@@ -119,3 +119,60 @@ export const pushSplitDataBill = async ({
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const pushGenerateVectorBill = async ({
|
||||||
|
isPay,
|
||||||
|
userId,
|
||||||
|
text,
|
||||||
|
type
|
||||||
|
}: {
|
||||||
|
isPay: boolean;
|
||||||
|
userId: string;
|
||||||
|
text: string;
|
||||||
|
type: DataType;
|
||||||
|
}) => {
|
||||||
|
await connectToDatabase();
|
||||||
|
|
||||||
|
let billId;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// 计算 token 数量
|
||||||
|
const tokens = encode(text);
|
||||||
|
|
||||||
|
console.log('text len: ', text.length);
|
||||||
|
console.log('token len:', tokens.length);
|
||||||
|
|
||||||
|
if (isPay) {
|
||||||
|
try {
|
||||||
|
// 获取模型单价格, 都是用 gpt35 拆分
|
||||||
|
const modelItem = modelList.find((item) => item.model === ChatModelNameEnum.GPT35);
|
||||||
|
const unitPrice = modelItem?.price || 5;
|
||||||
|
// 计算价格
|
||||||
|
const price = unitPrice * tokens.length;
|
||||||
|
|
||||||
|
console.log(`splitData bill, price: ${formatPrice(price)}元`);
|
||||||
|
|
||||||
|
// 插入 Bill 记录
|
||||||
|
const res = await Bill.create({
|
||||||
|
userId,
|
||||||
|
type,
|
||||||
|
modelName: ChatModelNameEnum.GPT35,
|
||||||
|
textLen: text.length,
|
||||||
|
tokenLen: tokens.length,
|
||||||
|
price
|
||||||
|
});
|
||||||
|
billId = res._id;
|
||||||
|
|
||||||
|
// 账号扣费
|
||||||
|
await User.findByIdAndUpdate(userId, {
|
||||||
|
$inc: { balance: -price }
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.log('创建账单失败:', error);
|
||||||
|
billId && Bill.findByIdAndDelete(billId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.log(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
@@ -17,7 +17,7 @@ export async function connectToDatabase(): Promise<void> {
|
|||||||
mongoose.set('strictQuery', true);
|
mongoose.set('strictQuery', true);
|
||||||
global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
|
global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
|
||||||
bufferCommands: true,
|
bufferCommands: true,
|
||||||
dbName: 'doc_gpt',
|
dbName: process.env.NODE_ENV === 'development' ? 'doc_gpt_test' : 'doc_gpt',
|
||||||
maxPoolSize: 5,
|
maxPoolSize: 5,
|
||||||
minPoolSize: 1,
|
minPoolSize: 1,
|
||||||
maxConnecting: 5
|
maxConnecting: 5
|
||||||
|
@@ -30,7 +30,7 @@ export const connectRedis = async () => {
|
|||||||
await global.redisClient.connect();
|
await global.redisClient.connect();
|
||||||
|
|
||||||
// 1 - 测试库,0 - 正式
|
// 1 - 测试库,0 - 正式
|
||||||
await global.redisClient.select(process.env.NODE_ENV === 'development' ? 0 : 0);
|
await global.redisClient.SELECT(0);
|
||||||
|
|
||||||
return global.redisClient;
|
return global.redisClient;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
@@ -123,13 +123,26 @@ export const readDocContent = (file: File) =>
|
|||||||
});
|
});
|
||||||
|
|
||||||
export const vectorToBuffer = (vector: number[]) => {
|
export const vectorToBuffer = (vector: number[]) => {
|
||||||
let npVector = new Float32Array(vector);
|
const npVector = new Float32Array(vector);
|
||||||
|
|
||||||
return Buffer.from(npVector.buffer);
|
const buffer = Buffer.from(npVector.buffer);
|
||||||
|
|
||||||
|
return buffer;
|
||||||
|
};
|
||||||
|
export const BufferToVector = (bufferStr: string) => {
|
||||||
|
let buffer = Buffer.from(`bufferStr`, 'binary'); // 将字符串转换成 Buffer 对象
|
||||||
|
const npVector = new Float32Array(
|
||||||
|
buffer,
|
||||||
|
buffer.byteOffset,
|
||||||
|
buffer.byteLength / Float32Array.BYTES_PER_ELEMENT
|
||||||
|
);
|
||||||
|
return Array.from(npVector);
|
||||||
};
|
};
|
||||||
export function formatVector(vector: number[]) {
|
export function formatVector(vector: number[]) {
|
||||||
let formattedVector = vector.slice(0, 1536); // 截取前1536个元素
|
let formattedVector = vector.slice(0, 1536); // 截取前1536个元素
|
||||||
formattedVector = formattedVector.concat(Array(1536 - formattedVector.length).fill(0)); // 在后面添加0
|
if (vector.length > 1536) {
|
||||||
|
formattedVector = formattedVector.concat(Array(1536 - formattedVector.length).fill(0)); // 在后面添加0
|
||||||
|
}
|
||||||
|
|
||||||
return formattedVector;
|
return formattedVector;
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user