feat: 拆分测试环境

This commit is contained in:
archer
2023-04-02 23:38:28 +08:00
parent ae4243b522
commit 05b2e9e99c
9 changed files with 177 additions and 12 deletions

View File

@@ -38,6 +38,11 @@ type GetModelDataListProps = RequestPaging & {
export const getModelDataList = (props: GetModelDataListProps) =>
GET(`/model/data/getModelData?${Obj2Query(props)}`);
export const getExportDataList = (modelId: string) =>
GET<{ prompt: string; completion: string; vector: number[] }>(
`/model/data/exportModelData?modelId=${modelId}`
);
export const getModelSplitDataList = (modelId: string) =>
GET<ModelSplitDataSchema[]>(`/model/data/getSplitData?modelId=${modelId}`);

View File

@@ -0,0 +1,61 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import { connectRedis } from '@/service/redis';
import { VecModelDataIdx } from '@/constants/redis';
import { BufferToVector } from '@/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { modelId } = req.query as {
modelId: string;
};
const { authorization } = req.headers;
if (!authorization) {
throw new Error('无权操作');
}
if (!modelId) {
throw new Error('缺少参数');
}
// 凭证校验
const userId = await authToken(authorization);
await connectToDatabase();
const redis = await connectRedis();
// 从 redis 中获取数据
const searchRes = await redis.ft.search(
VecModelDataIdx,
`@modelId:{${modelId}} @userId:{${userId}}`,
{
RETURN: ['q', 'text', 'vector'],
LIMIT: {
from: 0,
size: 10000
}
}
);
const data = searchRes.documents
.filter((item) => item?.value?.vector)
.map((item: any) => ({
prompt: item.value.q,
completion: item.value.text,
vector: BufferToVector(item.value.vector)
}));
jsonRes(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

View File

@@ -71,7 +71,6 @@ const Login = () => {
order={1}
flex={`0 0 ${isPc ? '400px' : '100%'}`}
height={'100%'}
maxH={'450px'}
border="1px"
borderColor="gray.200"
py={5}

View File

@@ -26,13 +26,14 @@ import {
getModelDataList,
delOneModelData,
putModelDataById,
getModelSplitDataList
getModelSplitDataList,
getExportDataList
} from '@/api/model';
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
import { useToast } from '@/hooks/useToast';
import { useLoading } from '@/hooks/useLoading';
import dynamic from 'next/dynamic';
import { useQuery } from '@tanstack/react-query';
import { useMutation, useQuery } from '@tanstack/react-query';
const InputModel = dynamic(() => import('./InputDataModal'));
const SelectFileModel = dynamic(() => import('./SelectFileModal'));
@@ -99,10 +100,29 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
[getData, refetch]
);
// 获取所有的数据,并导出 json
const { mutate: onclickExport, isLoading: isLoadingExport } = useMutation({
mutationFn: () => getExportDataList(model._id),
onSuccess(res) {
// 导出为文件
const blob = new Blob([JSON.stringify(res)], { type: 'application/json;charset=utf-8' });
// 创建下载链接
const downloadLink = document.createElement('a');
downloadLink.href = window.URL.createObjectURL(blob);
downloadLink.download = `data.json`;
// 添加链接到页面并触发下载
document.body.appendChild(downloadLink);
downloadLink.click();
document.body.removeChild(downloadLink);
}
});
return (
<>
<Flex>
<Box fontWeight={'bold'} fontSize={'lg'} flex={1}>
<Box fontWeight={'bold'} fontSize={'lg'} flex={1} mr={2}>
: {total}{' '}
<Box as={'span'} fontSize={'sm'}>
@@ -113,10 +133,22 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
aria-label={'refresh'}
variant={'outline'}
mr={4}
size={'sm'}
onClick={() => refetchData(pageNum)}
/>
{/* <Button
variant={'outline'}
mr={2}
size={'sm'}
isLoading={isLoadingExport}
onClick={() => onclickExport()}
>
导出
</Button> */}
<Menu>
<MenuButton as={Button}></MenuButton>
<MenuButton as={Button} size={'sm'}>
</MenuButton>
<MenuList>
<MenuItem onClick={onOpenInputModal}></MenuItem>
<MenuItem onClick={onOpenSelectFileModal}></MenuItem>

View File

@@ -7,8 +7,6 @@ import { ChatModelNameEnum } from '@/constants/model';
import { pushSplitDataBill } from '@/service/events/pushBill';
export async function generateAbstract(next = false): Promise<any> {
if (process.env.NODE_ENV === 'development') return;
if (global.generatingAbstract && !next) return;
global.generatingAbstract = true;

View File

@@ -119,3 +119,60 @@ export const pushSplitDataBill = async ({
console.log(error);
}
};
export const pushGenerateVectorBill = async ({
isPay,
userId,
text,
type
}: {
isPay: boolean;
userId: string;
text: string;
type: DataType;
}) => {
await connectToDatabase();
let billId;
try {
// 计算 token 数量
const tokens = encode(text);
console.log('text len: ', text.length);
console.log('token len:', tokens.length);
if (isPay) {
try {
// 获取模型单价格, 都是用 gpt35 拆分
const modelItem = modelList.find((item) => item.model === ChatModelNameEnum.GPT35);
const unitPrice = modelItem?.price || 5;
// 计算价格
const price = unitPrice * tokens.length;
console.log(`splitData bill, price: ${formatPrice(price)}`);
// 插入 Bill 记录
const res = await Bill.create({
userId,
type,
modelName: ChatModelNameEnum.GPT35,
textLen: text.length,
tokenLen: tokens.length,
price
});
billId = res._id;
// 账号扣费
await User.findByIdAndUpdate(userId, {
$inc: { balance: -price }
});
} catch (error) {
console.log('创建账单失败:', error);
billId && Bill.findByIdAndDelete(billId);
}
}
} catch (error) {
console.log(error);
}
};

View File

@@ -17,7 +17,7 @@ export async function connectToDatabase(): Promise<void> {
mongoose.set('strictQuery', true);
global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
bufferCommands: true,
dbName: 'doc_gpt',
dbName: process.env.NODE_ENV === 'development' ? 'doc_gpt_test' : 'doc_gpt',
maxPoolSize: 5,
minPoolSize: 1,
maxConnecting: 5

View File

@@ -30,7 +30,7 @@ export const connectRedis = async () => {
await global.redisClient.connect();
// 1 - 测试库0 - 正式
await global.redisClient.select(process.env.NODE_ENV === 'development' ? 0 : 0);
await global.redisClient.SELECT(0);
return global.redisClient;
} catch (error) {

View File

@@ -123,13 +123,26 @@ export const readDocContent = (file: File) =>
});
export const vectorToBuffer = (vector: number[]) => {
let npVector = new Float32Array(vector);
const npVector = new Float32Array(vector);
return Buffer.from(npVector.buffer);
const buffer = Buffer.from(npVector.buffer);
return buffer;
};
export const BufferToVector = (bufferStr: string) => {
let buffer = Buffer.from(`bufferStr`, 'binary'); // 将字符串转换成 Buffer 对象
const npVector = new Float32Array(
buffer,
buffer.byteOffset,
buffer.byteLength / Float32Array.BYTES_PER_ELEMENT
);
return Array.from(npVector);
};
export function formatVector(vector: number[]) {
let formattedVector = vector.slice(0, 1536); // 截取前1536个元素
formattedVector = formattedVector.concat(Array(1536 - formattedVector.length).fill(0)); // 在后面添加0
if (vector.length > 1536) {
formattedVector = formattedVector.concat(Array(1536 - formattedVector.length).fill(0)); // 在后面添加0
}
return formattedVector;
}