From 46eb96c72ec835480ec20925e92c25ec72e0c916 Mon Sep 17 00:00:00 2001 From: archer <545436317@qq.com> Date: Wed, 26 Apr 2023 21:59:22 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E5=8E=BB=E9=99=A4=E5=86=97=E4=BD=99?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 - src/api/model.ts | 15 -- src/api/request.ts | 2 +- src/api/response/chat.d.ts | 1 - src/constants/data.ts | 6 - src/constants/model.ts | 20 +-- src/constants/redis.ts | 6 - src/constants/responseCode.ts | 20 --- src/pages/api/chat/init.ts | 1 - src/pages/api/data/delData.ts | 47 ------ src/pages/api/data/getDataItems.ts | 48 ------ src/pages/api/data/getDataList.ts | 71 -------- src/pages/api/data/postData.ts | 35 ---- src/pages/api/data/putDataName.ts | 37 ----- src/pages/api/data/splitData.ts | 69 -------- src/pages/api/model/create.ts | 1 - src/pages/api/model/data/putModelData.ts | 2 +- src/pages/api/model/update.ts | 4 +- src/pages/chat/index.tsx | 3 +- .../model/detail/components/Training.tsx | 70 -------- src/pages/model/detail/index.tsx | 34 +--- src/service/errorCode.ts | 21 +++ src/service/events/generateAbstract.ts | 156 ------------------ src/service/events/pushBill.ts | 3 +- src/service/models/data.ts | 30 ---- src/service/models/dataItem.ts | 69 -------- src/service/models/model.ts | 14 -- src/service/models/training.ts | 29 ---- src/service/mongo.ts | 3 - src/service/utils/tools.ts | 8 +- src/types/data.d.ts | 8 - src/types/model.d.ts | 1 - src/types/mongoSchema.d.ts | 43 ----- src/types/training.d.ts | 9 - 34 files changed, 36 insertions(+), 851 deletions(-) delete mode 100644 src/constants/data.ts delete mode 100644 src/constants/redis.ts delete mode 100644 src/constants/responseCode.ts delete mode 100644 src/pages/api/data/delData.ts delete mode 100644 src/pages/api/data/getDataItems.ts delete mode 100644 src/pages/api/data/getDataList.ts delete mode 100644 src/pages/api/data/postData.ts delete mode 100644 src/pages/api/data/putDataName.ts delete mode 100644 src/pages/api/data/splitData.ts delete mode 100644 src/pages/model/detail/components/Training.tsx delete mode 100644 src/service/events/generateAbstract.ts delete mode 100644 src/service/models/data.ts delete mode 100644 src/service/models/dataItem.ts delete mode 100644 src/service/models/training.ts delete mode 100644 src/types/data.d.ts delete mode 100644 src/types/training.d.ts diff --git a/.gitignore b/.gitignore index a1ce0b9ff..7aaf627f5 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ yarn-error.log* # typescript *.tsbuildinfo next-env.d.ts -/public/trainData/ /.vscode/ platform.json testApi/ \ No newline at end of file diff --git a/src/api/model.ts b/src/api/model.ts index d00a70a3d..fc4e575e4 100644 --- a/src/api/model.ts +++ b/src/api/model.ts @@ -1,7 +1,6 @@ import { GET, POST, DELETE, PUT } from './request'; import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema'; import { ModelUpdateParams } from '@/types/model'; -import { TrainingItemType } from '../types/training'; import { RequestPaging } from '../types/index'; import { Obj2Query } from '@/utils/tools'; @@ -32,21 +31,7 @@ export const getModelById = (id: string) => GET(`/model/detail?mode export const putModelById = (id: string, data: ModelUpdateParams) => PUT(`/model/update?modelId=${id}`, data); -export const postTrainModel = (id: string, form: FormData) => - POST(`/model/train/train?modelId=${id}`, form, { - headers: { - 'content-type': 'multipart/form-data' - } - }); - -export const putModelTrainingStatus = (id: string) => - PUT(`/model/train/putTrainStatus?modelId=${id}`); - -export const getModelTrainings = (id: string) => - GET(`/model/train/getTrainings?modelId=${id}`); - /* 模型 data */ - type GetModelDataListProps = RequestPaging & { modelId: string; searchText: string; diff --git a/src/api/request.ts b/src/api/request.ts index c1ed57ac6..95059b135 100644 --- a/src/api/request.ts +++ b/src/api/request.ts @@ -1,6 +1,6 @@ import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios'; import { getToken, clearToken } from '@/utils/user'; -import { TOKEN_ERROR_CODE } from '@/constants/responseCode'; +import { TOKEN_ERROR_CODE } from '@/service/errorCode'; interface ConfigType { headers?: { [key: string]: string }; diff --git a/src/api/response/chat.d.ts b/src/api/response/chat.d.ts index 21b5725bd..baa154c55 100644 --- a/src/api/response/chat.d.ts +++ b/src/api/response/chat.d.ts @@ -6,7 +6,6 @@ export type InitChatResponse = { modelId: string; name: string; avatar: string; - intro: string; chatModel: ModelSchema.service.chatModel; // 对话模型名 modelName: ModelSchema.service.modelName; // 底层模型 history: ChatItemType[]; diff --git a/src/constants/data.ts b/src/constants/data.ts deleted file mode 100644 index 7c3e597fb..000000000 --- a/src/constants/data.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { DataType } from '@/types/data'; - -export const DataTypeTextMap: Record = { - QA: '问答拆分', - abstract: '摘要总结' -}; diff --git a/src/constants/model.ts b/src/constants/model.ts index bb56bd622..300630fc4 100644 --- a/src/constants/model.ts +++ b/src/constants/model.ts @@ -1,10 +1,5 @@ import type { ModelSchema } from '@/types/mongoSchema'; -export enum ModelDataStatusEnum { - ready = 'ready', - waiting = 'waiting' -} - export const embeddingModel = 'text-embedding-ada-002'; export enum ChatModelEnum { 'GPT35' = 'gpt-3.5-turbo', @@ -53,13 +48,6 @@ export const modelList: ModelConstantsData[] = [ } ]; -export enum TrainingStatusEnum { - pending = 'pending', - succeed = 'succeed', - errored = 'errored', - canceled = 'canceled' -} - export enum ModelStatusEnum { running = 'running', training = 'training', @@ -86,6 +74,11 @@ export const formatModelStatus = { } }; +export enum ModelDataStatusEnum { + ready = 'ready', + waiting = 'waiting' +} + export const ModelDataStatusMap: Record<`${ModelDataStatusEnum}`, string> = { ready: '训练完成', waiting: '训练中' @@ -126,15 +119,12 @@ export const defaultModel: ModelSchema = { avatar: '', status: ModelStatusEnum.pending, updateTime: Date.now(), - trainingTimes: 0, systemPrompt: '', - intro: '', temperature: 5, search: { mode: ModelVectorSearchModeEnum.hightSimilarity }, service: { - trainId: '', chatModel: ModelNameEnum.GPT35, modelName: ModelNameEnum.GPT35 }, diff --git a/src/constants/redis.ts b/src/constants/redis.ts deleted file mode 100644 index cb045a03c..000000000 --- a/src/constants/redis.ts +++ /dev/null @@ -1,6 +0,0 @@ -export const VecModelDataPrefix = 'model:data'; -export const VecModelDataIdx = `idx:${VecModelDataPrefix}:hash`; -export enum ModelDataStatusEnum { - ready = 'ready', - waiting = 'waiting' -} diff --git a/src/constants/responseCode.ts b/src/constants/responseCode.ts deleted file mode 100644 index 94dba1265..000000000 --- a/src/constants/responseCode.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const ERROR_CODE: { [key: number]: string } = { - 400: '请求失败', - 401: '无权访问', - 403: '紧张访问', - 404: '请求不存在', - 405: '请求方法错误', - 406: '请求的格式错误', - 410: '资源已删除', - 422: '验证错误', - 500: '服务器发生错误', - 502: '网关错误', - 503: '服务器暂时过载或维护', - 504: '网关超时' -}; - -export const TOKEN_ERROR_CODE: { [key: number]: string } = { - 506: '请先登录', - 507: '请重新登录', - 508: '登录已过期' -}; diff --git a/src/pages/api/chat/init.ts b/src/pages/api/chat/init.ts index 32939d896..9472f1851 100644 --- a/src/pages/api/chat/init.ts +++ b/src/pages/api/chat/init.ts @@ -53,7 +53,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) modelId: modelId, name: model.name, avatar: model.avatar, - intro: model.intro, modelName: model.service.modelName, chatModel: model.service.chatModel, history diff --git a/src/pages/api/data/delData.ts b/src/pages/api/data/delData.ts deleted file mode 100644 index 54337a7e3..000000000 --- a/src/pages/api/data/delData.ts +++ /dev/null @@ -1,47 +0,0 @@ -// Next.js API route support: https://nextjs.org/docs/api-routes/introduction -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@/service/response'; -import { connectToDatabase, Data, DataItem } from '@/service/mongo'; -import { authToken } from '@/service/utils/tools'; -import type { DataListItem } from '@/types/data'; -import type { PagingData } from '@/types'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { authorization } = req.headers; - - if (!authorization) { - throw new Error('缺少登录凭证'); - } - - await authToken(authorization); - - const { dataId } = req.query as { dataId: string }; - if (!dataId) { - throw new Error('缺少参数'); - } - - await connectToDatabase(); - - await Data.findByIdAndUpdate(dataId, { - isDeleted: true - }); - - // 改变 dataItem 状态为 0 - await DataItem.updateMany( - { - dataId - }, - { - status: 0 - } - ); - - jsonRes>(res); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } -} diff --git a/src/pages/api/data/getDataItems.ts b/src/pages/api/data/getDataItems.ts deleted file mode 100644 index 83668de03..000000000 --- a/src/pages/api/data/getDataItems.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@/service/response'; -import { connectToDatabase, DataItem } from '@/service/mongo'; -import { authToken } from '@/service/utils/tools'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - let { - dataId, - pageNum = 1, - pageSize = 10 - } = req.query as { dataId: string; pageNum: string; pageSize: string }; - pageNum = +pageNum; - pageSize = +pageSize; - - if (!dataId) { - throw new Error('参数错误'); - } - await connectToDatabase(); - - const { authorization } = req.headers; - - await authToken(authorization); - - const dataItems = await DataItem.find({ - dataId - }) - .sort({ _id: -1 }) // 按照创建时间倒序排列 - .skip((pageNum - 1) * pageSize) - .limit(pageSize); - - jsonRes(res, { - data: { - pageNum, - pageSize, - data: dataItems, - total: await DataItem.countDocuments({ - dataId - }) - } - }); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } -} diff --git a/src/pages/api/data/getDataList.ts b/src/pages/api/data/getDataList.ts deleted file mode 100644 index 9c26f1c22..000000000 --- a/src/pages/api/data/getDataList.ts +++ /dev/null @@ -1,71 +0,0 @@ -// Next.js API route support: https://nextjs.org/docs/api-routes/introduction -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@/service/response'; -import { connectToDatabase, Data, DataItem } from '@/service/mongo'; -import { authToken } from '@/service/utils/tools'; -import type { DataListItem } from '@/types/data'; -import mongoose from 'mongoose'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { authorization } = req.headers; - - if (!authorization) { - throw new Error('缺少登录凭证'); - } - - const userId = await authToken(authorization); - - await connectToDatabase(); - - const datalist = await Data.aggregate([ - { - $match: { - userId: new mongoose.Types.ObjectId(userId), - isDeleted: false - } - }, - { - $sort: { createTime: -1 } // 按照创建时间倒序排列 - }, - { - $lookup: { - from: 'dataitems', - localField: '_id', - foreignField: 'dataId', - as: 'items' - } - }, - { - $addFields: { - totalData: { - $size: '$items' // 统计dataItem的总数 - }, - trainingData: { - $size: { - $filter: { - input: '$items', - as: 'item', - cond: { $ne: ['$$item.status', 0] } // 统计 status 不为0的数量 - } - } - } - } - }, - { - $project: { - items: 0 // 不返回 items 字段 - } - } - ]); - - jsonRes(res, { - data: datalist - }); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } -} diff --git a/src/pages/api/data/postData.ts b/src/pages/api/data/postData.ts deleted file mode 100644 index b412892b9..000000000 --- a/src/pages/api/data/postData.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@/service/response'; -import { connectToDatabase, Data } from '@/service/mongo'; -import { authToken } from '@/service/utils/tools'; -import type { DataType } from '@/types/data'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - let { name, type } = req.body as { name: string; type: DataType }; - if (!name || !type) { - throw new Error('参数错误'); - } - await connectToDatabase(); - - const { authorization } = req.headers; - - const userId = await authToken(authorization); - - // 生成 data 集合 - const data = await Data.create({ - userId, - name, - type - }); - - jsonRes(res, { - data: data._id - }); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } -} diff --git a/src/pages/api/data/putDataName.ts b/src/pages/api/data/putDataName.ts deleted file mode 100644 index d075c1a3a..000000000 --- a/src/pages/api/data/putDataName.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Next.js API route support: https://nextjs.org/docs/api-routes/introduction -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@/service/response'; -import { connectToDatabase, Data } from '@/service/mongo'; -import { authToken } from '@/service/utils/tools'; -import type { DataListItem } from '@/types/data'; -import type { PagingData } from '@/types'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { authorization } = req.headers; - - if (!authorization) { - throw new Error('缺少登录凭证'); - } - - await authToken(authorization); - - const { dataId, name } = req.query as { dataId: string; name: string }; - if (!dataId || !name) { - throw new Error('缺少参数'); - } - - await connectToDatabase(); - - await Data.findByIdAndUpdate(dataId, { - name - }); - - jsonRes>(res); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } -} diff --git a/src/pages/api/data/splitData.ts b/src/pages/api/data/splitData.ts deleted file mode 100644 index 206578368..000000000 --- a/src/pages/api/data/splitData.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@/service/response'; -import { connectToDatabase, DataItem, Data } from '@/service/mongo'; -import { authToken } from '@/service/utils/tools'; -import { generateQA } from '@/service/events/generateQA'; -import { generateAbstract } from '@/service/events/generateAbstract'; -import { countChatTokens } from '@/utils/tools'; - -/* 拆分数据成QA */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - const { text, dataId } = req.body as { text: string; dataId: string }; - if (!text || !dataId) { - throw new Error('参数错误'); - } - await connectToDatabase(); - - const { authorization } = req.headers; - - const userId = await authToken(authorization); - - const DataRecord = await Data.findById(dataId); - - if (!DataRecord) { - throw new Error('找不到数据集'); - } - const replaceText = text.replace(/[\\n]+/g, ' '); - - // 文本拆分成 chunk - let chunks = replaceText.match(/[^!?.。]+[!?.。]/g) || []; - - const dataItems: any[] = []; - let splitText = ''; - - chunks.forEach((chunk) => { - splitText += chunk; - const tokens = countChatTokens({ messages: [{ role: 'system', content: splitText }] }); - if (tokens >= 780) { - dataItems.push({ - userId, - dataId, - type: DataRecord.type, - text: splitText, - status: 1 - }); - splitText = ''; - } - }); - - // 批量插入数据 - await DataItem.insertMany(dataItems); - - try { - generateQA(); - generateAbstract(); - } catch (error) { - error; - } - - jsonRes(res, { - data: { chunks, replaceText } - }); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } -} diff --git a/src/pages/api/model/create.ts b/src/pages/api/model/create.ts index f1a7e25e4..ad2e08035 100644 --- a/src/pages/api/model/create.ts +++ b/src/pages/api/model/create.ts @@ -47,7 +47,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< userId, status: ModelStatusEnum.running, service: { - trainId: '', chatModel: Model2ChatModelMap[modelItem.model], // 聊天时用的模型 modelName: modelItem.model // 最底层的模型，不会变，用于计费等核心操作 } diff --git a/src/pages/api/model/data/putModelData.ts b/src/pages/api/model/data/putModelData.ts index 9e07e650e..37d03e338 100644 --- a/src/pages/api/model/data/putModelData.ts +++ b/src/pages/api/model/data/putModelData.ts @@ -1,7 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@/service/response'; import { authToken } from '@/service/utils/tools'; -import { ModelDataStatusEnum } from '@/constants/redis'; +import { ModelDataStatusEnum } from '@/constants/model'; import { generateVector } from '@/service/events/generateVector'; import { PgClient } from '@/service/pg'; diff --git a/src/pages/api/model/update.ts b/src/pages/api/model/update.ts index 329aa2cb7..4b021d379 100644 --- a/src/pages/api/model/update.ts +++ b/src/pages/api/model/update.ts @@ -8,7 +8,7 @@ import type { ModelUpdateParams } from '@/types/model'; /* 获取我的模型 */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - const { name, search, service, security, systemPrompt, intro, temperature } = + const { name, search, service, security, systemPrompt, temperature } = req.body as ModelUpdateParams; const { modelId } = req.query as { modelId: string }; const { authorization } = req.headers; @@ -35,10 +35,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< { name, systemPrompt, - intro, temperature, search, - // service, security } ); diff --git a/src/pages/chat/index.tsx b/src/pages/chat/index.tsx index 0a2d7a82d..12b4eece9 100644 --- a/src/pages/chat/index.tsx +++ b/src/pages/chat/index.tsx @@ -64,7 +64,6 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => { modelId, name: '', avatar: '', - intro: '', chatModel: '', modelName: '', history: [] @@ -517,7 +516,7 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => { ))} - {chatData.history.length === 0 && } + {chatData.history.length === 0 && } {/* 发送区 */} diff --git a/src/pages/model/detail/components/Training.tsx b/src/pages/model/detail/components/Training.tsx deleted file mode 100644 index 96d7bb1eb..000000000 --- a/src/pages/model/detail/components/Training.tsx +++ /dev/null @@ -1,70 +0,0 @@ -import React, { useEffect, useCallback, useState } from 'react'; -import { Box, TableContainer, Table, Thead, Tbody, Tr, Th, Td } from '@chakra-ui/react'; -import type { ModelSchema } from '@/types/mongoSchema'; -import { getModelTrainings } from '@/api/model'; -import type { TrainingItemType } from '@/types/training'; - -const Training = ({ model }: { model: ModelSchema }) => { - const columns: { - title: string; - key: keyof TrainingItemType; - dataIndex: string; - }[] = [ - { - title: '训练ID', - key: 'tuneId', - dataIndex: 'tuneId' - }, - { - title: '状态', - key: 'status', - dataIndex: 'status' - } - ]; - - const [records, setRecords] = useState([]); - - const loadTrainingRecords = useCallback(async (id: string) => { - try { - const res = await getModelTrainings(id); - setRecords(res); - } catch (error) { - console.log('error->', error); - } - }, []); - - useEffect(() => { - model._id && loadTrainingRecords(model._id); - }, [loadTrainingRecords, model]); - - return ( - <> - - 训练记录: {model.trainingTimes}次 - - - - - - {columns.map((item) => ( - - ))} - - - - {records.map((item) => ( - - {columns.map((col) => ( - // @ts-ignore - - ))} - - ))} - -
{item.title}
{item[col.dataIndex]}
- - - ); -}; - -export default Training; diff --git a/src/pages/model/detail/index.tsx b/src/pages/model/detail/index.tsx index 0e7a18586..bb8ac7b8c 100644 --- a/src/pages/model/detail/index.tsx +++ b/src/pages/model/detail/index.tsx @@ -1,6 +1,6 @@ import React, { useCallback, useState, useMemo, useEffect } from 'react'; import { useRouter } from 'next/router'; -import { getModelById, delModelById, putModelTrainingStatus, putModelById } from '@/api/model'; +import { getModelById, delModelById, putModelById } from '@/api/model'; import type { ModelSchema } from '@/types/mongoSchema'; import { Card, Box, Flex, Button, Tag, Grid } from '@chakra-ui/react'; import { useToast } from '@/hooks/useToast'; @@ -76,29 +76,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { setLoading(false); }, [setLoading, router, modelId]); - /* 点击更新模型状态 */ - const handleClickUpdateStatus = useCallback(async () => { - if (!model || model.status !== ModelStatusEnum.training) return; - setLoading(true); - - try { - const res = await putModelTrainingStatus(model._id); - typeof res === 'string' && - toast({ - title: res, - status: 'info' - }); - loadModel(); - } catch (error: any) { - console.log('error->', error); - toast({ - title: error.message || '更新失败', - status: 'error' - }); - } - setLoading(false); - }, [model, setLoading, loadModel, toast]); - // 提交保存模型修改 const saveSubmitSuccess = useCallback( async (data: ModelSchema) => { @@ -107,7 +84,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { await putModelById(data._id, { name: data.name, systemPrompt: data.systemPrompt, - intro: data.intro, temperature: data.temperature, search: data.search, service: data.service, @@ -168,13 +144,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { {model.name} - + {formatModelStatus[model.status].text} diff --git a/src/service/errorCode.ts b/src/service/errorCode.ts index 790f51b76..6d82d19af 100644 --- a/src/service/errorCode.ts +++ b/src/service/errorCode.ts @@ -1,3 +1,24 @@ +export const ERROR_CODE: { [key: number]: string } = { + 400: '请求失败', + 401: '无权访问', + 403: '紧张访问', + 404: '请求不存在', + 405: '请求方法错误', + 406: '请求的格式错误', + 410: '资源已删除', + 422: '验证错误', + 500: '服务器发生错误', + 502: '网关错误', + 503: '服务器暂时过载或维护', + 504: '网关超时' +}; + +export const TOKEN_ERROR_CODE: { [key: number]: string } = { + 506: '请先登录', + 507: '请重新登录', + 508: '登录已过期' +}; + export const openaiError: Record = { context_length_exceeded: '内容超长了，请重置对话', Unauthorized: 'API-KEY 不合法', diff --git a/src/service/events/generateAbstract.ts b/src/service/events/generateAbstract.ts deleted file mode 100644 index ceb0d4899..000000000 --- a/src/service/events/generateAbstract.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { DataItem } from '@/service/mongo'; -import { getOpenAIApi } from '@/service/utils/auth'; -import { axiosConfig } from '@/service/utils/tools'; -import { getOpenApiKey } from '../utils/openai'; -import type { ChatCompletionRequestMessage } from 'openai'; -import { DataItemSchema } from '@/types/mongoSchema'; -import { ChatModelEnum } from '@/constants/model'; -import { pushSplitDataBill } from '@/service/events/pushBill'; - -export async function generateAbstract(next = false): Promise { - if (process.env.queueTask !== '1') { - fetch(process.env.parentUrl || ''); - return; - } - if (global.generatingAbstract && !next) return; - - global.generatingAbstract = true; - - const systemPrompt: ChatCompletionRequestMessage = { - role: 'system', - content: `总结助手,我会向你发送一段长文本,请从文本中归纳总结5至15条信息,如果是英文,请增加一条中文的总结,并按以下格式输出: A1:\nA2:\nA3:\n` - }; - let dataItem: DataItemSchema | null = null; - - try { - // 找出一个需要生成的 dataItem - dataItem = await DataItem.findOne({ - status: { $ne: 0 }, - times: { $gt: 0 }, - type: 'abstract' - }); - - if (!dataItem) { - console.log('没有需要生成 【摘要】 的数据'); - global.generatingAbstract = false; - return; - } - - // 更新状态为生成中 - await DataItem.findByIdAndUpdate(dataItem._id, { - status: 2 - }); - - // 获取 openapi Key - let userApiKey, systemKey; - try { - const key = await getOpenApiKey(dataItem.userId); - userApiKey = key.userApiKey; - systemKey = key.systemKey; - } catch (error: any) { - if (error?.code === 501) { - // 余额不够了, 把用户所有记录改成闲置 - await DataItem.updateMany({ - userId: dataItem.userId, - status: 0 - }); - } - - throw new Error('获取 openai key 失败'); - } - - console.log('正在生成一组摘要, ID:', dataItem._id); - - const startTime = Date.now(); - - // 获取 openai 请求实例 - const chatAPI = getOpenAIApi(userApiKey || systemKey); - // 请求 chatgpt 获取摘要 - const abstractResponse = await chatAPI.createChatCompletion( - { - model: ChatModelEnum.GPT35, - temperature: 0.8, - n: 1, - messages: [ - systemPrompt, - { - role: 'user', - content: dataItem?.text || '' - } - ] - }, - { - timeout: 180000, - ...axiosConfig - } - ); - - // 提取摘要内容 - const rawContent: string = abstractResponse?.data.choices[0].message?.content || ''; - // 从 content 中提取摘要内容 - const splitContents = splitText(rawContent); - - // 插入数据库，并修改状态 - await DataItem.findByIdAndUpdate(dataItem._id, { - status: 0, - $push: { - rawResponse: rawContent, - result: { - $each: splitContents - } - } - }); - - console.log( - `生成摘要成功，time: ${(Date.now() - startTime) / 1000}s`, - `摘要匹配数量: ${splitContents.length}` - ); - // 计费 - pushSplitDataBill({ - isPay: !userApiKey && splitContents.length > 0, - userId: dataItem.userId, - type: 'abstract', - text: systemPrompt.content + dataItem.text + rawContent, - tokenLen: 0 - }); - } catch (error: any) { - console.log('error: 生成摘要错误', dataItem?._id); - console.log('response:', error); - if (dataItem?._id) { - await DataItem.findByIdAndUpdate(dataItem._id, { - status: dataItem.times > 1 ? 1 : 0, // 还有重试次数则可以继续进行 - $inc: { - // 剩余尝试次数-1 - times: -1 - } - }); - } - } - - generateAbstract(true); -} - -/** - * 检查文本是否按格式返回 - */ -function splitText(text: string) { - const regex = /A\d+:(\s*)(.*?)\s*(?=A\d+:|$)/gs; - const matches = text.matchAll(regex); // 获取所有匹配到的结果 - - const result = []; // 存储最终的结果 - for (const match of matches) { - if (match[2]) { - result.push({ - abstract: match[2] as string - }); - } - } - - if (result.length === 0) { - result.push({ - abstract: text - }); - } - - return result; -} diff --git a/src/service/events/pushBill.ts b/src/service/events/pushBill.ts index de92c144e..968c37268 100644 --- a/src/service/events/pushBill.ts +++ b/src/service/events/pushBill.ts @@ -7,7 +7,6 @@ import { embeddingModel } from '@/constants/model'; import { BillTypeEnum } from '@/constants/user'; -import type { DataType } from '@/types/data'; import { countChatTokens } from '@/utils/tools'; export const pushChatBill = async ({ @@ -81,7 +80,7 @@ export const pushSplitDataBill = async ({ userId: string; tokenLen: number; text: string; - type: DataType; + type: `${BillTypeEnum}`; }) => { await connectToDatabase(); diff --git a/src/service/models/data.ts b/src/service/models/data.ts deleted file mode 100644 index 7e22dda35..000000000 --- a/src/service/models/data.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Schema, model, models, Model } from 'mongoose'; -import { DataSchema as Datatype } from '@/types/mongoSchema'; -import { DataTypeTextMap } from '@/constants/data'; - -const DataSchema = new Schema({ - userId: { - type: Schema.Types.ObjectId, - ref: 'user', - required: true - }, - name: { - type: String, - required: true - }, - createTime: { - type: Date, - default: () => new Date() - }, - type: { - type: String, - required: true, - enum: Object.keys(DataTypeTextMap) - }, - isDeleted: { - type: Boolean, - default: false - } -}); - -export const Data: Model = models['data'] || model('data', DataSchema); diff --git a/src/service/models/dataItem.ts b/src/service/models/dataItem.ts deleted file mode 100644 index 52eaf7e2a..000000000 --- a/src/service/models/dataItem.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { DataItemSchema as DataItemType } from '@/types/mongoSchema'; -import { Schema, model, models, Model } from 'mongoose'; -import { DataTypeTextMap } from '@/constants/data'; - -const DataItemSchema = new Schema({ - userId: { - type: Schema.Types.ObjectId, - ref: 'user', - required: true - }, - dataId: { - type: Schema.Types.ObjectId, - ref: 'data', - required: true - }, - type: { - type: String, - required: true, - enum: Object.keys(DataTypeTextMap) - }, - times: { - // 剩余重试次数 - type: Number, - default: 3 - }, - text: { - // 文本内容 - type: String, - required: true - }, - rawResponse: { - // 原始拆分结果 - type: [String], - default: [] - }, - result: { - type: [ - { - q: { - type: String, - default: '' - }, - a: { - type: String, - default: '' - }, - abstract: { - // 摘要 - type: String, - default: '' - }, - abstractVector: { - // 摘要对应的向量 - type: [Number], - default: [] - } - } - ], - default: [] - }, - status: { - // 0-闲置，1-待生成，2-生成中 - type: Number, - default: 1 - } -}); - -export const DataItem: Model = - models['dataItem'] || model('dataItem', DataItemSchema); diff --git a/src/service/models/model.ts b/src/service/models/model.ts index 53506b494..ebf8506cf 100644 --- a/src/service/models/model.ts +++ b/src/service/models/model.ts @@ -21,11 +21,6 @@ const ModelSchema = new Schema({ type: String, default: '' }, - intro: { - // 模型介绍 - type: String, - default: '' - }, status: { type: String, required: true, @@ -35,10 +30,6 @@ const ModelSchema = new Schema({ type: Date, default: () => new Date() }, - trainingTimes: { - type: Number, - default: 0 - }, temperature: { type: Number, min: 0, @@ -53,11 +44,6 @@ const ModelSchema = new Schema({ } }, service: { - trainId: { - // 训练时需要的 ID， 不能训练的模型没有这个值。 - type: String, - required: false - }, chatModel: { // 聊天时使用的模型 type: String, diff --git a/src/service/models/training.ts b/src/service/models/training.ts deleted file mode 100644 index 5d6147304..000000000 --- a/src/service/models/training.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Schema, model, models, Model } from 'mongoose'; -import { TrainingSchema as TrainingType } from '@/types/mongoSchema'; -const TrainingSChema = new Schema({ - serviceName: { - // 模型厂商名 - type: String, - required: true - }, - tuneId: { - // 微调进程 ID - type: String, - required: true - }, - modelId: { - // 关联模型的 ID - type: Schema.Types.ObjectId, - ref: 'model', - required: true - }, - status: { - // 状态值 - type: String, - required: true, - enum: ['pending', 'succeed', 'errored', 'canceled'] - } -}); - -export const Training: Model = - models['training'] || model('training', TrainingSChema); diff --git a/src/service/mongo.ts b/src/service/mongo.ts index a7f2bef72..ae1de9e99 100644 --- a/src/service/mongo.ts +++ b/src/service/mongo.ts @@ -45,11 +45,8 @@ export * from './models/authCode'; export * from './models/chat'; export * from './models/model'; export * from './models/user'; -export * from './models/training'; export * from './models/bill'; export * from './models/pay'; -export * from './models/data'; -export * from './models/dataItem'; export * from './models/splitData'; export * from './models/openapi'; export * from './models/promotionRecord'; diff --git a/src/service/utils/tools.ts b/src/service/utils/tools.ts index 05cac36e4..bd40b39f1 100644 --- a/src/service/utils/tools.ts +++ b/src/service/utils/tools.ts @@ -87,11 +87,9 @@ export const authOpenApiKey = async (req: NextApiRequest) => { /* openai axios config */ export const axiosConfig = { httpsAgent: global.httpsAgent, - headers: process.env.OPENAI_BASE_URL_AUTH - ? { - auth: process.env.OPENAI_BASE_URL_AUTH - } - : {} + headers: { + auth: process.env.OPENAI_BASE_URL_AUTH || '' + } }; /* delete invalid symbol */ diff --git a/src/types/data.d.ts b/src/types/data.d.ts deleted file mode 100644 index 942a71b2d..000000000 --- a/src/types/data.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type { DataSchema } from './mongoSchema'; - -export type DataType = 'QA' | 'abstract'; - -export interface DataListItem extends DataSchema { - trainingData: number; - totalData: number; -} diff --git a/src/types/model.d.ts b/src/types/model.d.ts index b62bce5c4..f3280a001 100644 --- a/src/types/model.d.ts +++ b/src/types/model.d.ts @@ -3,7 +3,6 @@ import type { ModelSchema } from './mongoSchema'; export interface ModelUpdateParams { name: string; systemPrompt: string; - intro: string; temperature: number; search: ModelSchema['search']; service: ModelSchema['service']; diff --git a/src/types/mongoSchema.d.ts b/src/types/mongoSchema.d.ts index 8ec31d61a..8e7c11f85 100644 --- a/src/types/mongoSchema.d.ts +++ b/src/types/mongoSchema.d.ts @@ -1,7 +1,6 @@ import type { ChatItemType } from './chat'; import { ModelStatusEnum, - TrainingStatusEnum, ModelNameEnum, ModelVectorSearchModeEnum, ChatModelEnum @@ -35,17 +34,14 @@ export interface ModelSchema { name: string; avatar: string; systemPrompt: string; - intro: string; userId: string; status: `${ModelStatusEnum}`; updateTime: number; - trainingTimes: number; temperature: number; search: { mode: `${ModelVectorSearchModeEnum}`; }; service: { - trainId: string; // 训练的模型，训练后就是训练的模型id chatModel: `${ChatModelEnum}`; // 聊天时用的模型，训练后就是训练的模型 modelName: `${ModelNameEnum}`; // 底层模型名称，不会变 }; @@ -81,17 +77,6 @@ export interface ModelSplitDataSchema { textList: string[]; } -export interface TrainingSchema { - _id: string; - tuneId: string; - modelId: string; - status: `${TrainingStatusEnum}`; -} - -export interface TrainingPopulate extends TrainingSchema { - modelId: ModelSchema; -} - export interface ChatSchema { _id: string; userId: string; @@ -126,34 +111,6 @@ export interface PaySchema { status: 'SUCCESS' | 'REFUND' | 'NOTPAY' | 'CLOSED'; } -export interface DataSchema { - _id: string; - userId: string; - name: string; - createTime: Date; - type: DataType; -} - -export interface DataItemSchema { - _id: string; - userId: string; - dataId: string; - type: DataType; - times: number; - text: string; - rawResponse: string[]; - result: { - q?: string; - a?: string; - abstract?: string; - }[]; - status: 0 | 1 | 2; -} - -export interface DataItemPopulate extends DataItemSchema { - userId: UserModelSchema; -} - export interface OpenApiSchema { _id: string; userId: string; diff --git a/src/types/training.d.ts b/src/types/training.d.ts deleted file mode 100644 index 43c150c53..000000000 --- a/src/types/training.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { TrainingStatusEnum } from '@/constants/model'; - -export type TrainingItemType = { - _id: string; - serviceName: string; - tuneId: string; - modelId: string; - status: `${TrainingStatusEnum}`; -};