mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 13:03:50 +00:00
perf: 压缩上下文
This commit is contained in:
@@ -60,10 +60,9 @@ export const chatProblem = `
|
||||
`;
|
||||
|
||||
export const versionIntro = `
|
||||
## Fast GPT V2.4
|
||||
* 优化文件拆分功能,可自定义提示词。
|
||||
* 优化文件拆分和索引生成的速度。
|
||||
* 定制知识库:创建模型时可以选择【知识库】模型, 可以手动导入知识点或者直接导入一个文件自动学习。
|
||||
## Fast GPT V2.5
|
||||
* 内容压缩,替换中文标点符号和多余符号,减少一些上下文tokens。
|
||||
* 优化 QA 拆分记账。
|
||||
`;
|
||||
|
||||
export const shareHint = `
|
||||
|
@@ -1,170 +0,0 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { createParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { getOpenAIApi, authChat } from '@/service/utils/chat';
|
||||
import { httpsAgent } from '@/service/utils/tools';
|
||||
import { ChatItemType } from '@/types/chat';
|
||||
import { jsonRes } from '@/service/response';
|
||||
import type { ModelSchema } from '@/types/mongoSchema';
|
||||
import { PassThrough } from 'stream';
|
||||
import { modelList } from '@/constants/model';
|
||||
import { pushChatBill } from '@/service/events/pushBill';
|
||||
|
||||
/* 发送提示词 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
let step = 0; // step=1时,表示开始了流响应
|
||||
const stream = new PassThrough();
|
||||
stream.on('error', () => {
|
||||
console.log('error: ', 'stream error');
|
||||
stream.destroy();
|
||||
});
|
||||
res.on('close', () => {
|
||||
stream.destroy();
|
||||
});
|
||||
res.on('error', () => {
|
||||
console.log('error: ', 'request error');
|
||||
stream.destroy();
|
||||
});
|
||||
|
||||
try {
|
||||
const { chatId, prompt } = req.body as {
|
||||
prompt: ChatItemType;
|
||||
chatId: string;
|
||||
};
|
||||
const { authorization } = req.headers;
|
||||
if (!chatId || !prompt) {
|
||||
throw new Error('缺少参数');
|
||||
}
|
||||
|
||||
await connectToDatabase();
|
||||
|
||||
const { chat, userApiKey, systemKey, userId } = await authChat(chatId, authorization);
|
||||
|
||||
const model: ModelSchema = chat.modelId;
|
||||
|
||||
// 读取对话内容
|
||||
const prompts = [...chat.content, prompt];
|
||||
|
||||
// 上下文长度过滤
|
||||
const maxContext = model.security.contextMaxLen;
|
||||
const filterPrompts =
|
||||
prompts.length > maxContext ? prompts.slice(prompts.length - maxContext) : prompts;
|
||||
|
||||
// 格式化文本内容
|
||||
const formatPrompts: string[] = filterPrompts.map((item: ChatItemType) => item.value);
|
||||
// 如果有系统提示词,自动插入
|
||||
if (model.systemPrompt) {
|
||||
formatPrompts.unshift(`${model.systemPrompt}`);
|
||||
}
|
||||
|
||||
const promptText = formatPrompts.join('</s>');
|
||||
|
||||
// 计算温度
|
||||
const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
|
||||
if (!modelConstantsData) {
|
||||
throw new Error('模型异常,请用 chatgpt 模型');
|
||||
}
|
||||
const temperature = modelConstantsData.maxTemperature * (model.temperature / 10);
|
||||
|
||||
// 获取 chatAPI
|
||||
const chatAPI = getOpenAIApi(userApiKey || systemKey);
|
||||
let startTime = Date.now();
|
||||
// console.log({
|
||||
// model: model.service.chatModel,
|
||||
// temperature: temperature,
|
||||
// prompt: promptText,
|
||||
// stream: true,
|
||||
// max_tokens:
|
||||
// model.trainingTimes > 0 ? modelConstantsData.trainedMaxToken : modelConstantsData.maxToken,
|
||||
// presence_penalty: -0.5, // 越大,越容易出现新内容
|
||||
// frequency_penalty: 0.5, // 越大,重复内容越少
|
||||
// stop: [`###`]
|
||||
// });
|
||||
// 发出请求
|
||||
const chatResponse = await chatAPI.createCompletion(
|
||||
{
|
||||
model: model.service.chatModel,
|
||||
temperature: temperature,
|
||||
prompt: promptText,
|
||||
stream: true,
|
||||
max_tokens: modelConstantsData.maxToken,
|
||||
presence_penalty: -0.5, // 越大,越容易出现新内容
|
||||
frequency_penalty: 0.5, // 越大,重复内容越少
|
||||
stop: [`###`, '。!?.!.']
|
||||
},
|
||||
{
|
||||
timeout: 40000,
|
||||
responseType: 'stream',
|
||||
httpsAgent
|
||||
}
|
||||
);
|
||||
|
||||
console.log('api response time:', `${(Date.now() - startTime) / 1000}s`);
|
||||
|
||||
// 创建响应流
|
||||
res.setHeader('Content-Type', 'text/event-stream;charset-utf-8');
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
res.setHeader('X-Accel-Buffering', 'no');
|
||||
res.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
step = 1;
|
||||
|
||||
let responseContent = '';
|
||||
stream.pipe(res);
|
||||
|
||||
const onParse = async (event: ParsedEvent | ReconnectInterval) => {
|
||||
if (event.type !== 'event') return;
|
||||
const data = event.data;
|
||||
if (data === '[DONE]') return;
|
||||
try {
|
||||
const json = JSON.parse(data);
|
||||
const content: string = json?.choices?.[0].text || '';
|
||||
// console.log('content:', content);
|
||||
if (!content || (responseContent === '' && content === '\n')) return;
|
||||
|
||||
responseContent += content;
|
||||
!stream.destroyed && stream.push(content.replace(/\n/g, '<br/>'));
|
||||
} catch (error) {
|
||||
error;
|
||||
}
|
||||
};
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
try {
|
||||
for await (const chunk of chatResponse.data as any) {
|
||||
if (stream.destroyed) {
|
||||
// 流被中断了,直接忽略后面的内容
|
||||
break;
|
||||
}
|
||||
const parser = createParser(onParse);
|
||||
parser.feed(decoder.decode(chunk));
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('pipe error', error);
|
||||
}
|
||||
// close stream
|
||||
!stream.destroyed && stream.push(null);
|
||||
stream.destroy();
|
||||
|
||||
// 只有使用平台的 key 才计费
|
||||
pushChatBill({
|
||||
isPay: !userApiKey,
|
||||
modelName: model.service.modelName,
|
||||
userId,
|
||||
chatId,
|
||||
text: promptText + responseContent
|
||||
});
|
||||
} catch (err: any) {
|
||||
// console.log(err?.response);
|
||||
if (step === 1) {
|
||||
// 直接结束流
|
||||
console.log('error,结束');
|
||||
stream.destroy();
|
||||
} else {
|
||||
res.status(500);
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error: err
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
@@ -54,21 +54,30 @@ export const httpsAgent =
|
||||
|
||||
/* tokens 截断 */
|
||||
export const openaiChatFilter = (prompts: ChatItemType[], maxTokens: number) => {
|
||||
const formatPrompts = prompts.map((item) => ({
|
||||
obj: item.obj,
|
||||
value: item.value
|
||||
.replace(/[\u3000\u3001\uff01-\uff5e\u3002]/g, ' ') // 中文标点改空格
|
||||
.replace(/\n+/g, '\n') // 连续空行
|
||||
.replace(/[^\S\r\n]+/g, ' ') // 连续空白内容
|
||||
.trim()
|
||||
}));
|
||||
|
||||
let res: ChatItemType[] = [];
|
||||
|
||||
let systemPrompt: ChatItemType | null = null;
|
||||
|
||||
// System 词保留
|
||||
if (prompts[0]?.obj === 'SYSTEM') {
|
||||
systemPrompt = prompts.shift() as ChatItemType;
|
||||
maxTokens -= encode(prompts[0].value).length;
|
||||
if (formatPrompts[0]?.obj === 'SYSTEM') {
|
||||
systemPrompt = formatPrompts.shift() as ChatItemType;
|
||||
maxTokens -= encode(formatPrompts[0].value).length;
|
||||
}
|
||||
|
||||
// 从后往前截取
|
||||
for (let i = prompts.length - 1; i >= 0; i--) {
|
||||
const tokens = encode(prompts[i].value).length;
|
||||
for (let i = formatPrompts.length - 1; i >= 0; i--) {
|
||||
const tokens = encode(formatPrompts[i].value).length;
|
||||
if (maxTokens >= tokens) {
|
||||
res.unshift(prompts[i]);
|
||||
res.unshift(formatPrompts[i]);
|
||||
maxTokens -= tokens;
|
||||
} else {
|
||||
break;
|
||||
|
Reference in New Issue
Block a user