mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 21:13:50 +00:00
v4.6.9-alpha (#918)
Co-authored-by: Mufei <327958099@qq.com> Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
@@ -34,6 +34,12 @@ export async function connectMongo({
|
||||
retryReads: true
|
||||
});
|
||||
|
||||
mongoose.connection.on('error', (error) => {
|
||||
console.log('mongo error', error);
|
||||
global.mongodb?.disconnect();
|
||||
global.mongodb = undefined;
|
||||
});
|
||||
|
||||
console.log('mongo connected');
|
||||
|
||||
afterHook && (await afterHook());
|
||||
|
@@ -3,9 +3,12 @@ import dayjs from 'dayjs';
|
||||
/* add logger */
|
||||
export const addLog = {
|
||||
log(level: 'info' | 'warn' | 'error', msg: string, obj: Record<string, any> = {}) {
|
||||
const stringifyObj = JSON.stringify(obj);
|
||||
const isEmpty = Object.keys(obj).length === 0;
|
||||
|
||||
console.log(
|
||||
`[${level.toLocaleUpperCase()}] ${dayjs().format('YYYY-MM-DD HH:mm:ss')} ${msg} ${
|
||||
level !== 'error' ? JSON.stringify(obj) : ''
|
||||
level !== 'error' && !isEmpty ? stringifyObj : ''
|
||||
}`
|
||||
);
|
||||
|
||||
|
@@ -11,6 +11,7 @@ export const initFastGPTConfig = (config?: FastGPTConfigFileType) => {
|
||||
if (!config) return;
|
||||
|
||||
global.feConfigs = config.feConfigs;
|
||||
global.systemEnv = config.systemEnv;
|
||||
global.subPlans = config.subPlans;
|
||||
|
||||
global.llmModels = config.llmModels;
|
||||
|
@@ -22,7 +22,7 @@ export const insertDatasetDataVector = async ({
|
||||
query: string;
|
||||
model: VectorModelItemType;
|
||||
}) => {
|
||||
const { vectors, charsLength } = await getVectorsByText({
|
||||
const { vectors, tokens } = await getVectorsByText({
|
||||
model,
|
||||
input: query
|
||||
});
|
||||
@@ -32,27 +32,7 @@ export const insertDatasetDataVector = async ({
|
||||
});
|
||||
|
||||
return {
|
||||
charsLength,
|
||||
tokens,
|
||||
insertId
|
||||
};
|
||||
};
|
||||
|
||||
// export const updateDatasetDataVector = async ({
|
||||
// id,
|
||||
// ...props
|
||||
// }: InsertVectorProps & {
|
||||
// id: string;
|
||||
// query: string;
|
||||
// model: VectorModelItemType;
|
||||
// }) => {
|
||||
// // insert new vector
|
||||
// const { charsLength, insertId } = await insertDatasetDataVector(props);
|
||||
|
||||
// // delete old vector
|
||||
// await deleteDatasetDataVector({
|
||||
// teamId: props.teamId,
|
||||
// id
|
||||
// });
|
||||
|
||||
// return { charsLength, insertId };
|
||||
// };
|
||||
|
@@ -1,6 +1,6 @@
|
||||
import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
|
||||
import { getAIApi } from '../config';
|
||||
import { replaceValidChars } from '../../chat/utils';
|
||||
import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
|
||||
|
||||
type GetVectorProps = {
|
||||
model: VectorModelItemType;
|
||||
@@ -37,7 +37,7 @@ export async function getVectorsByText({ model, input }: GetVectorProps) {
|
||||
}
|
||||
|
||||
return {
|
||||
charsLength: replaceValidChars(input).length,
|
||||
tokens: countPromptTokens(input),
|
||||
vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
|
||||
};
|
||||
});
|
||||
|
@@ -1,6 +1,6 @@
|
||||
import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
|
||||
import { getAIApi } from '../config';
|
||||
import { countGptMessagesChars } from '../../chat/utils';
|
||||
import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
|
||||
|
||||
export const Prompt_QuestionGuide = `我不太清楚问你什么问题,请帮我生成 3 个问题,引导我继续提问。问题的长度应小于20个字符,按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
|
||||
|
||||
@@ -34,12 +34,12 @@ export async function createQuestionGuide({
|
||||
const start = answer.indexOf('[');
|
||||
const end = answer.lastIndexOf(']');
|
||||
|
||||
const charsLength = countGptMessagesChars(concatMessages);
|
||||
const tokens = countGptMessagesTokens(concatMessages);
|
||||
|
||||
if (start === -1 || end === -1) {
|
||||
return {
|
||||
result: [],
|
||||
charsLength: 0
|
||||
tokens: 0
|
||||
};
|
||||
}
|
||||
|
||||
@@ -51,12 +51,12 @@ export async function createQuestionGuide({
|
||||
try {
|
||||
return {
|
||||
result: JSON.parse(jsonStr),
|
||||
charsLength
|
||||
tokens
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
result: [],
|
||||
charsLength: 0
|
||||
tokens: 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
@@ -1,7 +1,7 @@
|
||||
import { replaceVariable } from '@fastgpt/global/common/string/tools';
|
||||
import { getAIApi } from '../config';
|
||||
import { ChatItemType } from '@fastgpt/global/core/chat/type';
|
||||
import { countGptMessagesChars } from '../../chat/utils';
|
||||
import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
|
||||
|
||||
/*
|
||||
query extension - 问题扩展
|
||||
@@ -106,7 +106,7 @@ export const queryExtension = async ({
|
||||
rawQuery: string;
|
||||
extensionQueries: string[];
|
||||
model: string;
|
||||
charsLength: number;
|
||||
tokens: number;
|
||||
}> => {
|
||||
const systemFewShot = chatBg
|
||||
? `Q: 对话背景。
|
||||
@@ -148,7 +148,7 @@ A: ${chatBg}
|
||||
rawQuery: query,
|
||||
extensionQueries: [],
|
||||
model,
|
||||
charsLength: 0
|
||||
tokens: 0
|
||||
};
|
||||
}
|
||||
|
||||
@@ -161,7 +161,7 @@ A: ${chatBg}
|
||||
rawQuery: query,
|
||||
extensionQueries: queries,
|
||||
model,
|
||||
charsLength: countGptMessagesChars(messages)
|
||||
tokens: countGptMessagesTokens(messages)
|
||||
};
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
@@ -169,7 +169,7 @@ A: ${chatBg}
|
||||
rawQuery: query,
|
||||
extensionQueries: [],
|
||||
model,
|
||||
charsLength: 0
|
||||
tokens: 0
|
||||
};
|
||||
}
|
||||
};
|
||||
|
@@ -1,11 +1,7 @@
|
||||
import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
|
||||
import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
|
||||
import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
|
||||
import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
|
||||
import type {
|
||||
ChatCompletionContentPart,
|
||||
ChatMessageItemType
|
||||
} from '@fastgpt/global/core/ai/type.d';
|
||||
import { countMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
|
||||
import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
|
||||
import axios from 'axios';
|
||||
|
||||
/* slice chat context by tokens */
|
||||
@@ -32,26 +28,34 @@ export function ChatContextFilter({
|
||||
const chatPrompts: ChatItemType[] = messages.slice(chatStartIndex);
|
||||
|
||||
// reduce token of systemPrompt
|
||||
maxTokens -= countMessagesTokens({
|
||||
messages: systemPrompts
|
||||
});
|
||||
maxTokens -= countMessagesTokens(systemPrompts);
|
||||
|
||||
// 根据 tokens 截断内容
|
||||
const chats: ChatItemType[] = [];
|
||||
// Save the last chat prompt(question)
|
||||
const question = chatPrompts.pop();
|
||||
if (!question) {
|
||||
return systemPrompts;
|
||||
}
|
||||
const chats: ChatItemType[] = [question];
|
||||
|
||||
// 从后往前截取对话内容
|
||||
for (let i = chatPrompts.length - 1; i >= 0; i--) {
|
||||
const item = chatPrompts[i];
|
||||
chats.unshift(item);
|
||||
// 从后往前截取对话内容, 每次需要截取2个
|
||||
while (1) {
|
||||
const assistant = chatPrompts.pop();
|
||||
const user = chatPrompts.pop();
|
||||
if (!assistant || !user) {
|
||||
break;
|
||||
}
|
||||
|
||||
const tokens = countPromptTokens(item.value, adaptRole_Chat2Message(item.obj));
|
||||
const tokens = countMessagesTokens([assistant, user]);
|
||||
maxTokens -= tokens;
|
||||
/* 整体 tokens 超出范围,截断 */
|
||||
if (maxTokens < 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
/* 整体 tokens 超出范围, system必须保留 */
|
||||
if (maxTokens <= 0) {
|
||||
if (chats.length > 1) {
|
||||
chats.shift();
|
||||
}
|
||||
chats.unshift(assistant);
|
||||
chats.unshift(user);
|
||||
|
||||
if (chatPrompts.length === 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -59,16 +63,6 @@ export function ChatContextFilter({
|
||||
return [...systemPrompts, ...chats];
|
||||
}
|
||||
|
||||
export const replaceValidChars = (str: string) => {
|
||||
const reg = /[\s\r\n]+/g;
|
||||
return str.replace(reg, '');
|
||||
};
|
||||
export const countMessagesChars = (messages: ChatItemType[]) => {
|
||||
return messages.reduce((sum, item) => sum + replaceValidChars(item.value).length, 0);
|
||||
};
|
||||
export const countGptMessagesChars = (messages: ChatMessageItemType[]) =>
|
||||
messages.reduce((sum, item) => sum + replaceValidChars(item.content).length, 0);
|
||||
|
||||
/**
|
||||
string to vision model. Follow the markdown code block rule for interception:
|
||||
|
||||
|
@@ -4,6 +4,7 @@ import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||
import {
|
||||
DatasetStatusEnum,
|
||||
DatasetStatusMap,
|
||||
DatasetTypeEnum,
|
||||
DatasetTypeMap
|
||||
} from '@fastgpt/global/core/dataset/constants';
|
||||
import {
|
||||
@@ -39,7 +40,7 @@ const DatasetSchema = new Schema({
|
||||
type: String,
|
||||
enum: Object.keys(DatasetTypeMap),
|
||||
required: true,
|
||||
default: 'dataset'
|
||||
default: DatasetTypeEnum.dataset
|
||||
},
|
||||
status: {
|
||||
type: String,
|
||||
|
@@ -46,12 +46,16 @@ export async function pushDataListToTrainingQueue({
|
||||
} = await getCollectionWithDataset(collectionId);
|
||||
|
||||
const checkModelValid = async () => {
|
||||
if (trainingMode === TrainingModeEnum.chunk) {
|
||||
const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
|
||||
if (!vectorModelData) {
|
||||
return Promise.reject(`File model ${vectorModel} is inValid`);
|
||||
}
|
||||
const agentModelData = datasetModelList?.find((item) => item.model === agentModel);
|
||||
if (!agentModelData) {
|
||||
return Promise.reject(`Vector model ${agentModel} is inValid`);
|
||||
}
|
||||
const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
|
||||
if (!vectorModelData) {
|
||||
return Promise.reject(`File model ${vectorModel} is inValid`);
|
||||
}
|
||||
|
||||
if (trainingMode === TrainingModeEnum.chunk) {
|
||||
return {
|
||||
maxToken: vectorModelData.maxToken * 1.3,
|
||||
model: vectorModelData.model,
|
||||
@@ -59,17 +63,14 @@ export async function pushDataListToTrainingQueue({
|
||||
};
|
||||
}
|
||||
|
||||
if (trainingMode === TrainingModeEnum.qa) {
|
||||
const qaModelData = datasetModelList?.find((item) => item.model === agentModel);
|
||||
if (!qaModelData) {
|
||||
return Promise.reject(`Vector model ${agentModel} is inValid`);
|
||||
}
|
||||
if (trainingMode === TrainingModeEnum.qa || trainingMode === TrainingModeEnum.auto) {
|
||||
return {
|
||||
maxToken: qaModelData.maxContext * 0.8,
|
||||
model: qaModelData.model,
|
||||
maxToken: agentModelData.maxContext * 0.8,
|
||||
model: agentModelData.model,
|
||||
weight: 0
|
||||
};
|
||||
}
|
||||
|
||||
return Promise.reject(`Training mode "${trainingMode}" is inValid`);
|
||||
};
|
||||
|
||||
|
41
packages/service/core/dataset/training/utils.ts
Normal file
41
packages/service/core/dataset/training/utils.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { addLog } from '../../../common/system/log';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { MongoDatasetTraining } from './schema';
|
||||
|
||||
export const checkInvalidChunkAndLock = async ({
|
||||
err,
|
||||
errText,
|
||||
data
|
||||
}: {
|
||||
err: any;
|
||||
errText: string;
|
||||
data: DatasetTrainingSchemaType;
|
||||
}) => {
|
||||
if (err?.response) {
|
||||
addLog.error(`openai error: ${errText}`, {
|
||||
status: err.response?.status,
|
||||
statusText: err.response?.statusText,
|
||||
data: err.response?.data
|
||||
});
|
||||
} else {
|
||||
addLog.error(getErrText(err, errText), err);
|
||||
}
|
||||
|
||||
if (
|
||||
err?.message === 'invalid message format' ||
|
||||
err?.type === 'invalid_request_error' ||
|
||||
err?.code === 500
|
||||
) {
|
||||
addLog.info('Lock training data');
|
||||
console.log(err);
|
||||
|
||||
try {
|
||||
await MongoDatasetTraining.findByIdAndUpdate(data._id, {
|
||||
lockTime: new Date('2998/5/5')
|
||||
});
|
||||
} catch (error) {}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
@@ -6,6 +6,7 @@ import {
|
||||
TeamCollectionName,
|
||||
TeamMemberCollectionName
|
||||
} from '@fastgpt/global/support/user/team/constant';
|
||||
import { appCollectionName } from '../../core/app/schema';
|
||||
|
||||
const OutLinkSchema = new Schema({
|
||||
shareId: {
|
||||
@@ -24,7 +25,7 @@ const OutLinkSchema = new Schema({
|
||||
},
|
||||
appId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: 'model',
|
||||
ref: appCollectionName,
|
||||
required: true
|
||||
},
|
||||
type: {
|
||||
|
@@ -1,7 +1,5 @@
|
||||
import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
|
||||
import { parseHeaderCert } from '../controller';
|
||||
import { AuthModeType } from '../type';
|
||||
import { authOutLinkValid } from './outLink';
|
||||
import { SERVICE_LOCAL_HOST } from '../../../common/system/tools';
|
||||
|
||||
export const authCert = async (props: AuthModeType) => {
|
||||
@@ -13,25 +11,6 @@ export const authCert = async (props: AuthModeType) => {
|
||||
canWrite: true
|
||||
};
|
||||
};
|
||||
export async function authCertOrShareId({
|
||||
shareId,
|
||||
...props
|
||||
}: AuthModeType & { shareId?: string }) {
|
||||
if (!shareId) {
|
||||
return authCert(props);
|
||||
}
|
||||
|
||||
const { shareChat } = await authOutLinkValid({ shareId });
|
||||
|
||||
return {
|
||||
teamId: String(shareChat.teamId),
|
||||
tmbId: String(shareChat.tmbId),
|
||||
authType: AuthUserTypeEnum.outLink,
|
||||
apikey: '',
|
||||
isOwner: false,
|
||||
canWrite: false
|
||||
};
|
||||
}
|
||||
|
||||
/* auth the request from local service */
|
||||
export const authRequestFromLocal = ({ req }: AuthModeType) => {
|
||||
|
@@ -23,11 +23,11 @@ async function getTeamMember(match: Record<string, any>): Promise<TeamItemType>
|
||||
avatar: tmb.teamId.avatar,
|
||||
balance: tmb.teamId.balance,
|
||||
tmbId: String(tmb._id),
|
||||
teamDomain: tmb.teamId?.teamDomain,
|
||||
role: tmb.role,
|
||||
status: tmb.status,
|
||||
defaultTeam: tmb.defaultTeam,
|
||||
canWrite: tmb.role !== TeamMemberRoleEnum.visitor,
|
||||
maxSize: tmb.teamId.maxSize
|
||||
canWrite: tmb.role !== TeamMemberRoleEnum.visitor
|
||||
};
|
||||
}
|
||||
|
||||
@@ -55,14 +55,12 @@ export async function createDefaultTeam({
|
||||
teamName = 'My Team',
|
||||
avatar = '/icon/logo.svg',
|
||||
balance,
|
||||
maxSize = 5,
|
||||
session
|
||||
}: {
|
||||
userId: string;
|
||||
teamName?: string;
|
||||
avatar?: string;
|
||||
balance?: number;
|
||||
maxSize?: number;
|
||||
session: ClientSession;
|
||||
}) {
|
||||
// auth default team
|
||||
@@ -82,7 +80,6 @@ export async function createDefaultTeam({
|
||||
name: teamName,
|
||||
avatar,
|
||||
balance,
|
||||
maxSize,
|
||||
createTime: new Date()
|
||||
}
|
||||
],
|
||||
@@ -106,8 +103,7 @@ export async function createDefaultTeam({
|
||||
console.log('default team exist', userId);
|
||||
await MongoTeam.findByIdAndUpdate(tmb.teamId, {
|
||||
$set: {
|
||||
...(balance !== undefined && { balance }),
|
||||
maxSize
|
||||
...(balance !== undefined && { balance })
|
||||
}
|
||||
});
|
||||
}
|
||||
|
@@ -25,11 +25,7 @@ const TeamSchema = new Schema({
|
||||
type: Number,
|
||||
default: 0
|
||||
},
|
||||
maxSize: {
|
||||
type: Number,
|
||||
default: 1
|
||||
},
|
||||
tagsUrl: {
|
||||
teamDomain: {
|
||||
type: String
|
||||
},
|
||||
limit: {
|
||||
|
@@ -1,23 +1,24 @@
|
||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
||||
const { Schema, model, models } = connectionMongo;
|
||||
import { TeamTagsSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
|
||||
import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
|
||||
import {
|
||||
TeamCollectionName,
|
||||
TeamTagsCollectionName
|
||||
} from '@fastgpt/global/support/user/team/constant';
|
||||
|
||||
const TeamTagsSchema = new Schema({
|
||||
label: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
const TeamTagSchema = new Schema({
|
||||
teamId: {
|
||||
type: Schema.Types.ObjectId,
|
||||
ref: TeamCollectionName,
|
||||
required: true
|
||||
},
|
||||
key: {
|
||||
type: String
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
label: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
createTime: {
|
||||
type: Date,
|
||||
@@ -26,10 +27,10 @@ const TeamTagsSchema = new Schema({
|
||||
});
|
||||
|
||||
try {
|
||||
TeamTagsSchema.index({ teamId: 1 });
|
||||
TeamTagSchema.index({ teamId: 1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoTeamTags: Model<TeamTagsSchemaType> =
|
||||
models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagsSchema);
|
||||
models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagSchema);
|
||||
|
@@ -31,13 +31,19 @@ export const createTrainingUsage = async ({
|
||||
{
|
||||
moduleName: 'support.wallet.moduleName.index',
|
||||
model: vectorModel,
|
||||
charsLength: 0,
|
||||
tokens: 0,
|
||||
amount: 0
|
||||
},
|
||||
{
|
||||
moduleName: 'support.wallet.moduleName.qa',
|
||||
model: agentModel,
|
||||
charsLength: 0,
|
||||
tokens: 0,
|
||||
amount: 0
|
||||
},
|
||||
{
|
||||
moduleName: 'core.dataset.training.Auto mode',
|
||||
model: agentModel,
|
||||
tokens: 0,
|
||||
amount: 0
|
||||
}
|
||||
]
|
||||
|
28
packages/service/support/wallet/usage/utils.ts
Normal file
28
packages/service/support/wallet/usage/utils.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { ModelTypeEnum, getModelMap } from '../../../core/ai/model';
|
||||
|
||||
export const formatModelChars2Points = ({
|
||||
model,
|
||||
tokens = 0,
|
||||
modelType,
|
||||
multiple = 1000
|
||||
}: {
|
||||
model: string;
|
||||
tokens: number;
|
||||
modelType: `${ModelTypeEnum}`;
|
||||
multiple?: number;
|
||||
}) => {
|
||||
const modelData = getModelMap?.[modelType]?.(model);
|
||||
if (!modelData) {
|
||||
return {
|
||||
totalPoints: 0,
|
||||
modelName: ''
|
||||
};
|
||||
}
|
||||
|
||||
const totalPoints = (modelData.charsPointsPrice || 0) * (tokens / multiple);
|
||||
|
||||
return {
|
||||
modelName: modelData.name,
|
||||
totalPoints
|
||||
};
|
||||
};
|
3
packages/service/type.d.ts
vendored
3
packages/service/type.d.ts
vendored
@@ -1,4 +1,4 @@
|
||||
import { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types';
|
||||
import { FastGPTFeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types';
|
||||
import {
|
||||
AudioSpeechModelType,
|
||||
ReRankModelItemType,
|
||||
@@ -10,6 +10,7 @@ import { SubPlanType } from '@fastgpt/global/support/wallet/sub/type';
|
||||
|
||||
declare global {
|
||||
var feConfigs: FastGPTFeConfigsType;
|
||||
var systemEnv: SystemEnvType;
|
||||
var subPlans: SubPlanType | undefined;
|
||||
|
||||
var llmModels: LLMModelItemType[];
|
||||
|
Reference in New Issue
Block a user