v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-03-04 00:05:25 +08:00
committed by GitHub
parent f9f0b4bffd
commit 42a8184ea0
153 changed files with 4906 additions and 4307 deletions

View File

@@ -34,6 +34,12 @@ export async function connectMongo({
retryReads: true
});
mongoose.connection.on('error', (error) => {
console.log('mongo error', error);
global.mongodb?.disconnect();
global.mongodb = undefined;
});
console.log('mongo connected');
afterHook && (await afterHook());

View File

@@ -3,9 +3,12 @@ import dayjs from 'dayjs';
/* add logger */
export const addLog = {
log(level: 'info' | 'warn' | 'error', msg: string, obj: Record<string, any> = {}) {
const stringifyObj = JSON.stringify(obj);
const isEmpty = Object.keys(obj).length === 0;
console.log(
`[${level.toLocaleUpperCase()}] ${dayjs().format('YYYY-MM-DD HH:mm:ss')} ${msg} ${
level !== 'error' ? JSON.stringify(obj) : ''
level !== 'error' && !isEmpty ? stringifyObj : ''
}`
);

View File

@@ -11,6 +11,7 @@ export const initFastGPTConfig = (config?: FastGPTConfigFileType) => {
if (!config) return;
global.feConfigs = config.feConfigs;
global.systemEnv = config.systemEnv;
global.subPlans = config.subPlans;
global.llmModels = config.llmModels;

View File

@@ -22,7 +22,7 @@ export const insertDatasetDataVector = async ({
query: string;
model: VectorModelItemType;
}) => {
const { vectors, charsLength } = await getVectorsByText({
const { vectors, tokens } = await getVectorsByText({
model,
input: query
});
@@ -32,27 +32,7 @@ export const insertDatasetDataVector = async ({
});
return {
charsLength,
tokens,
insertId
};
};
// export const updateDatasetDataVector = async ({
// id,
// ...props
// }: InsertVectorProps & {
// id: string;
// query: string;
// model: VectorModelItemType;
// }) => {
// // insert new vector
// const { charsLength, insertId } = await insertDatasetDataVector(props);
// // delete old vector
// await deleteDatasetDataVector({
// teamId: props.teamId,
// id
// });
// return { charsLength, insertId };
// };

View File

@@ -1,6 +1,6 @@
import { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
import { getAIApi } from '../config';
import { replaceValidChars } from '../../chat/utils';
import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
type GetVectorProps = {
model: VectorModelItemType;
@@ -37,7 +37,7 @@ export async function getVectorsByText({ model, input }: GetVectorProps) {
}
return {
charsLength: replaceValidChars(input).length,
tokens: countPromptTokens(input),
vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
};
});

View File

@@ -1,6 +1,6 @@
import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
import { getAIApi } from '../config';
import { countGptMessagesChars } from '../../chat/utils';
import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
export const Prompt_QuestionGuide = `我不太清楚问你什么问题,请帮我生成 3 个问题引导我继续提问。问题的长度应小于20个字符按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
@@ -34,12 +34,12 @@ export async function createQuestionGuide({
const start = answer.indexOf('[');
const end = answer.lastIndexOf(']');
const charsLength = countGptMessagesChars(concatMessages);
const tokens = countGptMessagesTokens(concatMessages);
if (start === -1 || end === -1) {
return {
result: [],
charsLength: 0
tokens: 0
};
}
@@ -51,12 +51,12 @@ export async function createQuestionGuide({
try {
return {
result: JSON.parse(jsonStr),
charsLength
tokens
};
} catch (error) {
return {
result: [],
charsLength: 0
tokens: 0
};
}
}

View File

@@ -1,7 +1,7 @@
import { replaceVariable } from '@fastgpt/global/common/string/tools';
import { getAIApi } from '../config';
import { ChatItemType } from '@fastgpt/global/core/chat/type';
import { countGptMessagesChars } from '../../chat/utils';
import { countGptMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
/*
query extension - 问题扩展
@@ -106,7 +106,7 @@ export const queryExtension = async ({
rawQuery: string;
extensionQueries: string[];
model: string;
charsLength: number;
tokens: number;
}> => {
const systemFewShot = chatBg
? `Q: 对话背景。
@@ -148,7 +148,7 @@ A: ${chatBg}
rawQuery: query,
extensionQueries: [],
model,
charsLength: 0
tokens: 0
};
}
@@ -161,7 +161,7 @@ A: ${chatBg}
rawQuery: query,
extensionQueries: queries,
model,
charsLength: countGptMessagesChars(messages)
tokens: countGptMessagesTokens(messages)
};
} catch (error) {
console.log(error);
@@ -169,7 +169,7 @@ A: ${chatBg}
rawQuery: query,
extensionQueries: [],
model,
charsLength: 0
tokens: 0
};
}
};

View File

@@ -1,11 +1,7 @@
import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
import type {
ChatCompletionContentPart,
ChatMessageItemType
} from '@fastgpt/global/core/ai/type.d';
import { countMessagesTokens } from '@fastgpt/global/common/string/tiktoken';
import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
import axios from 'axios';
/* slice chat context by tokens */
@@ -32,26 +28,34 @@ export function ChatContextFilter({
const chatPrompts: ChatItemType[] = messages.slice(chatStartIndex);
// reduce token of systemPrompt
maxTokens -= countMessagesTokens({
messages: systemPrompts
});
maxTokens -= countMessagesTokens(systemPrompts);
// 根据 tokens 截断内容
const chats: ChatItemType[] = [];
// Save the last chat prompt(question)
const question = chatPrompts.pop();
if (!question) {
return systemPrompts;
}
const chats: ChatItemType[] = [question];
// 从后往前截取对话内容
for (let i = chatPrompts.length - 1; i >= 0; i--) {
const item = chatPrompts[i];
chats.unshift(item);
// 从后往前截取对话内容, 每次需要截取2个
while (1) {
const assistant = chatPrompts.pop();
const user = chatPrompts.pop();
if (!assistant || !user) {
break;
}
const tokens = countPromptTokens(item.value, adaptRole_Chat2Message(item.obj));
const tokens = countMessagesTokens([assistant, user]);
maxTokens -= tokens;
/* 整体 tokens 超出范围,截断 */
if (maxTokens < 0) {
break;
}
/* 整体 tokens 超出范围, system必须保留 */
if (maxTokens <= 0) {
if (chats.length > 1) {
chats.shift();
}
chats.unshift(assistant);
chats.unshift(user);
if (chatPrompts.length === 0) {
break;
}
}
@@ -59,16 +63,6 @@ export function ChatContextFilter({
return [...systemPrompts, ...chats];
}
export const replaceValidChars = (str: string) => {
const reg = /[\s\r\n]+/g;
return str.replace(reg, '');
};
export const countMessagesChars = (messages: ChatItemType[]) => {
return messages.reduce((sum, item) => sum + replaceValidChars(item.value).length, 0);
};
export const countGptMessagesChars = (messages: ChatMessageItemType[]) =>
messages.reduce((sum, item) => sum + replaceValidChars(item.content).length, 0);
/**
string to vision model. Follow the markdown code block rule for interception:

View File

@@ -4,6 +4,7 @@ import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
import {
DatasetStatusEnum,
DatasetStatusMap,
DatasetTypeEnum,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constants';
import {
@@ -39,7 +40,7 @@ const DatasetSchema = new Schema({
type: String,
enum: Object.keys(DatasetTypeMap),
required: true,
default: 'dataset'
default: DatasetTypeEnum.dataset
},
status: {
type: String,

View File

@@ -46,12 +46,16 @@ export async function pushDataListToTrainingQueue({
} = await getCollectionWithDataset(collectionId);
const checkModelValid = async () => {
if (trainingMode === TrainingModeEnum.chunk) {
const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
if (!vectorModelData) {
return Promise.reject(`File model ${vectorModel} is inValid`);
}
const agentModelData = datasetModelList?.find((item) => item.model === agentModel);
if (!agentModelData) {
return Promise.reject(`Vector model ${agentModel} is inValid`);
}
const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
if (!vectorModelData) {
return Promise.reject(`File model ${vectorModel} is inValid`);
}
if (trainingMode === TrainingModeEnum.chunk) {
return {
maxToken: vectorModelData.maxToken * 1.3,
model: vectorModelData.model,
@@ -59,17 +63,14 @@ export async function pushDataListToTrainingQueue({
};
}
if (trainingMode === TrainingModeEnum.qa) {
const qaModelData = datasetModelList?.find((item) => item.model === agentModel);
if (!qaModelData) {
return Promise.reject(`Vector model ${agentModel} is inValid`);
}
if (trainingMode === TrainingModeEnum.qa || trainingMode === TrainingModeEnum.auto) {
return {
maxToken: qaModelData.maxContext * 0.8,
model: qaModelData.model,
maxToken: agentModelData.maxContext * 0.8,
model: agentModelData.model,
weight: 0
};
}
return Promise.reject(`Training mode "${trainingMode}" is inValid`);
};

View File

@@ -0,0 +1,41 @@
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
import { addLog } from '../../../common/system/log';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { MongoDatasetTraining } from './schema';
export const checkInvalidChunkAndLock = async ({
err,
errText,
data
}: {
err: any;
errText: string;
data: DatasetTrainingSchemaType;
}) => {
if (err?.response) {
addLog.error(`openai error: ${errText}`, {
status: err.response?.status,
statusText: err.response?.statusText,
data: err.response?.data
});
} else {
addLog.error(getErrText(err, errText), err);
}
if (
err?.message === 'invalid message format' ||
err?.type === 'invalid_request_error' ||
err?.code === 500
) {
addLog.info('Lock training data');
console.log(err);
try {
await MongoDatasetTraining.findByIdAndUpdate(data._id, {
lockTime: new Date('2998/5/5')
});
} catch (error) {}
return true;
}
return false;
};

View File

@@ -6,6 +6,7 @@ import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
import { appCollectionName } from '../../core/app/schema';
const OutLinkSchema = new Schema({
shareId: {
@@ -24,7 +25,7 @@ const OutLinkSchema = new Schema({
},
appId: {
type: Schema.Types.ObjectId,
ref: 'model',
ref: appCollectionName,
required: true
},
type: {

View File

@@ -1,7 +1,5 @@
import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
import { parseHeaderCert } from '../controller';
import { AuthModeType } from '../type';
import { authOutLinkValid } from './outLink';
import { SERVICE_LOCAL_HOST } from '../../../common/system/tools';
export const authCert = async (props: AuthModeType) => {
@@ -13,25 +11,6 @@ export const authCert = async (props: AuthModeType) => {
canWrite: true
};
};
export async function authCertOrShareId({
shareId,
...props
}: AuthModeType & { shareId?: string }) {
if (!shareId) {
return authCert(props);
}
const { shareChat } = await authOutLinkValid({ shareId });
return {
teamId: String(shareChat.teamId),
tmbId: String(shareChat.tmbId),
authType: AuthUserTypeEnum.outLink,
apikey: '',
isOwner: false,
canWrite: false
};
}
/* auth the request from local service */
export const authRequestFromLocal = ({ req }: AuthModeType) => {

View File

@@ -23,11 +23,11 @@ async function getTeamMember(match: Record<string, any>): Promise<TeamItemType>
avatar: tmb.teamId.avatar,
balance: tmb.teamId.balance,
tmbId: String(tmb._id),
teamDomain: tmb.teamId?.teamDomain,
role: tmb.role,
status: tmb.status,
defaultTeam: tmb.defaultTeam,
canWrite: tmb.role !== TeamMemberRoleEnum.visitor,
maxSize: tmb.teamId.maxSize
canWrite: tmb.role !== TeamMemberRoleEnum.visitor
};
}
@@ -55,14 +55,12 @@ export async function createDefaultTeam({
teamName = 'My Team',
avatar = '/icon/logo.svg',
balance,
maxSize = 5,
session
}: {
userId: string;
teamName?: string;
avatar?: string;
balance?: number;
maxSize?: number;
session: ClientSession;
}) {
// auth default team
@@ -82,7 +80,6 @@ export async function createDefaultTeam({
name: teamName,
avatar,
balance,
maxSize,
createTime: new Date()
}
],
@@ -106,8 +103,7 @@ export async function createDefaultTeam({
console.log('default team exist', userId);
await MongoTeam.findByIdAndUpdate(tmb.teamId, {
$set: {
...(balance !== undefined && { balance }),
maxSize
...(balance !== undefined && { balance })
}
});
}

View File

@@ -25,11 +25,7 @@ const TeamSchema = new Schema({
type: Number,
default: 0
},
maxSize: {
type: Number,
default: 1
},
tagsUrl: {
teamDomain: {
type: String
},
limit: {

View File

@@ -1,23 +1,24 @@
import { connectionMongo, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { TeamTagsSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
import {
TeamCollectionName,
TeamTagsCollectionName
} from '@fastgpt/global/support/user/team/constant';
const TeamTagsSchema = new Schema({
label: {
type: String,
required: true
},
const TeamTagSchema = new Schema({
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
key: {
type: String
type: String,
required: true
},
label: {
type: String,
required: true
},
createTime: {
type: Date,
@@ -26,10 +27,10 @@ const TeamTagsSchema = new Schema({
});
try {
TeamTagsSchema.index({ teamId: 1 });
TeamTagSchema.index({ teamId: 1 });
} catch (error) {
console.log(error);
}
export const MongoTeamTags: Model<TeamTagsSchemaType> =
models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagsSchema);
models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagSchema);

View File

@@ -31,13 +31,19 @@ export const createTrainingUsage = async ({
{
moduleName: 'support.wallet.moduleName.index',
model: vectorModel,
charsLength: 0,
tokens: 0,
amount: 0
},
{
moduleName: 'support.wallet.moduleName.qa',
model: agentModel,
charsLength: 0,
tokens: 0,
amount: 0
},
{
moduleName: 'core.dataset.training.Auto mode',
model: agentModel,
tokens: 0,
amount: 0
}
]

View File

@@ -0,0 +1,28 @@
import { ModelTypeEnum, getModelMap } from '../../../core/ai/model';
export const formatModelChars2Points = ({
model,
tokens = 0,
modelType,
multiple = 1000
}: {
model: string;
tokens: number;
modelType: `${ModelTypeEnum}`;
multiple?: number;
}) => {
const modelData = getModelMap?.[modelType]?.(model);
if (!modelData) {
return {
totalPoints: 0,
modelName: ''
};
}
const totalPoints = (modelData.charsPointsPrice || 0) * (tokens / multiple);
return {
modelName: modelData.name,
totalPoints
};
};

View File

@@ -1,4 +1,4 @@
import { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types';
import { FastGPTFeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types';
import {
AudioSpeechModelType,
ReRankModelItemType,
@@ -10,6 +10,7 @@ import { SubPlanType } from '@fastgpt/global/support/wallet/sub/type';
declare global {
var feConfigs: FastGPTFeConfigsType;
var systemEnv: SystemEnvType;
var subPlans: SubPlanType | undefined;
var llmModels: LLMModelItemType[];