ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-10-18 09:24:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.8.6 fix (#1963)

Browse Source 
* feat: log store

* fix: full text search match query

* perf: mongo schema import, Avoid duplicate import
This commit is contained in:
Archer
2024-07-05 17:37:42 +08:00
committed by GitHub
parent 88d10451c9
commit 5605f1a892
39 changed files with 252 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
packages/service/common/buffer/rawText/schema.ts
View File

@@ -1,4 +1,4 @@
import { connectionMongo, type Model } from '../../mongo';
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
const { Schema, model, models } = connectionMongo;
import { RawTextBufferSchemaType } from './type';
@@ -28,6 +28,7 @@ try {
console.log(error);
}
export const MongoRawTextBuffer: Model<RawTextBufferSchemaType> =
models[collectionName] || model(collectionName, RawTextBufferSchema);
MongoRawTextBuffer.syncIndexes();
export const MongoRawTextBuffer = getMongoModel<RawTextBufferSchemaType>(
collectionName,
RawTextBufferSchema
);

6
packages/service/common/buffer/tts/schema.ts
View File

@@ -1,4 +1,4 @@
import { connectionMongo, type Model } from '../../../common/mongo';
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { TTSBufferSchemaType } from './type.d';
@@ -31,6 +31,4 @@ try {
console.log(error);
}
export const MongoTTSBuffer: Model<TTSBufferSchemaType> =
models[collectionName] || model(collectionName, TTSBufferSchema);
MongoTTSBuffer.syncIndexes();
export const MongoTTSBuffer = getMongoModel<TTSBufferSchemaType>(collectionName, TTSBufferSchema);

6
packages/service/common/file/gridfs/schema.ts
View File

@@ -1,4 +1,4 @@
import { connectionMongo, type Model } from '../../mongo';
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
const { Schema, model, models } = connectionMongo;
const FileSchema = new Schema({});
@@ -10,6 +10,4 @@ try {
console.log(error);
}
export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema);
MongoFileSchema.syncIndexes();
export const MongoFileSchema = getMongoModel('dataset.files', FileSchema);

7
packages/service/common/file/image/schema.ts
View File

@@ -1,5 +1,5 @@
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
import { connectionMongo, type Model } from '../../mongo';
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d';
import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants';
const { Schema, model, models } = connectionMongo;
@@ -41,7 +41,4 @@ try {
console.log(error);
}
export const MongoImage: Model<MongoImageSchemaType> =
models['image'] || model('image', ImageSchema);
MongoImage.syncIndexes();
export const MongoImage = getMongoModel<MongoImageSchemaType>('image', ImageSchema);

72
packages/service/common/mongo/index.ts
View File

@@ -1,7 +1,5 @@
import mongoose from 'mongoose';
export default mongoose;
export * from 'mongoose';
import { addLog } from '../../common/system/log';
import mongoose, { Model } from 'mongoose';
export const connectionMongo = (() => {
if (!global.mongodb) {
@@ -11,4 +9,68 @@ export const connectionMongo = (() => {
return global.mongodb;
})();
export const ReadPreference = mongoose.mongo.ReadPreference;
export default mongoose;
export * from 'mongoose';
const addCommonMiddleware = (schema: mongoose.Schema) => {
const operations = [
/^find/,
'save',
'create',
/^update/,
/^delete/,
'aggregate',
'count',
'countDocuments',
'estimatedDocumentCount',
'distinct',
'insertMany'
];
operations.forEach((op: any) => {
schema.pre(op, function (this: any, next) {
this._startTime = Date.now();
this._query = this.getQuery ? this.getQuery() : null;
next();
});
schema.post(op, function (this: any, result: any, next) {
if (this._startTime) {
const duration = Date.now() - this._startTime;
const warnLogData = {
query: this._query,
op,
duration
};
if (duration > 1000) {
addLog.warn(`Slow operation ${duration}ms`, warnLogData);
} else if (duration > 300) {
addLog.error(`Slow operation ${duration}ms`, warnLogData);
}
}
next();
});
});
return schema;
};
export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
if (connectionMongo.models[name]) return connectionMongo.models[name] as Model<T>;
console.log('Load model======', name);
addCommonMiddleware(schema);
const model = connectionMongo.model<T>(name, schema);
try {
model.syncIndexes();
} catch (error) {
addLog.error('Create index error', error);
}
return model;
};
export const ReadPreference = connectionMongo.mongo.ReadPreference;

9
packages/service/common/system/config/schema.ts
View File

@@ -1,5 +1,5 @@
import { SystemConfigsType } from '@fastgpt/global/common/system/config/type';
import { connectionMongo, type Model } from '../../../common/mongo';
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
import { SystemConfigsTypeMap } from '@fastgpt/global/common/system/config/constants';
const { Schema, model, models } = connectionMongo;
@@ -27,6 +27,7 @@ try {
console.log(error);
}
export const MongoSystemConfigs: Model<SystemConfigsType> =
models[collectionName] || model(collectionName, systemConfigSchema);
MongoSystemConfigs.syncIndexes();
export const MongoSystemConfigs = getMongoModel<SystemConfigsType>(
collectionName,
systemConfigSchema
);

71
packages/service/common/system/log.ts
View File

@@ -1,13 +1,9 @@
import dayjs from 'dayjs';
import chalk from 'chalk';
import { isProduction } from './constants';
import { LogLevelEnum } from './log/constant';
// import { MongoLog } from './log/schema';
import connectionMongo from '../mongo/index';
enum LogLevelEnum {
debug = 0,
info = 1,
warn = 2,
error = 3
}
const logMap = {
[LogLevelEnum.debug]: {
levelLog: chalk.green('[Debug]')
@@ -23,23 +19,26 @@ const logMap = {
}
};
const envLogLevelMap: Record<string, number> = {
debug: 0,
info: 1,
warn: 2,
error: 3
debug: LogLevelEnum.debug,
info: LogLevelEnum.info,
warn: LogLevelEnum.warn,
error: LogLevelEnum.error
};
const logLevel = (() => {
if (!isProduction) return LogLevelEnum.debug;
const envLogLevel = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase();
if (!envLogLevel || envLogLevelMap[envLogLevel] === undefined) return LogLevelEnum.info;
return envLogLevelMap[envLogLevel];
const { LOG_LEVEL, STORE_LOG_LEVEL } = (() => {
const LOG_LEVEL = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase();
const STORE_LOG_LEVEL = (process.env.STORE_LOG_LEVEL || '').toLocaleLowerCase();
return {
LOG_LEVEL: envLogLevelMap[LOG_LEVEL] || LogLevelEnum.info,
STORE_LOG_LEVEL: envLogLevelMap[STORE_LOG_LEVEL] ?? 99
};
})();
/* add logger */
export const addLog = {
log(level: LogLevelEnum, msg: string, obj: Record<string, any> = {}) {
if (level < logLevel) return;
if (level < LOG_LEVEL) return;
const stringifyObj = JSON.stringify(obj);
const isEmpty = Object.keys(obj).length === 0;
@@ -52,35 +51,15 @@ export const addLog = {
level === LogLevelEnum.error && console.error(obj);
const lokiUrl = process.env.LOKI_LOG_URL as string;
if (!lokiUrl) return;
try {
fetch(lokiUrl, {
method: 'POST',
headers: {
'Content-type': 'application/json'
},
body: JSON.stringify({
streams: [
{
stream: {
level
},
values: [
[
`${Date.now() * 1000000}`,
JSON.stringify({
message: msg,
...obj
})
]
]
}
]
})
});
} catch (error) {}
// store
// if (level >= STORE_LOG_LEVEL && connectionMongo.connection.readyState === 1) {
// // store log
// MongoLog.create({
// text: msg,
// level,
// metadata: obj
// });
// }
},
debug(msg: string, obj?: Record<string, any>) {
this.log(LogLevelEnum.debug, msg, obj);

10
packages/service/common/system/log/constant.ts Normal file
View File

@@ -0,0 +1,10 @@
export enum LogLevelEnum {
debug = 0,
info = 1,
warn = 2,
error = 3
}
export enum LogSignEnum {
slowOperation = 'slowOperation'
}

27
packages/service/common/system/log/schema.ts Normal file
View File

@@ -0,0 +1,27 @@
import { getMongoModel, Schema } from '../../../common/mongo';
import { SystemLogType } from './type';
import { LogLevelEnum } from './constant';
export const LogCollectionName = 'system_logs';
const SystemLogSchema = new Schema({
text: {
type: String,
required: true
},
level: {
type: String,
required: true,
enum: Object.values(LogLevelEnum)
},
time: {
type: Date,
default: () => new Date()
},
metadata: Object
});
SystemLogSchema.index({ time: 1 }, { expires: '15d' });
SystemLogSchema.index({ level: 1 });
export const MongoLog = getMongoModel<SystemLogType>(LogCollectionName, SystemLogSchema);

9
packages/service/common/system/log/type.d.ts vendored Normal file
View File

@@ -0,0 +1,9 @@
import { LogLevelEnum, LogSignEnum } from './constant';
export type SystemLogType = {
_id: string;
text: string;
level: LogLevelEnum;
time: Date;
metadata?: Record<string, any>;
};

6
packages/service/common/system/timerLock/schema.ts
View File

@@ -1,4 +1,4 @@
import { connectionMongo, type Model } from '../../mongo';
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
import { timerIdMap } from './constants';
const { Schema, model, models } = connectionMongo;
import { TimerLockSchemaType } from './type.d';
@@ -24,6 +24,4 @@ try {
console.log(error);
}
export const MongoTimerLock: Model<TimerLockSchemaType> =
models[collectionName] || model(collectionName, TimerLockSchema);
MongoTimerLock.syncIndexes();
export const MongoTimerLock = getMongoModel<TimerLockSchemaType>(collectionName, TimerLockSchema);