mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-22 20:37:48 +00:00
4.8.6 fix (#1963)
* feat: log store * fix: full text search match query * perf: mongo schema import, Avoid duplicate import
This commit is contained in:
5
.vscode/extensions.json
vendored
5
.vscode/extensions.json
vendored
@@ -1,5 +0,0 @@
|
|||||||
{
|
|
||||||
"recommendations": [
|
|
||||||
"inlang.vs-code-extension"
|
|
||||||
]
|
|
||||||
}
|
|
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
@@ -2,7 +2,7 @@
|
|||||||
"editor.formatOnSave": true,
|
"editor.formatOnSave": true,
|
||||||
"editor.mouseWheelZoom": true,
|
"editor.mouseWheelZoom": true,
|
||||||
"editor.defaultFormatter": "esbenp.prettier-vscode",
|
"editor.defaultFormatter": "esbenp.prettier-vscode",
|
||||||
"prettier.prettierPath": "../node_modules/prettier",
|
"prettier.prettierPath": "node_modules/prettier",
|
||||||
"typescript.tsdk": "node_modules/typescript/lib",
|
"typescript.tsdk": "node_modules/typescript/lib",
|
||||||
"i18n-ally.localesPaths": [
|
"i18n-ally.localesPaths": [
|
||||||
"packages/web/i18n",
|
"packages/web/i18n",
|
||||||
|
@@ -156,6 +156,7 @@ services:
|
|||||||
- SANDBOX_URL=http://sandbox:3000
|
- SANDBOX_URL=http://sandbox:3000
|
||||||
# 日志等级: debug, info, warn, error
|
# 日志等级: debug, info, warn, error
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
|
- STORE_LOG_LEVEL=warn
|
||||||
volumes:
|
volumes:
|
||||||
- ./config.json:/app/data/config.json
|
- ./config.json:/app/data/config.json
|
||||||
|
|
||||||
|
@@ -113,6 +113,7 @@ services:
|
|||||||
- SANDBOX_URL=http://sandbox:3000
|
- SANDBOX_URL=http://sandbox:3000
|
||||||
# 日志等级: debug, info, warn, error
|
# 日志等级: debug, info, warn, error
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
|
- STORE_LOG_LEVEL=warn
|
||||||
volumes:
|
volumes:
|
||||||
- ./config.json:/app/data/config.json
|
- ./config.json:/app/data/config.json
|
||||||
|
|
||||||
|
@@ -94,6 +94,7 @@ services:
|
|||||||
- SANDBOX_URL=http://sandbox:3000
|
- SANDBOX_URL=http://sandbox:3000
|
||||||
# 日志等级: debug, info, warn, error
|
# 日志等级: debug, info, warn, error
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=info
|
||||||
|
- STORE_LOG_LEVEL=warn
|
||||||
volumes:
|
volumes:
|
||||||
- ./config.json:/app/data/config.json
|
- ./config.json:/app/data/config.json
|
||||||
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { RawTextBufferSchemaType } from './type';
|
import { RawTextBufferSchemaType } from './type';
|
||||||
|
|
||||||
@@ -28,6 +28,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoRawTextBuffer: Model<RawTextBufferSchemaType> =
|
export const MongoRawTextBuffer = getMongoModel<RawTextBufferSchemaType>(
|
||||||
models[collectionName] || model(collectionName, RawTextBufferSchema);
|
collectionName,
|
||||||
MongoRawTextBuffer.syncIndexes();
|
RawTextBufferSchema
|
||||||
|
);
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { TTSBufferSchemaType } from './type.d';
|
import { TTSBufferSchemaType } from './type.d';
|
||||||
|
|
||||||
@@ -31,6 +31,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoTTSBuffer: Model<TTSBufferSchemaType> =
|
export const MongoTTSBuffer = getMongoModel<TTSBufferSchemaType>(collectionName, TTSBufferSchema);
|
||||||
models[collectionName] || model(collectionName, TTSBufferSchema);
|
|
||||||
MongoTTSBuffer.syncIndexes();
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
|
|
||||||
const FileSchema = new Schema({});
|
const FileSchema = new Schema({});
|
||||||
@@ -10,6 +10,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema);
|
export const MongoFileSchema = getMongoModel('dataset.files', FileSchema);
|
||||||
|
|
||||||
MongoFileSchema.syncIndexes();
|
|
||||||
|
@@ -1,5 +1,5 @@
|
|||||||
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
|
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
|
||||||
import { connectionMongo, type Model } from '../../mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
|
||||||
import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d';
|
import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d';
|
||||||
import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants';
|
import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
@@ -41,7 +41,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoImage: Model<MongoImageSchemaType> =
|
export const MongoImage = getMongoModel<MongoImageSchemaType>('image', ImageSchema);
|
||||||
models['image'] || model('image', ImageSchema);
|
|
||||||
|
|
||||||
MongoImage.syncIndexes();
|
|
||||||
|
@@ -1,7 +1,5 @@
|
|||||||
import mongoose from 'mongoose';
|
import { addLog } from '../../common/system/log';
|
||||||
|
import mongoose, { Model } from 'mongoose';
|
||||||
export default mongoose;
|
|
||||||
export * from 'mongoose';
|
|
||||||
|
|
||||||
export const connectionMongo = (() => {
|
export const connectionMongo = (() => {
|
||||||
if (!global.mongodb) {
|
if (!global.mongodb) {
|
||||||
@@ -11,4 +9,68 @@ export const connectionMongo = (() => {
|
|||||||
return global.mongodb;
|
return global.mongodb;
|
||||||
})();
|
})();
|
||||||
|
|
||||||
export const ReadPreference = mongoose.mongo.ReadPreference;
|
export default mongoose;
|
||||||
|
export * from 'mongoose';
|
||||||
|
|
||||||
|
const addCommonMiddleware = (schema: mongoose.Schema) => {
|
||||||
|
const operations = [
|
||||||
|
/^find/,
|
||||||
|
'save',
|
||||||
|
'create',
|
||||||
|
/^update/,
|
||||||
|
/^delete/,
|
||||||
|
'aggregate',
|
||||||
|
'count',
|
||||||
|
'countDocuments',
|
||||||
|
'estimatedDocumentCount',
|
||||||
|
'distinct',
|
||||||
|
'insertMany'
|
||||||
|
];
|
||||||
|
|
||||||
|
operations.forEach((op: any) => {
|
||||||
|
schema.pre(op, function (this: any, next) {
|
||||||
|
this._startTime = Date.now();
|
||||||
|
this._query = this.getQuery ? this.getQuery() : null;
|
||||||
|
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
|
||||||
|
schema.post(op, function (this: any, result: any, next) {
|
||||||
|
if (this._startTime) {
|
||||||
|
const duration = Date.now() - this._startTime;
|
||||||
|
|
||||||
|
const warnLogData = {
|
||||||
|
query: this._query,
|
||||||
|
op,
|
||||||
|
duration
|
||||||
|
};
|
||||||
|
|
||||||
|
if (duration > 1000) {
|
||||||
|
addLog.warn(`Slow operation ${duration}ms`, warnLogData);
|
||||||
|
} else if (duration > 300) {
|
||||||
|
addLog.error(`Slow operation ${duration}ms`, warnLogData);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return schema;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
|
||||||
|
if (connectionMongo.models[name]) return connectionMongo.models[name] as Model<T>;
|
||||||
|
console.log('Load model======', name);
|
||||||
|
addCommonMiddleware(schema);
|
||||||
|
|
||||||
|
const model = connectionMongo.model<T>(name, schema);
|
||||||
|
try {
|
||||||
|
model.syncIndexes();
|
||||||
|
} catch (error) {
|
||||||
|
addLog.error('Create index error', error);
|
||||||
|
}
|
||||||
|
|
||||||
|
return model;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const ReadPreference = connectionMongo.mongo.ReadPreference;
|
||||||
|
@@ -1,5 +1,5 @@
|
|||||||
import { SystemConfigsType } from '@fastgpt/global/common/system/config/type';
|
import { SystemConfigsType } from '@fastgpt/global/common/system/config/type';
|
||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
import { SystemConfigsTypeMap } from '@fastgpt/global/common/system/config/constants';
|
import { SystemConfigsTypeMap } from '@fastgpt/global/common/system/config/constants';
|
||||||
|
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
@@ -27,6 +27,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoSystemConfigs: Model<SystemConfigsType> =
|
export const MongoSystemConfigs = getMongoModel<SystemConfigsType>(
|
||||||
models[collectionName] || model(collectionName, systemConfigSchema);
|
collectionName,
|
||||||
MongoSystemConfigs.syncIndexes();
|
systemConfigSchema
|
||||||
|
);
|
||||||
|
@@ -1,13 +1,9 @@
|
|||||||
import dayjs from 'dayjs';
|
import dayjs from 'dayjs';
|
||||||
import chalk from 'chalk';
|
import chalk from 'chalk';
|
||||||
import { isProduction } from './constants';
|
import { LogLevelEnum } from './log/constant';
|
||||||
|
// import { MongoLog } from './log/schema';
|
||||||
|
import connectionMongo from '../mongo/index';
|
||||||
|
|
||||||
enum LogLevelEnum {
|
|
||||||
debug = 0,
|
|
||||||
info = 1,
|
|
||||||
warn = 2,
|
|
||||||
error = 3
|
|
||||||
}
|
|
||||||
const logMap = {
|
const logMap = {
|
||||||
[LogLevelEnum.debug]: {
|
[LogLevelEnum.debug]: {
|
||||||
levelLog: chalk.green('[Debug]')
|
levelLog: chalk.green('[Debug]')
|
||||||
@@ -23,23 +19,26 @@ const logMap = {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
const envLogLevelMap: Record<string, number> = {
|
const envLogLevelMap: Record<string, number> = {
|
||||||
debug: 0,
|
debug: LogLevelEnum.debug,
|
||||||
info: 1,
|
info: LogLevelEnum.info,
|
||||||
warn: 2,
|
warn: LogLevelEnum.warn,
|
||||||
error: 3
|
error: LogLevelEnum.error
|
||||||
};
|
};
|
||||||
|
|
||||||
const logLevel = (() => {
|
const { LOG_LEVEL, STORE_LOG_LEVEL } = (() => {
|
||||||
if (!isProduction) return LogLevelEnum.debug;
|
const LOG_LEVEL = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase();
|
||||||
const envLogLevel = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase();
|
const STORE_LOG_LEVEL = (process.env.STORE_LOG_LEVEL || '').toLocaleLowerCase();
|
||||||
if (!envLogLevel || envLogLevelMap[envLogLevel] === undefined) return LogLevelEnum.info;
|
|
||||||
return envLogLevelMap[envLogLevel];
|
return {
|
||||||
|
LOG_LEVEL: envLogLevelMap[LOG_LEVEL] || LogLevelEnum.info,
|
||||||
|
STORE_LOG_LEVEL: envLogLevelMap[STORE_LOG_LEVEL] ?? 99
|
||||||
|
};
|
||||||
})();
|
})();
|
||||||
|
|
||||||
/* add logger */
|
/* add logger */
|
||||||
export const addLog = {
|
export const addLog = {
|
||||||
log(level: LogLevelEnum, msg: string, obj: Record<string, any> = {}) {
|
log(level: LogLevelEnum, msg: string, obj: Record<string, any> = {}) {
|
||||||
if (level < logLevel) return;
|
if (level < LOG_LEVEL) return;
|
||||||
|
|
||||||
const stringifyObj = JSON.stringify(obj);
|
const stringifyObj = JSON.stringify(obj);
|
||||||
const isEmpty = Object.keys(obj).length === 0;
|
const isEmpty = Object.keys(obj).length === 0;
|
||||||
@@ -52,35 +51,15 @@ export const addLog = {
|
|||||||
|
|
||||||
level === LogLevelEnum.error && console.error(obj);
|
level === LogLevelEnum.error && console.error(obj);
|
||||||
|
|
||||||
const lokiUrl = process.env.LOKI_LOG_URL as string;
|
// store
|
||||||
if (!lokiUrl) return;
|
// if (level >= STORE_LOG_LEVEL && connectionMongo.connection.readyState === 1) {
|
||||||
|
// // store log
|
||||||
try {
|
// MongoLog.create({
|
||||||
fetch(lokiUrl, {
|
// text: msg,
|
||||||
method: 'POST',
|
// level,
|
||||||
headers: {
|
// metadata: obj
|
||||||
'Content-type': 'application/json'
|
// });
|
||||||
},
|
// }
|
||||||
body: JSON.stringify({
|
|
||||||
streams: [
|
|
||||||
{
|
|
||||||
stream: {
|
|
||||||
level
|
|
||||||
},
|
|
||||||
values: [
|
|
||||||
[
|
|
||||||
`${Date.now() * 1000000}`,
|
|
||||||
JSON.stringify({
|
|
||||||
message: msg,
|
|
||||||
...obj
|
|
||||||
})
|
|
||||||
]
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
})
|
|
||||||
});
|
|
||||||
} catch (error) {}
|
|
||||||
},
|
},
|
||||||
debug(msg: string, obj?: Record<string, any>) {
|
debug(msg: string, obj?: Record<string, any>) {
|
||||||
this.log(LogLevelEnum.debug, msg, obj);
|
this.log(LogLevelEnum.debug, msg, obj);
|
||||||
|
10
packages/service/common/system/log/constant.ts
Normal file
10
packages/service/common/system/log/constant.ts
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
export enum LogLevelEnum {
|
||||||
|
debug = 0,
|
||||||
|
info = 1,
|
||||||
|
warn = 2,
|
||||||
|
error = 3
|
||||||
|
}
|
||||||
|
|
||||||
|
export enum LogSignEnum {
|
||||||
|
slowOperation = 'slowOperation'
|
||||||
|
}
|
27
packages/service/common/system/log/schema.ts
Normal file
27
packages/service/common/system/log/schema.ts
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
import { getMongoModel, Schema } from '../../../common/mongo';
|
||||||
|
import { SystemLogType } from './type';
|
||||||
|
import { LogLevelEnum } from './constant';
|
||||||
|
|
||||||
|
export const LogCollectionName = 'system_logs';
|
||||||
|
|
||||||
|
const SystemLogSchema = new Schema({
|
||||||
|
text: {
|
||||||
|
type: String,
|
||||||
|
required: true
|
||||||
|
},
|
||||||
|
level: {
|
||||||
|
type: String,
|
||||||
|
required: true,
|
||||||
|
enum: Object.values(LogLevelEnum)
|
||||||
|
},
|
||||||
|
time: {
|
||||||
|
type: Date,
|
||||||
|
default: () => new Date()
|
||||||
|
},
|
||||||
|
metadata: Object
|
||||||
|
});
|
||||||
|
|
||||||
|
SystemLogSchema.index({ time: 1 }, { expires: '15d' });
|
||||||
|
SystemLogSchema.index({ level: 1 });
|
||||||
|
|
||||||
|
export const MongoLog = getMongoModel<SystemLogType>(LogCollectionName, SystemLogSchema);
|
9
packages/service/common/system/log/type.d.ts
vendored
Normal file
9
packages/service/common/system/log/type.d.ts
vendored
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
import { LogLevelEnum, LogSignEnum } from './constant';
|
||||||
|
|
||||||
|
export type SystemLogType = {
|
||||||
|
_id: string;
|
||||||
|
text: string;
|
||||||
|
level: LogLevelEnum;
|
||||||
|
time: Date;
|
||||||
|
metadata?: Record<string, any>;
|
||||||
|
};
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../mongo';
|
||||||
import { timerIdMap } from './constants';
|
import { timerIdMap } from './constants';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { TimerLockSchemaType } from './type.d';
|
import { TimerLockSchemaType } from './type.d';
|
||||||
@@ -24,6 +24,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoTimerLock: Model<TimerLockSchemaType> =
|
export const MongoTimerLock = getMongoModel<TimerLockSchemaType>(collectionName, TimerLockSchema);
|
||||||
models[collectionName] || model(collectionName, TimerLockSchema);
|
|
||||||
MongoTimerLock.syncIndexes();
|
|
||||||
|
@@ -1,6 +1,5 @@
|
|||||||
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
|
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
|
||||||
import { connectionMongo, type Model } from '../../common/mongo';
|
import { Schema, getMongoModel } from '../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
|
||||||
import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
|
import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
|
||||||
import {
|
import {
|
||||||
TeamCollectionName,
|
TeamCollectionName,
|
||||||
@@ -21,6 +20,7 @@ export const chatConfigType = {
|
|||||||
chatInputGuide: Object
|
chatInputGuide: Object
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// schema
|
||||||
const AppSchema = new Schema({
|
const AppSchema = new Schema({
|
||||||
parentId: {
|
parentId: {
|
||||||
type: Schema.Types.ObjectId,
|
type: Schema.Types.ObjectId,
|
||||||
@@ -112,15 +112,8 @@ const AppSchema = new Schema({
|
|||||||
...getPermissionSchema(AppDefaultPermissionVal)
|
...getPermissionSchema(AppDefaultPermissionVal)
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
|
||||||
AppSchema.index({ updateTime: -1 });
|
AppSchema.index({ updateTime: -1 });
|
||||||
AppSchema.index({ teamId: 1, type: 1 });
|
AppSchema.index({ teamId: 1, type: 1 });
|
||||||
AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
|
AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
|
||||||
} catch (error) {
|
|
||||||
console.log(error);
|
|
||||||
}
|
|
||||||
|
|
||||||
export const MongoApp: Model<AppType> =
|
export const MongoApp = getMongoModel<AppType>(AppCollectionName, AppSchema);
|
||||||
models[AppCollectionName] || model(AppCollectionName, AppSchema);
|
|
||||||
|
|
||||||
MongoApp.syncIndexes();
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
|
import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
|
||||||
import { chatConfigType } from '../schema';
|
import { chatConfigType } from '../schema';
|
||||||
@@ -34,7 +34,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoAppVersion: Model<AppVersionSchemaType> =
|
export const MongoAppVersion = getMongoModel<AppVersionSchemaType>(
|
||||||
models[AppVersionCollectionName] || model(AppVersionCollectionName, AppVersionSchema);
|
AppVersionCollectionName,
|
||||||
|
AppVersionSchema
|
||||||
MongoAppVersion.syncIndexes();
|
);
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type';
|
import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type';
|
||||||
import { ChatRoleMap } from '@fastgpt/global/core/chat/constants';
|
import { ChatRoleMap } from '@fastgpt/global/core/chat/constants';
|
||||||
@@ -9,7 +9,6 @@ import {
|
|||||||
} from '@fastgpt/global/support/user/team/constant';
|
} from '@fastgpt/global/support/user/team/constant';
|
||||||
import { AppCollectionName } from '../app/schema';
|
import { AppCollectionName } from '../app/schema';
|
||||||
import { userCollectionName } from '../../support/user/schema';
|
import { userCollectionName } from '../../support/user/schema';
|
||||||
import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
|
|
||||||
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
|
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
|
||||||
|
|
||||||
export const ChatItemCollectionName = 'chatitems';
|
export const ChatItemCollectionName = 'chatitems';
|
||||||
@@ -99,7 +98,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoChatItem: Model<ChatItemType> =
|
export const MongoChatItem = getMongoModel<ChatItemType>(ChatItemCollectionName, ChatItemSchema);
|
||||||
models[ChatItemCollectionName] || model(ChatItemCollectionName, ChatItemSchema);
|
|
||||||
|
|
||||||
MongoChatItem.syncIndexes();
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
|
import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
|
||||||
import { ChatSourceMap } from '@fastgpt/global/core/chat/constants';
|
import { ChatSourceMap } from '@fastgpt/global/core/chat/constants';
|
||||||
@@ -98,6 +98,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoChat: Model<ChatType> =
|
export const MongoChat = getMongoModel<ChatType>(chatCollectionName, ChatSchema);
|
||||||
models[chatCollectionName] || model(chatCollectionName, ChatSchema);
|
|
||||||
MongoChat.syncIndexes();
|
|
||||||
|
@@ -1,5 +1,5 @@
|
|||||||
import { AppCollectionName } from '../../app/schema';
|
import { AppCollectionName } from '../../app/schema';
|
||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d';
|
import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d';
|
||||||
|
|
||||||
@@ -23,7 +23,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoChatInputGuide: Model<ChatInputGuideSchemaType> =
|
export const MongoChatInputGuide = getMongoModel<ChatInputGuideSchemaType>(
|
||||||
models[ChatInputGuideCollectionName] || model(ChatInputGuideCollectionName, ChatInputGuideSchema);
|
ChatInputGuideCollectionName,
|
||||||
|
ChatInputGuideSchema
|
||||||
MongoChatInputGuide.syncIndexes();
|
);
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||||
import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants';
|
import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants';
|
||||||
@@ -94,9 +94,6 @@ const DatasetCollectionSchema = new Schema({
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
export const MongoDatasetCollection: Model<DatasetCollectionSchemaType> =
|
|
||||||
models[DatasetColCollectionName] || model(DatasetColCollectionName, DatasetCollectionSchema);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// auth file
|
// auth file
|
||||||
DatasetCollectionSchema.index({ teamId: 1, fileId: 1 });
|
DatasetCollectionSchema.index({ teamId: 1, fileId: 1 });
|
||||||
@@ -111,8 +108,11 @@ try {
|
|||||||
|
|
||||||
// get forbid
|
// get forbid
|
||||||
// DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 });
|
// DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 });
|
||||||
|
|
||||||
MongoDatasetCollection.syncIndexes({ background: true });
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const MongoDatasetCollection = getMongoModel<DatasetCollectionSchemaType>(
|
||||||
|
DatasetColCollectionName,
|
||||||
|
DatasetCollectionSchema
|
||||||
|
);
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||||
import {
|
import {
|
||||||
@@ -77,10 +77,6 @@ const DatasetDataSchema = new Schema({
|
|||||||
rebuilding: Boolean
|
rebuilding: Boolean
|
||||||
});
|
});
|
||||||
|
|
||||||
export const MongoDatasetData: Model<DatasetDataSchemaType> =
|
|
||||||
models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
|
|
||||||
|
|
||||||
try {
|
|
||||||
// list collection and count data; list data; delete collection(relate data)
|
// list collection and count data; list data; delete collection(relate data)
|
||||||
DatasetDataSchema.index({
|
DatasetDataSchema.index({
|
||||||
teamId: 1,
|
teamId: 1,
|
||||||
@@ -97,7 +93,7 @@ try {
|
|||||||
// rebuild data
|
// rebuild data
|
||||||
DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
|
DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
|
||||||
|
|
||||||
MongoDatasetData.syncIndexes({ background: true });
|
export const MongoDatasetData = getMongoModel<DatasetDataSchemaType>(
|
||||||
} catch (error) {
|
DatasetDataCollectionName,
|
||||||
console.log(error);
|
DatasetDataSchema
|
||||||
}
|
);
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
|
||||||
import {
|
import {
|
||||||
@@ -11,7 +11,6 @@ import {
|
|||||||
TeamCollectionName,
|
TeamCollectionName,
|
||||||
TeamMemberCollectionName
|
TeamMemberCollectionName
|
||||||
} from '@fastgpt/global/support/user/team/constant';
|
} from '@fastgpt/global/support/user/team/constant';
|
||||||
import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
|
|
||||||
import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
|
import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
|
||||||
|
|
||||||
export const DatasetCollectionName = 'datasets';
|
export const DatasetCollectionName = 'datasets';
|
||||||
@@ -99,6 +98,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoDataset: Model<DatasetSchemaType> =
|
export const MongoDataset = getMongoModel<DatasetSchemaType>(DatasetCollectionName, DatasetSchema);
|
||||||
models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);
|
|
||||||
MongoDataset.syncIndexes();
|
|
||||||
|
@@ -212,7 +212,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
|
|||||||
{
|
{
|
||||||
$match: {
|
$match: {
|
||||||
$expr: { $eq: ['$_id', '$$collectionId'] },
|
$expr: { $eq: ['$_id', '$$collectionId'] },
|
||||||
forbid: { $eq: false } // 直接在lookup阶段过滤
|
forbid: { $eq: true } // 匹配被禁用的数据
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -226,7 +226,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
$match: {
|
$match: {
|
||||||
collection: { $ne: [] }
|
collection: { $eq: [] } // 没有 forbid=true 的数据
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@@ -1,5 +1,5 @@
|
|||||||
/* 模型的知识库 */
|
/* 模型的知识库 */
|
||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||||
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
|
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
|
||||||
@@ -103,7 +103,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> =
|
export const MongoDatasetTraining = getMongoModel<DatasetTrainingSchemaType>(
|
||||||
models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema);
|
DatasetTrainingCollectionName,
|
||||||
|
TrainingDataSchema
|
||||||
MongoDatasetTraining.syncIndexes();
|
);
|
||||||
|
@@ -198,7 +198,7 @@ ${description ? `- ${description}` : ''}
|
|||||||
required: []
|
required: []
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
console.log(properties);
|
|
||||||
return {
|
return {
|
||||||
filterMessages,
|
filterMessages,
|
||||||
agentFunction
|
agentFunction
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { PromotionRecordSchema as PromotionRecordType } from '@fastgpt/global/support/activity/type.d';
|
import { PromotionRecordSchema as PromotionRecordType } from '@fastgpt/global/support/activity/type.d';
|
||||||
|
|
||||||
@@ -29,6 +29,7 @@ const PromotionRecordSchema = new Schema({
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
export const MongoPromotionRecord: Model<PromotionRecordType> =
|
export const MongoPromotionRecord = getMongoModel<PromotionRecordType>(
|
||||||
models['promotionRecord'] || model('promotionRecord', PromotionRecordSchema);
|
'promotionRecord',
|
||||||
MongoPromotionRecord.syncIndexes();
|
PromotionRecordSchema
|
||||||
|
);
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
|
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
|
||||||
import {
|
import {
|
||||||
@@ -64,6 +64,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoOpenApi: Model<OpenApiSchema> =
|
export const MongoOpenApi = getMongoModel<OpenApiSchema>('openapi', OpenApiSchema);
|
||||||
models['openapi'] || model('openapi', OpenApiSchema);
|
|
||||||
MongoOpenApi.syncIndexes();
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type';
|
import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type';
|
||||||
import {
|
import {
|
||||||
@@ -90,7 +90,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoOutLink: Model<SchemaType> =
|
export const MongoOutLink = getMongoModel<SchemaType>('outlinks', OutLinkSchema);
|
||||||
models['outlinks'] || model('outlinks', OutLinkSchema);
|
|
||||||
|
|
||||||
MongoOutLink.syncIndexes();
|
|
||||||
|
@@ -2,7 +2,7 @@ import {
|
|||||||
TeamCollectionName,
|
TeamCollectionName,
|
||||||
TeamMemberCollectionName
|
TeamMemberCollectionName
|
||||||
} from '@fastgpt/global/support/user/team/constant';
|
} from '@fastgpt/global/support/user/team/constant';
|
||||||
import { Model, connectionMongo } from '../../common/mongo';
|
import { Model, connectionMongo, getMongoModel } from '../../common/mongo';
|
||||||
import type { ResourcePermissionType } from '@fastgpt/global/support/permission/type';
|
import type { ResourcePermissionType } from '@fastgpt/global/support/permission/type';
|
||||||
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
|
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
@@ -54,8 +54,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoResourcePermission: Model<ResourcePermissionType> =
|
export const MongoResourcePermission = getMongoModel<ResourcePermissionType>(
|
||||||
models[ResourcePermissionCollectionName] ||
|
ResourcePermissionCollectionName,
|
||||||
model(ResourcePermissionCollectionName, ResourcePermissionSchema);
|
ResourcePermissionSchema
|
||||||
|
);
|
||||||
MongoResourcePermission.syncIndexes();
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { hashStr } from '@fastgpt/global/common/string/tools';
|
import { hashStr } from '@fastgpt/global/common/string/tools';
|
||||||
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
|
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
|
||||||
@@ -74,6 +74,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoUser: Model<UserModelSchema> =
|
export const MongoUser = getMongoModel<UserModelSchema>(userCollectionName, UserSchema);
|
||||||
models[userCollectionName] || model(userCollectionName, UserSchema);
|
|
||||||
MongoUser.syncIndexes();
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { TeamMemberSchema as TeamMemberType } from '@fastgpt/global/support/user/team/type.d';
|
import { TeamMemberSchema as TeamMemberType } from '@fastgpt/global/support/user/team/type.d';
|
||||||
import { userCollectionName } from '../../user/schema';
|
import { userCollectionName } from '../../user/schema';
|
||||||
@@ -49,5 +49,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoTeamMember: Model<TeamMemberType> =
|
export const MongoTeamMember = getMongoModel<TeamMemberType>(
|
||||||
models[TeamMemberCollectionName] || model(TeamMemberCollectionName, TeamMemberSchema);
|
TeamMemberCollectionName,
|
||||||
|
TeamMemberSchema
|
||||||
|
);
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { TeamSchema as TeamType } from '@fastgpt/global/support/user/team/type.d';
|
import { TeamSchema as TeamType } from '@fastgpt/global/support/user/team/type.d';
|
||||||
import { userCollectionName } from '../../user/schema';
|
import { userCollectionName } from '../../user/schema';
|
||||||
@@ -61,5 +61,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoTeam: Model<TeamType> =
|
export const MongoTeam = getMongoModel<TeamType>(TeamCollectionName, TeamSchema);
|
||||||
models[TeamCollectionName] || model(TeamCollectionName, TeamSchema);
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
|
import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
|
||||||
import {
|
import {
|
||||||
@@ -32,5 +32,7 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoTeamTags: Model<TeamTagsSchemaType> =
|
export const MongoTeamTags = getMongoModel<TeamTagsSchemaType>(
|
||||||
models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagSchema);
|
TeamTagsCollectionName,
|
||||||
|
TeamTagSchema
|
||||||
|
);
|
||||||
|
@@ -3,7 +3,7 @@
|
|||||||
1. type=standard: There will only be 1, and each team will have one
|
1. type=standard: There will only be 1, and each team will have one
|
||||||
2. type=extraDatasetSize/extraPoints: Can buy multiple
|
2. type=extraDatasetSize/extraPoints: Can buy multiple
|
||||||
*/
|
*/
|
||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
|
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
|
||||||
import {
|
import {
|
||||||
@@ -93,5 +93,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoTeamSub: Model<TeamSubSchema> =
|
export const MongoTeamSub = getMongoModel<TeamSubSchema>(subCollectionName, SubSchema);
|
||||||
models[subCollectionName] || model(subCollectionName, SubSchema);
|
|
||||||
|
@@ -1,4 +1,4 @@
|
|||||||
import { connectionMongo, type Model } from '../../../common/mongo';
|
import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
|
||||||
const { Schema, model, models } = connectionMongo;
|
const { Schema, model, models } = connectionMongo;
|
||||||
import { UsageSchemaType } from '@fastgpt/global/support/wallet/usage/type';
|
import { UsageSchemaType } from '@fastgpt/global/support/wallet/usage/type';
|
||||||
import { UsageSourceMap } from '@fastgpt/global/support/wallet/usage/constants';
|
import { UsageSourceMap } from '@fastgpt/global/support/wallet/usage/constants';
|
||||||
@@ -70,6 +70,4 @@ try {
|
|||||||
console.log(error);
|
console.log(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
export const MongoUsage: Model<UsageSchemaType> =
|
export const MongoUsage = getMongoModel<UsageSchemaType>(UsageCollectionName, UsageSchema);
|
||||||
models[UsageCollectionName] || model(UsageCollectionName, UsageSchema);
|
|
||||||
MongoUsage.syncIndexes();
|
|
||||||
|
@@ -33,6 +33,6 @@ PRO_URL=
|
|||||||
# 首页路径
|
# 首页路径
|
||||||
HOME_URL=/
|
HOME_URL=/
|
||||||
# 日志等级: debug, info, warn, error
|
# 日志等级: debug, info, warn, error
|
||||||
LOG_LEVEL=info
|
LOG_LEVEL=debug
|
||||||
|
STORE_LOG_LEVEL=warn
|
||||||
# Loki Log Path
|
# Loki Log Path
|
||||||
# LOKI_LOG_URL=
|
|
Reference in New Issue
Block a user