Test mongo log (#4443)

* feat: mongodb-log (#4426)

* perf: mongo log

* feat: completions stop reasoner

* mongo db log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

projects/app/.env.template

@@ -24,7 +24,8 @@ MULTIPLE_DATA_TO_BASE64=true
 REDIS_URL=redis://default:password@127.0.0.1:6379
 # mongo 数据库连接参数，本地开发连接远程数据库时，可能需要增加 directConnection=true 参数，才能连接上。
 MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt?authSource=admin
-
+# 日志库
+MONGODB_LOG_URI=mongodb://username:password@0.0.0.0:27017/fastgpt?authSource=admin
 # 向量库优先级: pg > oceanbase > milvus
 # PG 向量库连接参数
 PG_URL=postgresql://username:password@host:port/postgres

projects/app/src/components/core/ai/SettingLLMModel/index.tsx

@@ -99,7 +99,6 @@ const SettingLLMModel = ({
         <AISettingModal
           onClose={onCloseAIChatSetting}
           onSuccess={(e) => {
-            console.log(e);
             onChange(e);
             onCloseAIChatSetting();
           }}

projects/app/src/components/core/chat/components/WholeResponseModal.tsx

@@ -17,6 +17,7 @@ import { ChatBoxContext } from '../ChatContainer/ChatBox/Provider';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
 import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
+import { completionFinishReasonMap } from '@fastgpt/global/core/ai/constants';
 
 type sideTabItemType = {
   moduleLogo?: string;
@@ -196,6 +197,13 @@ export const WholeResponseContent = ({
           label={t('common:core.chat.response.module maxToken')}
           value={activeModule?.maxToken}
         />
+        {activeModule?.finishReason && (
+          <Row
+            label={t('chat:completion_finish_reason')}
+            value={t(completionFinishReasonMap[activeModule?.finishReason])}
+          />
+        )}
+
         <Row label={t('chat:reasoning_text')} value={activeModule?.reasoningText} />
         <Row
           label={t('common:core.chat.response.module historyPreview')}

projects/app/src/instrumentation.ts

@@ -9,6 +9,7 @@ export async function register() {
       // 基础系统初始化
       const [
         { connectMongo },
+        { connectionMongo, connectionLogMongo, MONGO_URL, MONGO_LOG_URL },
         { systemStartCb },
         { initGlobalVariables, getInitConfig, initSystemPluginGroups, initAppTemplateTypes },
         { initVectorStore },
@@ -19,6 +20,7 @@ export async function register() {
         { startTrainingQueue }
       ] = await Promise.all([
         import('@fastgpt/service/common/mongo/init'),
+        import('@fastgpt/service/common/mongo/index'),
         import('@fastgpt/service/common/system/tools'),
         import('@/service/common/system'),
         import('@fastgpt/service/common/vectorStore/controller'),
@@ -34,7 +36,8 @@ export async function register() {
       initGlobalVariables();
 
       // Connect to MongoDB
-      await connectMongo();
+      await connectMongo(connectionMongo, MONGO_URL);
+      connectMongo(connectionLogMongo, MONGO_LOG_URL);
 
       //init system config；init vector database；init root user
       await Promise.all([getInitConfig(), initVectorStore(), initRootUser()]);

projects/app/src/pages/api/admin/clearInvalidData.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { addHours } from 'date-fns';
 import { MongoImage } from '@fastgpt/service/common/file/image/schema';
@@ -56,7 +55,6 @@ async function checkInvalidImg(start: Date, end: Date, limit = 50) {
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
     const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };
 

projects/app/src/pages/api/admin/initv47.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
 import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
@@ -8,7 +7,6 @@ import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoPlugin.updateMany(

projects/app/src/pages/api/admin/initv471.ts

@@ -1,13 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
 
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     // 删除索引

projects/app/src/pages/api/admin/initv4810.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
 import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
@@ -9,7 +8,6 @@ import { POST } from '@fastgpt/service/common/api/plusRequest';
 /* 初始化发布的版本 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoAppVersion.updateMany(

projects/app/src/pages/api/admin/initv4823.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { addHours } from 'date-fns';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
@@ -174,7 +173,6 @@ const checkInvalidDataText = async () => {
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
     const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };
 

projects/app/src/pages/api/admin/initv486.ts

@@ -1,14 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
 
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoApp.updateMany(

projects/app/src/pages/api/admin/initv488.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
@@ -8,7 +7,6 @@ import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoDataset.updateMany(

projects/app/src/pages/api/common/file/read.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authFileToken } from '@fastgpt/service/support/permission/controller';
 import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
@@ -23,8 +22,6 @@ const previewableExtensions = [
 // Abandoned, use: file/read/[filename].ts
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
-
     const { token } = req.query as { token: string };
 
     const { fileId, bucketName } = await authFileToken(token);

projects/app/src/pages/api/common/file/read/[filename].ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authFileToken } from '@fastgpt/service/support/permission/controller';
 import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
@@ -21,8 +20,6 @@ const previewableExtensions = [
 ];
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
-
     const { token, filename } = req.query as { token: string; filename: string };
 
     const { fileId, bucketName } = await authFileToken(token);

projects/app/src/pages/api/common/file/uploadImage.ts

@@ -1,5 +1,4 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
-import { connectToDatabase } from '@/service/mongo';
 import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
 import { UploadImgProps } from '@fastgpt/global/common/file/api';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
@@ -9,7 +8,6 @@ import { NextAPI } from '@/service/middleware/entry';
   Upload avatar image
 */
 async function handler(req: NextApiRequest, res: NextApiResponse): Promise<string> {
-  await connectToDatabase();
   const body = req.body as UploadImgProps;
 
   const { teamId } = await authCert({ req, authToken: true });

projects/app/src/pages/api/common/system/unlockTask.ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
     await authCert({ req, authToken: true });
     startTrainingQueue();
   } catch (error) {}

projects/app/src/pages/api/common/tools/urlFetch.ts

@@ -2,13 +2,12 @@
 import { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { connectToDatabase } from '@/service/mongo';
+
 import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
 import { urlsFetch } from '@fastgpt/service/common/string/cheerio';
 
 const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
   try {
-    await connectToDatabase();
     let { urlList = [], selector } = req.body as UrlFetchParams;
 
     if (!urlList || urlList.length === 0) {

projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts

@@ -1,6 +1,6 @@
 import type { NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { pushQuestionGuideUsage } from '@/service/support/wallet/usage/push';
 import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
@@ -27,7 +27,6 @@ async function handler(
   res: NextApiResponse<any>
 ) {
   try {
-    await connectToDatabase();
     const { messages } = req.body;
 
     const { tmbId, teamId } = await authChatCert({

projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import type { AdminUpdateFeedbackParams } from '@/global/core/chat/api.d';
 import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
 import { authChatCrud } from '@/service/support/permission/auth/chat';
@@ -8,7 +8,6 @@ import { authChatCrud } from '@/service/support/permission/auth/chat';
 /* 初始化我的聊天框，需要身份验证 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { appId, chatId, dataId, datasetId, feedbackDataId, q, a } =
       req.body as AdminUpdateFeedbackParams;
 

projects/app/src/pages/api/core/chat/feedback/closeCustom.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d';
 import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
@@ -10,7 +10,6 @@ import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 /* remove custom feedback */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { appId, chatId, dataId, index } = req.body as CloseCustomFeedbackParams;
 
     if (!dataId || !appId || !chatId) {

projects/app/src/pages/api/core/chat/item/getSpeech.ts

@@ -1,6 +1,6 @@
 import type { NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { GetChatSpeechProps } from '@/global/core/chat/api.d';
 import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
 import { pushAudioSpeechUsage } from '@/service/support/wallet/usage/push';
@@ -17,7 +17,6 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
 */
 async function handler(req: ApiRequestProps<GetChatSpeechProps>, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { ttsConfig, input } = req.body;
 
     if (!ttsConfig.model || !ttsConfig.voice) {

projects/app/src/pages/api/lafApi/[...path].ts

@@ -1,11 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { request } from 'https';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { path = [], ...query } = req.query as any;
 
     const queryStr = new URLSearchParams(query).toString();

projects/app/src/pages/api/proApi/[...path].ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { request } from 'http';
 import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { path = [], ...query } = req.query as any;
     const requestPath = `/api/${path?.join('/')}?${new URLSearchParams(query).toString()}`;
 

projects/app/src/pages/api/support/user/account/updatePasswordByOld.ts

@@ -2,12 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { connectToDatabase } from '@/service/mongo';
+
 import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
     const { oldPsw, newPsw } = req.body as { oldPsw: string; newPsw: string };
 
     if (!oldPsw || !newPsw) {

projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
     const { size } = req.query as {
       size: string;
     };

projects/app/src/pages/api/support/user/team/limit/webSyncLimit.ts

@@ -1,13 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { checkWebSyncLimit } from '@fastgpt/service/support/user/utils';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
-
     // 凭证校验
     const { teamId } = await authCert({ req, authToken: true });
 

projects/app/src/pages/api/system/img/[id].ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { readMongoImg } from '@fastgpt/service/common/file/image/controller';
 
 // get the models available to the system
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { id } = req.query as { id: string };
 
     const { binary, mime } = await readMongoImg({ id });

projects/app/src/pages/chat/share.tsx

@@ -17,7 +17,7 @@ import { getInitOutLinkChatInfo } from '@/web/core/chat/api';
 import { getChatTitleFromChatMessage } from '@fastgpt/global/core/chat/utils';
 import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
 import { addLog } from '@fastgpt/service/common/system/log';
-import { connectToDatabase } from '@/service/mongo';
+
 import NextHead from '@/components/common/NextHead';
 import { useContextSelector } from 'use-context-selector';
 import ChatContextProvider, { ChatContext } from '@/web/core/chat/context/chatContext';
@@ -391,7 +391,6 @@ export async function getServerSideProps(context: any) {
 
   const app = await (async () => {
     try {
-      await connectToDatabase();
       return MongoOutLink.findOne(
         {
           shareId

projects/app/src/service/mongo.ts

@@ -1,19 +1,9 @@
-import { PRICE_SCALE } from '@fastgpt/global/support/wallet/constants';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { connectMongo } from '@fastgpt/service/common/mongo/init';
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { createDefaultTeam } from '@fastgpt/service/support/user/team/controller';
 import { exit } from 'process';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 
-/**
- * This function is equivalent to the entry to the service
- * connect MongoDB and init data
- */
-export function connectToDatabase() {
-  return connectMongo();
-}
-
 export async function initRootUser(retry = 3): Promise<any> {
   try {
     const rootUser = await MongoUser.findOne({