4.8.7 fix (#2076)

projects/app/src/service/common/system/index.ts

@@ -1,5 +1,14 @@
 import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
-import { existsSync, readFileSync } from 'fs';
+import { existsSync, readdirSync, readFileSync } from 'fs';
+import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d';
+import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d';
+import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants';
+import { getFastGPTConfigFromDB } from '@fastgpt/service/common/system/config/controller';
+import { PluginTemplateType } from '@fastgpt/global/core/plugin/type';
+import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
+import { initFastGPTConfig } from '@fastgpt/service/common/system/tools';
+import json5 from 'json5';
+import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type';
 
 export const readConfigData = (name: string) => {
   const isDev = process.env.NODE_ENV === 'development';
@@ -25,6 +34,7 @@ export const readConfigData = (name: string) => {
   return content;
 };
 
+/* Init global variables */
 export function initGlobal() {
   if (global.communityPlugins) return;
 
@@ -33,3 +43,144 @@ export function initGlobal() {
   global.vectorQueueLen = global.vectorQueueLen ?? 0;
   initHttpAgent();
 }
+
+/* Init system data(Need to connected db). It only needs to run once */
+export async function getInitConfig() {
+  return Promise.all([
+    initSystemConfig(),
+    getSystemVersion(),
+
+    // abandon
+    getSystemPlugin(),
+    getSystemPluginV1()
+  ]);
+}
+
+const defaultFeConfigs: FastGPTFeConfigsType = {
+  show_emptyChat: true,
+  show_git: true,
+  docUrl: 'https://doc.fastgpt.in',
+  openAPIDocUrl: 'https://doc.fastgpt.in/docs/development/openapi',
+  systemTitle: 'FastGPT',
+  concatMd:
+    '项目开源地址: [FastGPT GitHub](https://github.com/labring/FastGPT)\n交流群: ![](https://oss.laf.run/htr4n1-images/fastgpt-qr-code.jpg)',
+  limit: {
+    exportDatasetLimitMinutes: 0,
+    websiteSyncLimitMinuted: 0
+  },
+  scripts: [],
+  favicon: '/favicon.ico',
+  uploadFileMaxSize: 500
+};
+
+export async function initSystemConfig() {
+  // load config
+  const [dbConfig, fileConfig] = await Promise.all([
+    getFastGPTConfigFromDB(),
+    readConfigData('config.json')
+  ]);
+  const fileRes = json5.parse(fileConfig) as FastGPTConfigFileType;
+
+  // get config from database
+  const config: FastGPTConfigFileType = {
+    feConfigs: {
+      ...fileRes?.feConfigs,
+      ...defaultFeConfigs,
+      ...(dbConfig.feConfigs || {}),
+      isPlus: !!FastGPTProUrl
+    },
+    systemEnv: {
+      ...fileRes.systemEnv,
+      ...(dbConfig.systemEnv || {})
+    },
+    subPlans: dbConfig.subPlans || fileRes.subPlans,
+    llmModels: dbConfig.llmModels || fileRes.llmModels || [],
+    vectorModels: dbConfig.vectorModels || fileRes.vectorModels || [],
+    reRankModels: dbConfig.reRankModels || fileRes.reRankModels || [],
+    audioSpeechModels: dbConfig.audioSpeechModels || fileRes.audioSpeechModels || [],
+    whisperModel: dbConfig.whisperModel || fileRes.whisperModel
+  };
+
+  // set config
+  initFastGPTConfig(config);
+  console.log({
+    feConfigs: global.feConfigs,
+    systemEnv: global.systemEnv,
+    subPlans: global.subPlans,
+    llmModels: global.llmModels,
+    vectorModels: global.vectorModels,
+    reRankModels: global.reRankModels,
+    audioSpeechModels: global.audioSpeechModels,
+    whisperModel: global.whisperModel
+  });
+}
+
+function getSystemVersion() {
+  if (global.systemVersion) return;
+  try {
+    if (process.env.NODE_ENV === 'development') {
+      global.systemVersion = process.env.npm_package_version || '0.0.0';
+    } else {
+      const packageJson = json5.parse(readFileSync('/app/package.json', 'utf-8'));
+
+      global.systemVersion = packageJson?.version;
+    }
+    console.log(`System Version: ${global.systemVersion}`);
+  } catch (error) {
+    console.log(error);
+
+    global.systemVersion = '0.0.0';
+  }
+}
+
+function getSystemPlugin() {
+  if (global.communityPlugins && global.communityPlugins.length > 0) return;
+
+  const basePath =
+    process.env.NODE_ENV === 'development' ? 'data/pluginTemplates' : '/app/data/pluginTemplates';
+  // read data/pluginTemplates directory, get all json file
+  const files = readdirSync(basePath);
+  // filter json file
+  const filterFiles = files.filter((item) => item.endsWith('.json'));
+
+  // read json file
+  const fileTemplates = filterFiles.map<SystemPluginTemplateItemType>((filename) => {
+    const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
+    return {
+      ...json5.parse(content),
+      originCost: 0,
+      currentCost: 0,
+      id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`
+    };
+  });
+
+  fileTemplates.sort((a, b) => (b.weight || 0) - (a.weight || 0));
+
+  global.communityPlugins = fileTemplates;
+}
+function getSystemPluginV1() {
+  if (global.communityPluginsV1 && global.communityPluginsV1.length > 0) return;
+
+  const basePath =
+    process.env.NODE_ENV === 'development'
+      ? 'data/pluginTemplates/v1'
+      : '/app/data/pluginTemplates/v1';
+  // read data/pluginTemplates directory, get all json file
+  const files = readdirSync(basePath);
+  // filter json file
+  const filterFiles = files.filter((item) => item.endsWith('.json'));
+
+  // read json file
+  const fileTemplates: (PluginTemplateType & { weight: number })[] = filterFiles.map((filename) => {
+    const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
+    return {
+      ...JSON.parse(content),
+      id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`,
+      source: PluginSourceEnum.community
+    };
+  });
+
+  fileTemplates.sort((a, b) => b.weight - a.weight);
+
+  global.communityPluginsV1 = fileTemplates;
+}

projects/app/src/service/common/system/volumnMongoWatch.ts

@@ -1,4 +1,4 @@
-import { initSystemConfig } from '@/pages/api/common/system/getInitData';
+import { initSystemConfig } from '.';
 import { createDatasetTrainingMongoWatch } from '@/service/core/dataset/training/utils';
 import { MongoSystemConfigs } from '@fastgpt/service/common/system/config/schema';
 

projects/app/src/service/mongo.ts

@@ -5,23 +5,29 @@ import { hashStr } from '@fastgpt/global/common/string/tools';
 import { createDefaultTeam } from '@fastgpt/service/support/user/team/controller';
 import { exit } from 'process';
 import { initVectorStore } from '@fastgpt/service/common/vectorStore/controller';
-import { getInitConfig } from '@/pages/api/common/system/getInitData';
 import { startCron } from './common/system/cron';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
-import { initGlobal } from './common/system';
+import { initGlobal, getInitConfig } from './common/system';
 import { startMongoWatch } from './common/system/volumnMongoWatch';
 import { startTrainingQueue } from './core/dataset/training/utils';
 import { systemStartCb } from '@fastgpt/service/common/system/tools';
+import { addLog } from '@fastgpt/service/common/system/log';
 
 /**
+ * This function is equivalent to the entry to the service
  * connect MongoDB and init data
  */
 export function connectToDatabase() {
-  return connectMongo({
-    beforeHook: () => {
-      initGlobal();
-    },
-    afterHook: async () => {
+  if (!global.systemLoadedGlobalVariables) {
+    global.systemLoadedGlobalVariables = true;
+    initGlobal();
+  }
+
+  return connectMongo().then(async () => {
+    if (global.systemLoadedGlobalConfig) return;
+    global.systemLoadedGlobalConfig = true;
+
+    try {
       systemStartCb();
 
       //init system config；init vector database；init root user
@@ -33,6 +39,9 @@ export function connectToDatabase() {
 
       // start queue
       startTrainingQueue(true);
+    } catch (error) {
+      addLog.error('init error', error);
+      exit(1);
     }
   });
 }