feat: admin set env

admin/.env.template

@@ -3,4 +3,6 @@ MONGODB_NAME=fastgpt
 ADMIN_USER=username
 ADMIN_PASS=password
 ADMIN_SECRET=any
+PARENT_URL=http://localhost:3000 # FastGpt服务的地址
+PARENT_ROOT_KEY=rootkey # FastGpt 的rootkey
 VITE_PUBLIC_SERVER_URL=http://localhost:3001 # 和server.js一致

admin/service/route/system.js

@@ -1,4 +1,5 @@
 import jwt from 'jsonwebtoken';
+import { System } from '../schema.js';
 
 const adminAuth = {
   username: process.env.ADMIN_USER,
@@ -6,6 +7,14 @@ const adminAuth = {
 };
 const authSecret = process.env.ADMIN_SECRET;
 
+const postParent = () => {
+  fetch(`${process.env.PARENT_URL}/api/system/updateEnv`, {
+    headers: {
+      rootkey: process.env.PARENT_ROOT_KEY
+    }
+  });
+};
+
 export const useSystemRoute = (app) => {
   app.post('/api/login', (req, res) => {
     if (!adminAuth.username || !adminAuth.password) {
@@ -37,6 +46,63 @@ export const useSystemRoute = (app) => {
       res.status(401).end('username or password incorrect');
     }
   });
+  app.get('/system', auth(), async (req, res) => {
+    try {
+      const data = await System.find();
+      const totalCount = await System.countDocuments();
+
+      res.header('Access-Control-Expose-Headers', 'X-Total-Count');
+      res.header('X-Total-Count', totalCount);
+      res.json(
+        data.map((item) => {
+          const obj = item.toObject();
+          return {
+            ...obj,
+            id: obj._id
+          };
+        })
+      );
+    } catch (error) {
+      console.log(error);
+
+      res.status(500).json({ error: 'Error creating system env' });
+    }
+  });
+  app.post('/system', auth(), async (req, res) => {
+    try {
+      await System.create({
+        ...req.body,
+        sensitiveCheck: req.body.sensitiveCheck === 'true'
+      });
+      postParent();
+      res.json({});
+    } catch (error) {
+      res.status(500).json({ error: 'Error creating system env' });
+    }
+  });
+  app.put('/system/:id', auth(), async (req, res) => {
+    try {
+      const _id = req.params.id;
+      await System.findByIdAndUpdate(_id, {
+        ...req.body,
+        sensitiveCheck: req.body.sensitiveCheck === 'true'
+      });
+      postParent();
+      res.json({});
+    } catch (error) {
+      res.status(500).json({ error: 'Error updating system env' });
+    }
+  });
+  app.delete('/system/:id', auth(), async (req, res) => {
+    try {
+      const _id = req.params.id;
+      await System.findByIdAndDelete(_id);
+
+      res.json({});
+    } catch (error) {
+      res.status(500).json({ error: 'Error updating system env' });
+    }
+  });
 };
 
 export const auth = () => {

admin/service/schema.js

@@ -84,7 +84,35 @@ const modelSchema = new mongoose.Schema({
   updateTime: Date
 });
 
-export const Model = mongoose.model('Model', modelSchema);
+const SystemSchema = new mongoose.Schema({
-export const Kb = mongoose.model('Kb', kbSchema);
+  openAIKeys: {
-export const User = mongoose.model('User', userSchema, 'users');
+    type: String,
-export const Pay = mongoose.model('Pay', paySchema, 'pays');
+    default: ''
+  },
+  openAITrainingKeys: {
+    type: String,
+    default: ''
+  },
+  gpt4Key: {
+    type: String,
+    default: ''
+  },
+  vectorMaxProcess: {
+    type: Number,
+    default: 10
+  },
+  qaMaxProcess: {
+    type: Number,
+    default: 10
+  },
+  sensitiveCheck: {
+    type: Boolean,
+    default: false
+  }
+});
+
+export const Model = mongoose.models['model'] || mongoose.model('model', modelSchema);
+export const Kb = mongoose.models['kb'] || mongoose.model('kb', kbSchema);
+export const User = mongoose.models['user'] || mongoose.model('user', userSchema);
+export const Pay = mongoose.models['pay'] || mongoose.model('pay', paySchema);
+export const System = mongoose.models['system'] || mongoose.model('system', SystemSchema);

admin/src/App.tsx

@@ -7,7 +7,7 @@ import {
   fetchJSON
 } from 'tushan';
 import { authProvider } from './auth';
-import { userFields, payFields, kbFields, ModelFields } from './fields';
+import { userFields, payFields, kbFields, ModelFields, SystemFields } from './fields';
 import { Dashboard } from './Dashboard';
 
 const authStorageKey = 'tushan:auth';
@@ -88,6 +88,16 @@ function App() {
         label="应用"
         list={<ListTable fields={ModelFields} action={{ detail: true }} />}
       />
+      <Resource
+        name="system"
+        label="系统"
+        list={
+          <ListTable
+            fields={SystemFields}
+            action={{ detail: true, edit: true, create: true, delete: true }}
+          />
+        }
+      />
     </Tushan>
   );
 }

admin/src/fields.ts

@@ -38,3 +38,12 @@ export const ModelFields = [
   }),
   createTextField('temperature', { label: '温度' })
 ];
+
+export const SystemFields = [
+  createTextField('openAIKeys', { label: 'openAIKeys，逗号隔开' }),
+  createTextField('openAITrainingKeys', { label: 'openAITrainingKeys' }),
+  createTextField('gpt4Key', { label: 'gpt4Key' }),
+  createTextField('vectorMaxProcess', { label: '向量最大进程' }),
+  createTextField('qaMaxProcess', { label: 'qa最大进程' }),
+  createTextField('sensitiveCheck', { label: '敏感词校验(true,false)' })
+];

client/src/pages/api/openapi/kb/appKbSearch.ts

@@ -201,7 +201,7 @@ export async function appKbSearch({
     searchPrompts: [
       {
         obj: ChatRoleEnum.System,
-        value: `知识库:${systemPrompt}`
+        value: `知识库:<${systemPrompt}>`
       },
       guidePrompt
     ]

client/src/pages/api/openapi/text/sensitiveCheck.ts

@@ -8,7 +8,7 @@ import { axiosConfig } from '@/service/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    if (process.env.SENSITIVE_CHECK !== '1') {
+    if (global.systemEnv.sensitiveCheck) {
       return jsonRes(res);
     }
 

client/src/pages/api/system/updateEnv.ts

@@ -0,0 +1,32 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { System } from '@/service/models/system';
+import { authUser } from '@/service/utils/auth';
+
+export type InitDateResponse = {
+  beianText: string;
+  googleVerKey: string;
+};
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  await authUser({ req, authRoot: true });
+  updateSystemEnv();
+  jsonRes<InitDateResponse>(res);
+}
+
+export async function updateSystemEnv() {
+  try {
+    const mongoData = await System.findOne();
+
+    if (mongoData) {
+      const obj = mongoData.toObject();
+      global.systemEnv = {
+        ...global.systemEnv,
+        ...obj
+      };
+    }
+    console.log('update env', global.systemEnv);
+  } catch (error) {
+    console.log('update system env error');
+  }
+}

client/src/service/events/generateQA.ts

@@ -16,9 +16,7 @@ const reduceQueue = () => {
 };
 
 export async function generateQA(): Promise<any> {
-  const maxProcess = Number(process.env.QA_MAX_PROCESS || 10);
+  if (global.qaQueueLen >= global.systemEnv.qaMaxProcess) return;
-
-  if (global.qaQueueLen >= maxProcess) return;
   global.qaQueueLen++;
 
   let trainingId = '';

client/src/service/events/generateVector.ts

@@ -12,9 +12,7 @@ const reduceQueue = () => {
 
 /* 索引生成队列。每导入一次，就是一个单独的线程 */
 export async function generateVector(): Promise<any> {
-  const maxProcess = Number(process.env.VECTOR_MAX_PROCESS || 10);
+  if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return;
-
-  if (global.vectorQueueLen >= maxProcess) return;
   global.vectorQueueLen++;
 
   let trainingId = '';

client/src/service/models/system.ts

@@ -0,0 +1,30 @@
+import { Schema, model, models } from 'mongoose';
+
+const SystemSchema = new Schema({
+  openAIKeys: {
+    type: String,
+    default: ''
+  },
+  openAITrainingKeys: {
+    type: String,
+    default: ''
+  },
+  gpt4Key: {
+    type: String,
+    default: ''
+  },
+  vectorMaxProcess: {
+    type: Number,
+    default: 10
+  },
+  qaMaxProcess: {
+    type: Number,
+    default: 10
+  },
+  sensitiveCheck: {
+    type: Boolean,
+    default: false
+  }
+});
+
+export const System = models['system'] || model('system', SystemSchema);

client/src/service/mongo.ts

@@ -1,7 +1,7 @@
 import mongoose from 'mongoose';
 import tunnel from 'tunnel';
-import { TrainingData } from './mongo';
 import { startQueue } from './utils/tools';
+import { updateSystemEnv } from '@/pages/api/system/updateEnv';
 
 /**
  * 连接 MongoDB 数据库
@@ -11,6 +11,27 @@ export async function connectToDatabase(): Promise<void> {
     return;
   }
 
+  // init global data
+  global.qaQueueLen = 0;
+  global.vectorQueueLen = 0;
+  global.systemEnv = {
+    openAIKeys: process.env.OPENAIKEY || '',
+    openAITrainingKeys: process.env.OPENAI_TRAINING_KEY || '',
+    gpt4Key: process.env.GPT4KEY || '',
+    vectorMaxProcess: 10,
+    qaMaxProcess: 10,
+    sensitiveCheck: false
+  };
+  // proxy obj
+  if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) {
+    global.httpsAgent = tunnel.httpsOverHttp({
+      proxy: {
+        host: process.env.AXIOS_PROXY_HOST,
+        port: +process.env.AXIOS_PROXY_PORT
+      }
+    });
+  }
+
   global.mongodb = 'connecting';
   try {
     mongoose.set('strictQuery', true);
@@ -27,20 +48,8 @@ export async function connectToDatabase(): Promise<void> {
     global.mongodb = null;
   }
 
-  // 创建代理对象
+  // init function
-  if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) {
+  updateSystemEnv();
-    global.httpsAgent = tunnel.httpsOverHttp({
-      proxy: {
-        host: process.env.AXIOS_PROXY_HOST,
-        port: +process.env.AXIOS_PROXY_PORT
-      }
-    });
-  }
-
-  // 初始化队列
-  global.qaQueueLen = 0;
-  global.vectorQueueLen = 0;
-
   startQueue();
 }
 
@@ -57,3 +66,4 @@ export * from './models/collection';
 export * from './models/shareChat';
 export * from './models/kb';
 export * from './models/inform';
+export * from './models/system';

client/src/service/pg.ts

@@ -6,14 +6,13 @@ export const connectPg = async () => {
     return global.pgClient;
   }
 
-  const maxLink = Number(process.env.VECTOR_MAX_PROCESS || 10);
   global.pgClient = new Pool({
     host: process.env.PG_HOST,
     port: process.env.PG_PORT ? +process.env.PG_PORT : 5432,
     user: process.env.PG_USER,
     password: process.env.PG_PASSWORD,
     database: process.env.PG_DB_NAME,
-    max: Math.floor(maxLink * 0.5),
+    max: 80,
     idleTimeoutMillis: 60000,
     connectionTimeoutMillis: 20000
   });

client/src/service/utils/auth.ts

@@ -123,14 +123,21 @@ export const authUser = async ({
 export const getSystemOpenAiKey = (type: ApiKeyType) => {
   const keys = (() => {
     if (type === 'training') {
-      return process.env.OPENAI_TRAINING_KEY?.split(',') || [];
+      return global.systemEnv.openAITrainingKeys?.split(',') || [];
     }
-    return process.env.OPENAIKEY?.split(',') || [];
+    return global.systemEnv.openAIKeys?.split(',') || [];
   })();
 
   // 纯字符串类型
   const i = Math.floor(Math.random() * keys.length);
-  return keys[i] || (process.env.OPENAIKEY as string);
+  return keys[i] || (global.systemEnv.openAIKeys as string);
+};
+export const getGpt4Key = () => {
+  const keys = global.systemEnv.gpt4Key?.split(',') || [];
+
+  // 纯字符串类型
+  const i = Math.floor(Math.random() * keys.length);
+  return keys[i] || (global.systemEnv.openAIKeys as string);
 };
 
 /* 获取 api 请求的 key */
@@ -157,11 +164,11 @@ export const getApiKey = async ({
     },
     [OpenAiChatEnum.GPT4]: {
       userOpenAiKey: user.openaiKey || '',
-      systemAuthKey: process.env.GPT4KEY as string
+      systemAuthKey: getGpt4Key() as string
     },
     [OpenAiChatEnum.GPT432k]: {
       userOpenAiKey: user.openaiKey || '',
-      systemAuthKey: process.env.GPT4KEY as string
+      systemAuthKey: getGpt4Key() as string
     },
     [ClaudeEnum.Claude]: {
       userOpenAiKey: '',

client/src/service/utils/tools.ts

@@ -60,13 +60,10 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
 }
 
 export const startQueue = () => {
-  const qaMax = Number(process.env.QA_MAX_PROCESS || 10);
+  for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) {
-  const vectorMax = Number(process.env.VECTOR_MAX_PROCESS || 10);
-
-  for (let i = 0; i < qaMax; i++) {
     generateQA();
   }
-  for (let i = 0; i < vectorMax; i++) {
+  for (let i = 0; i < global.systemEnv.vectorMaxProcess; i++) {
     generateVector();
   }
 };

client/src/types/index.d.ts

@@ -3,6 +3,15 @@ import type { Agent } from 'http';
 import type { Pool } from 'pg';
 import type { Tiktoken } from '@dqbd/tiktoken';
 
+export type PagingData<T> = {
+  pageNum: number;
+  pageSize: number;
+  data: T[];
+  total?: number;
+};
+
+export type RequestPaging = { pageNum: number; pageSize: number; [key]: any };
+
 declare global {
   var mongodb: Mongoose | string | null;
   var pgClient: Pool | null;
@@ -13,17 +22,16 @@ declare global {
   var qaQueueLen: number;
   var vectorQueueLen: number;
   var OpenAiEncMap: Record<string, Tiktoken>;
+  var systemEnv: {
+    openAIKeys: string;
+    openAITrainingKeys: string;
+    gpt4Key: string;
+    vectorMaxProcess: number;
+    qaMaxProcess: number;
+    sensitiveCheck: boolean;
+  };
 
   interface Window {
     ['pdfjs-dist/build/pdf']: any;
   }
 }
-
-export type PagingData<T> = {
-  pageNum: number;
-  pageSize: number;
-  data: T[];
-  total?: number;
-};
-
-export type RequestPaging = { pageNum: number; pageSize: number; [key]: any };

docs/deploy/fastgpt/docker-compose.yml

@@ -55,15 +55,10 @@ services:
       # google V3 安全校验（可选）
       - CLIENT_GOOGLE_VER_TOKEN=xxx
       - SERVICE_GOOGLE_VER_TOKEN=xx
-      # QA和向量生成最大进程数
-      - QA_MAX_PROCESS=10
-      - VECTOR_MAX_PROCESS=10
       # token加密凭证（随便填，作为登录凭证）
       - TOKEN_KEY=xxxx
       # root key, 最高权限，可以内部接口互相调用
       - ROOT_KEY=xxx
-      # 是否进行内容安全校验(1: 开启，0: 关闭)
-      - SENSITIVE_CHECK=1
       # 和上方mongo镜像的username,password对应
       - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin
       - MONGODB_NAME=fastgpt

docs/dev/.env.template

@@ -1,5 +1,3 @@
-QA_MAX_PROCESS=30
-VECTOR_MAX_PROCESS=30
 # 运行端口，如果不是 3000 口运行，需要改成其他的。注意：不是改了这个变量就会变成其他端口，而是因为改成其他端口，才用这个变量。
 PORT=3000
 # 代理
@@ -17,8 +15,6 @@ aliTemplateCode=xxxx
 TOKEN_KEY=dfdasfdas
 # root key, 最高权限
 ROOT_KEY=fdafasd
-# 是否进行安全校验(1: 开启，0: 关闭)
-SENSITIVE_CHECK=0
 # openai
 # OPENAI_BASE_URL=http://ai.openai.com/v1
 # OPENAI_BASE_URL_AUTH=可选安全凭证，会放到 header.auth 里