Extraction schema (#398)

packages/common/constant/errorCode.ts

@@ -0,0 +1,78 @@
+export const ERROR_CODE: { [key: number]: string } = {
+  400: '请求失败',
+  401: '无权访问',
+  403: '紧张访问',
+  404: '请求不存在',
+  405: '请求方法错误',
+  406: '请求的格式错误',
+  410: '资源已删除',
+  422: '验证错误',
+  500: '服务器发生错误',
+  502: '网关错误',
+  503: '服务器暂时过载或维护',
+  504: '网关超时'
+};
+
+export const TOKEN_ERROR_CODE: Record<number, string> = {
+  403: '登录状态无效,请重新登录'
+};
+
+export const proxyError: Record<string, boolean> = {
+  ECONNABORTED: true,
+  ECONNRESET: true
+};
+
+export enum ERROR_ENUM {
+  unAuthorization = 'unAuthorization',
+  insufficientQuota = 'insufficientQuota',
+  unAuthModel = 'unAuthModel',
+  unAuthApiKey = 'unAuthApiKey',
+  unAuthKb = 'unAuthKb',
+  unAuthFile = 'unAuthFile'
+}
+export const ERROR_RESPONSE: Record<
+  any,
+  {
+    code: number;
+    statusText: string;
+    message: string;
+    data?: any;
+  }
+> = {
+  [ERROR_ENUM.unAuthorization]: {
+    code: 403,
+    statusText: ERROR_ENUM.unAuthorization,
+    message: '凭证错误',
+    data: null
+  },
+  [ERROR_ENUM.insufficientQuota]: {
+    code: 510,
+    statusText: ERROR_ENUM.insufficientQuota,
+    message: '账号余额不足',
+    data: null
+  },
+  [ERROR_ENUM.unAuthModel]: {
+    code: 511,
+    statusText: ERROR_ENUM.unAuthModel,
+    message: '无权使用该模型',
+    data: null
+  },
+  [ERROR_ENUM.unAuthKb]: {
+    code: 512,
+    statusText: ERROR_ENUM.unAuthKb,
+    message: '无权使用该知识库',
+    data: null
+  },
+  [ERROR_ENUM.unAuthFile]: {
+    code: 513,
+    statusText: ERROR_ENUM.unAuthFile,
+    message: '无权阅读该文件',
+    data: null
+  },
+  [ERROR_ENUM.unAuthApiKey]: {
+    code: 514,
+    statusText: ERROR_ENUM.unAuthApiKey,
+    message: 'Api Key 不合法',
+    data: null
+  }
+};

packages/common/mongo/index.ts

@@ -0,0 +1,6 @@
+import mongoose from 'mongoose';
+
+export default mongoose;
+export * from 'mongoose';
+
+export const connectionMongo = global.mongodb || mongoose;

packages/common/mongo/init.ts

@@ -0,0 +1,74 @@
+import mongoose from './index';
+import 'winston-mongodb';
+import { createLogger, format, transports } from 'winston';
+
+/**
+ * connect MongoDB and init data
+ */
+export async function connectMongo({
+  beforeHook,
+  afterHook
+}: {
+  beforeHook?: () => any;
+  afterHook?: () => any;
+}): Promise<void> {
+  if (global.mongodb) {
+    return;
+  }
+  global.mongodb = mongoose;
+
+  beforeHook && (await beforeHook());
+
+  // logger
+  initLogger();
+
+  console.log('mongo start connect');
+  try {
+    mongoose.set('strictQuery', true);
+    await mongoose.connect(process.env.MONGODB_URI as string, {
+      bufferCommands: true,
+      maxConnecting: Number(process.env.DB_MAX_LINK || 5),
+      maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
+      minPoolSize: 2
+    });
+
+    console.log('mongo connected');
+
+    afterHook && (await afterHook());
+  } catch (error) {
+    console.log('error->', 'mongo connect error', error);
+    global.mongodb = undefined;
+  }
+}
+
+function initLogger() {
+  global.logger = createLogger({
+    transports: [
+      new transports.MongoDB({
+        db: process.env.MONGODB_URI as string,
+        collection: 'server_logs',
+        options: {
+          useUnifiedTopology: true
+        },
+        cappedSize: 500000000,
+        tryReconnect: true,
+        metaKey: 'meta',
+        format: format.combine(format.timestamp(), format.json())
+      }),
+      new transports.Console({
+        format: format.combine(
+          format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+          format.printf((info) => {
+            if (info.level === 'error') {
+              console.log(info.meta);
+              return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
+            }
+            return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
+              info.meta ? `: ${JSON.stringify(info.meta)}` : ''
+            }`;
+          })
+        )
+      })
+    ]
+  });
+}

packages/common/package.json

@@ -1,4 +1,13 @@
 {
   "name": "@fastgpt/common",
-  "version": "1.0.0"
+  "version": "1.0.0",
+  "dependencies": {
+    "mongoose": "^7.0.2",
+    "winston": "^3.10.0",
+    "winston-mongodb": "^5.1.1",
+    "axios": "^1.5.1"
+  },
+  "devDependencies": {
+    "@types/node": "^20.8.5"
+  }
 }

packages/common/plusApi/censor.ts

@@ -0,0 +1,15 @@
+import { POST } from './request';
+
+export const postTextCensor = (data: { text: string }) =>
+  POST<{ code?: number; message: string }>('/common/censor/text_baidu', data)
+    .then((res) => {
+      if (res?.code === 5000) {
+        return Promise.reject(res);
+      }
+    })
+    .catch((err) => {
+      if (err?.code === 5000) {
+        return Promise.reject(err.message);
+      }
+      return Promise.resolve('');
+    });

packages/common/plusApi/request.ts

@@ -0,0 +1,120 @@
+import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios';
+
+interface ConfigType {
+  headers?: { [key: string]: string };
+  hold?: boolean;
+  timeout?: number;
+}
+interface ResponseDataType {
+  code: number;
+  message: string;
+  data: any;
+}
+
+/**
+ * 请求开始
+ */
+function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig {
+  if (config.headers) {
+    config.headers.rootkey = process.env.ROOT_KEY;
+  }
+
+  return config;
+}
+
+/**
+ * 请求成功,检查请求头
+ */
+function responseSuccess(response: AxiosResponse<ResponseDataType>) {
+  return response;
+}
+/**
+ * 响应数据检查
+ */
+function checkRes(data: ResponseDataType) {
+  if (data === undefined) {
+    console.log('error->', data, 'data is empty');
+    return Promise.reject('服务器异常');
+  } else if (data?.code && (data.code < 200 || data.code >= 400)) {
+    return Promise.reject(data);
+  }
+  return data.data;
+}
+
+/**
+ * 响应错误
+ */
+function responseError(err: any) {
+  if (!err) {
+    return Promise.reject({ message: '未知错误' });
+  }
+  if (typeof err === 'string') {
+    return Promise.reject({ message: err });
+  }
+
+  if (err?.response?.data) {
+    return Promise.reject(err?.response?.data);
+  }
+  return Promise.reject(err);
+}
+
+/* 创建请求实例 */
+const instance = axios.create({
+  timeout: 60000, // 超时时间
+  headers: {
+    'content-type': 'application/json'
+  }
+});
+
+/* 请求拦截 */
+instance.interceptors.request.use(requestStart, (err) => Promise.reject(err));
+/* 响应拦截 */
+instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err));
+
+export function request(url: string, data: any, config: ConfigType, method: Method): any {
+  if (!global.systemEnv?.pluginBaseUrl) {
+    return Promise.reject('商业版插件加载中...');
+  }
+
+  /* 去空 */
+  for (const key in data) {
+    if (data[key] === null || data[key] === undefined) {
+      delete data[key];
+    }
+  }
+
+  return instance
+    .request({
+      baseURL: global.systemEnv.pluginBaseUrl,
+      url,
+      method,
+      data: ['POST', 'PUT'].includes(method) ? data : null,
+      params: !['POST', 'PUT'].includes(method) ? data : null,
+      ...config // 用户自定义配置，可以覆盖前面的配置
+    })
+    .then((res) => checkRes(res.data))
+    .catch((err) => responseError(err));
+}
+
+/**
+ * api请求方式
+ * @param {String} url
+ * @param {Any} params
+ * @param {Object} config
+ * @returns
+ */
+export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, params, config, 'GET');
+}
+
+export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'POST');
+}
+
+export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'PUT');
+}
+
+export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'DELETE');
+}

packages/common/tools/str.ts

@@ -1,5 +1,32 @@
+import crypto from 'crypto';
+
 export function strIsLink(str?: string) {
   if (!str) return false;
   if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true;
   return false;
 }
+
+export const hashStr = (psw: string) => {
+  return crypto.createHash('sha256').update(psw).digest('hex');
+};
+
+/* simple text, remove chinese space and extra \n */
+export const simpleText = (text: string) => {
+  text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
+  text = text.replace(/\n{2,}/g, '\n');
+  text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
+  text = text.replace(/[\x00-\x08]/g, ' ');
+
+  // replace empty \n
+  let newText = '';
+  let lastChar = '';
+  for (let i = 0; i < text.length; i++) {
+    const currentChar = text[i];
+    if (currentChar === '\n' && !/[。？！；.?!;]/g.test(lastChar)) {
+    } else {
+      newText += currentChar;
+    }
+    lastChar = currentChar;
+  }
+  return newText;
+};

packages/common/type/index.d.ts

@@ -0,0 +1,43 @@
+import type { Mongoose } from '../mongo';
+import type { Logger } from 'winston';
+
+export type FeConfigsType = {
+  show_emptyChat?: boolean;
+  show_register?: boolean;
+  show_appStore?: boolean;
+  show_contact?: boolean;
+  show_git?: boolean;
+  show_doc?: boolean;
+  show_pay?: boolean;
+  show_openai_account?: boolean;
+  show_promotion?: boolean;
+  hide_app_flow?: boolean;
+  openAPIUrl?: string;
+  systemTitle?: string;
+  authorText?: string;
+  googleClientVerKey?: string;
+  isPlus?: boolean;
+  oauth?: {
+    github?: string;
+    google?: string;
+  };
+  limit?: {
+    exportLimitMinutes?: number;
+  };
+  scripts?: { [key: string]: string }[];
+};
+
+export type SystemEnvType = {
+  pluginBaseUrl?: string;
+  openapiPrefix?: string;
+  vectorMaxProcess: number;
+  qaMaxProcess: number;
+  pgHNSWEfSearch: number;
+};
+
+declare global {
+  var mongodb: Mongoose | undefined;
+  var logger: Logger;
+  var feConfigs: FeConfigsType;
+  var systemEnv: SystemEnvType;
+}