ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-01 03:48:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extraction schema (#398)

Browse Source
This commit is contained in:
Archer
2023-10-14 23:02:01 +08:00
committed by GitHub
parent 7db8d3ea0f
commit dd8f2744bf
193 changed files with 2036 additions and 15694 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
projects/app/src/service/common/api/plugins.ts
View File

@@ -1,15 +0,0 @@
import { POST } from './request';
export const postTextCensor = (data: { text: string }) =>
POST<{ code?: number; message: string }>('/plugins/censor/text_baidu', data)
.then((res) => {
if (res?.code === 5000) {
return Promise.reject(res);
}
})
.catch((err) => {
if (err?.code === 5000) {
return Promise.reject(err.message);
}
return Promise.resolve('');
});

120
projects/app/src/service/common/api/request.ts
View File

@@ -1,120 +0,0 @@
import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios';
interface ConfigType {
headers?: { [key: string]: string };
hold?: boolean;
timeout?: number;
}
interface ResponseDataType {
code: number;
message: string;
data: any;
}
/**
* 请求开始
*/
function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig {
if (config.headers) {
config.headers.rootkey = process.env.ROOT_KEY;
}
return config;
}
/**
* 请求成功,检查请求头
*/
function responseSuccess(response: AxiosResponse<ResponseDataType>) {
return response;
}
/**
* 响应数据检查
*/
function checkRes(data: ResponseDataType) {
if (data === undefined) {
console.log('error->', data, 'data is empty');
return Promise.reject('服务器异常');
} else if (data?.code && (data.code < 200 || data.code >= 400)) {
return Promise.reject(data);
}
return data.data;
}
/**
* 响应错误
*/
function responseError(err: any) {
if (!err) {
return Promise.reject({ message: '未知错误' });
}
if (typeof err === 'string') {
return Promise.reject({ message: err });
}
if (err?.response?.data) {
return Promise.reject(err?.response?.data);
}
return Promise.reject(err);
}
/* 创建请求实例 */
const instance = axios.create({
timeout: 60000, // 超时时间
headers: {
'content-type': 'application/json'
}
});
/* 请求拦截 */
instance.interceptors.request.use(requestStart, (err) => Promise.reject(err));
/* 响应拦截 */
instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err));
export function request(url: string, data: any, config: ConfigType, method: Method): any {
if (!global.systemEnv?.pluginBaseUrl) {
return Promise.reject('商业版插件加载中...');
}
/* 去空 */
for (const key in data) {
if (data[key] === null || data[key] === undefined) {
delete data[key];
}
}
return instance
.request({
baseURL: global.systemEnv.pluginBaseUrl,
url,
method,
data: ['POST', 'PUT'].includes(method) ? data : null,
params: !['POST', 'PUT'].includes(method) ? data : null,
...config // 用户自定义配置，可以覆盖前面的配置
})
.then((res) => checkRes(res.data))
.catch((err) => responseError(err));
}
/**
* api请求方式
* @param {String} url
* @param {Any} params
* @param {Object} config
* @returns
*/
export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
return request(url, params, config, 'GET');
}
export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'POST');
}
export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'PUT');
}
export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
return request(url, data, config, 'DELETE');
}

11
projects/app/src/service/common/bill/push.ts
View File

@@ -1,4 +1,5 @@
import { Bill, User, OutLink } from '@/service/mongo';
import { Bill } from '@/service/mongo';
import { MongoUser } from '@fastgpt/support/user/schema';
import { BillSourceEnum } from '@/constants/user';
import { getModel } from '@/service/utils/data';
import { ChatHistoryItemResType } from '@/types/chat';
@@ -10,7 +11,7 @@ import { defaultQGModel } from '@/pages/api/system/getInitData';
async function createBill(data: CreateBillType) {
try {
await Promise.all([
User.findByIdAndUpdate(data.userId, {
MongoUser.findByIdAndUpdate(data.userId, {
$inc: { balance: -data.total }
}),
Bill.create(data)
@@ -50,7 +51,7 @@ async function concatBill({
}
}
),
User.findByIdAndUpdate(userId, {
MongoUser.findByIdAndUpdate(userId, {
$inc: { balance: -total }
})
]);
@@ -177,12 +178,12 @@ export const pushQuestionGuideBill = ({ tokens, userId }: { tokens: number; user
const total = qgModel.price * tokens;
createBill({
userId,
appName: '问题指引',
appName: '下一步指引',
total,
source: BillSourceEnum.fastgpt,
list: [
{
moduleName: '问题指引',
moduleName: '下一步指引',
amount: total,
model: qgModel.name,
tokenLen: tokens

3
projects/app/src/service/common/bill/schema.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { BillSchema as BillType } from '@/types/common/bill';
import { BillSourceMap } from '@/constants/user';

24
projects/app/src/service/common/ipLimit/schema.ts
View File

@@ -1,24 +0,0 @@
import { Schema, model, models, Model } from 'mongoose';
import type { IpLimitSchemaType } from '@/types/common/ipLimit';
const IpLimitSchema = new Schema({
eventId: {
type: String,
required: true
},
ip: {
type: String,
required: true
},
account: {
type: Number,
default: 0
},
lastMinute: {
type: Date,
default: () => new Date()
}
});
export const IpLimit: Model<IpLimitSchemaType> =
models['ip_limit'] || model('ip_limit', IpLimitSchema);

2
projects/app/src/service/dataset/auth.ts
View File

@@ -1,6 +1,6 @@
import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
import { GridFSStorage } from '../lib/gridfs';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
export async function authFileIdValid(fileId?: string) {
if (!fileId) return true;

78
projects/app/src/service/errorCode.ts
View File

@@ -1,78 +0,0 @@
export const ERROR_CODE: { [key: number]: string } = {
400: '请求失败',
401: '无权访问',
403: '紧张访问',
404: '请求不存在',
405: '请求方法错误',
406: '请求的格式错误',
410: '资源已删除',
422: '验证错误',
500: '服务器发生错误',
502: '网关错误',
503: '服务器暂时过载或维护',
504: '网关超时'
};
export const TOKEN_ERROR_CODE: Record<number, string> = {
403: '登录状态无效,请重新登录'
};
export const proxyError: Record<string, boolean> = {
ECONNABORTED: true,
ECONNRESET: true
};
export enum ERROR_ENUM {
unAuthorization = 'unAuthorization',
insufficientQuota = 'insufficientQuota',
unAuthModel = 'unAuthModel',
unAuthApiKey = 'unAuthApiKey',
unAuthKb = 'unAuthKb',
unAuthFile = 'unAuthFile'
}
export const ERROR_RESPONSE: Record<
any,
{
code: number;
statusText: string;
message: string;
data?: any;
}
> = {
[ERROR_ENUM.unAuthorization]: {
code: 403,
statusText: ERROR_ENUM.unAuthorization,
message: '凭证错误',
data: null
},
[ERROR_ENUM.insufficientQuota]: {
code: 510,
statusText: ERROR_ENUM.insufficientQuota,
message: '账号余额不足',
data: null
},
[ERROR_ENUM.unAuthModel]: {
code: 511,
statusText: ERROR_ENUM.unAuthModel,
message: '无权使用该模型',
data: null
},
[ERROR_ENUM.unAuthKb]: {
code: 512,
statusText: ERROR_ENUM.unAuthKb,
message: '无权使用该知识库',
data: null
},
[ERROR_ENUM.unAuthFile]: {
code: 513,
statusText: ERROR_ENUM.unAuthFile,
message: '无权阅读该文件',
data: null
},
[ERROR_ENUM.unAuthApiKey]: {
code: 514,
statusText: ERROR_ENUM.unAuthApiKey,
message: 'Api Key 不合法',
data: null
}
};

4
projects/app/src/service/events/generateQA.ts
View File

@@ -1,9 +1,9 @@
import { TrainingData } from '@/service/mongo';
import { pushQABill } from '@/service/common/bill/push';
import { TrainingModeEnum } from '@/constants/plugin';
import { ERROR_ENUM } from '../errorCode';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { sendInform } from '@/pages/api/user/inform/send';
import { authBalanceByUid } from '../utils/auth';
import { authBalanceByUid } from '@fastgpt/support/user/auth';
import { getAIApi } from '@fastgpt/core/ai/config';
import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type';
import { addLog } from '../utils/tools';

2
projects/app/src/service/events/generateVector.ts
View File

@@ -1,7 +1,7 @@
import { insertData2Dataset } from '@/service/pg';
import { getVector } from '@/pages/api/openapi/plugin/vector';
import { TrainingData } from '../models/trainingData';
import { ERROR_ENUM } from '../errorCode';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { TrainingModeEnum } from '@/constants/plugin';
import { sendInform } from '@/pages/api/user/inform/send';
import { addLog } from '../utils/tools';

8
projects/app/src/service/lib/gridfs.ts
View File

@@ -1,7 +1,7 @@
import mongoose, { Types } from 'mongoose';
import { Types, connectionMongo } from '@fastgpt/common/mongo';
import fs from 'fs';
import fsp from 'fs/promises';
import { ERROR_ENUM } from '../errorCode';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import type { GSFileInfoType } from '@/types/common/file';
enum BucketNameEnum {
@@ -18,10 +18,10 @@ export class GridFSStorage {
this.uid = String(uid);
}
Collection() {
return mongoose.connection.db.collection(`${this.bucket}.files`);
return connectionMongo.connection.db.collection(`${this.bucket}.files`);
}
GridFSBucket() {
return new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db, {
bucketName: this.bucket
});
}

3
projects/app/src/service/models/app.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { AppSchema as AppType } from '@/types/mongoSchema';
const AppSchema = new Schema({

3
projects/app/src/service/models/chat.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatSchema as ChatType } from '@/types/mongoSchema';
import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat';
import { ChatSourceMap } from '@/constants/chat';

3
projects/app/src/service/models/chatItem.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatItemSchema as ChatItemType } from '@/types/mongoSchema';
import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat';
import { customAlphabet } from 'nanoid';

5
projects/app/src/service/models/collection.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model as MongoModel } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { CollectionSchema as CollectionType } from '@/types/mongoSchema';
const CollectionSchema = new Schema({
@@ -14,5 +15,5 @@ const CollectionSchema = new Schema({
}
});
export const Collection: MongoModel<CollectionType> =
export const Collection: Model<CollectionType> =
models['collection'] || model('collection', CollectionSchema);

3
projects/app/src/service/models/image.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
const ImageSchema = new Schema({
userId: {

3
projects/app/src/service/models/inform.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { informSchema } from '@/types/mongoSchema';
import { InformTypeMap } from '@/constants/user';

45
projects/app/src/service/models/kb.ts
View File

@@ -1,45 +0,0 @@
import { Schema, model, models, Model } from 'mongoose';
import { kbSchema as SchemaType } from '@/types/mongoSchema';
import { KbTypeMap } from '@/constants/dataset';
const kbSchema = new Schema({
parentId: {
type: Schema.Types.ObjectId,
ref: 'kb',
default: null
},
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
updateTime: {
type: Date,
default: () => new Date()
},
avatar: {
type: String,
default: '/icon/logo.svg'
},
name: {
type: String,
required: true
},
vectorModel: {
type: String,
required: true,
default: 'text-embedding-ada-002'
},
type: {
type: String,
enum: Object.keys(KbTypeMap),
required: true,
default: 'dataset'
},
tags: {
type: [String],
default: []
}
});
export const KB: Model<SchemaType> = models['kb'] || model('kb', kbSchema);

3
projects/app/src/service/models/pay.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { PaySchema as PayType } from '@/types/mongoSchema';
const PaySchema = new Schema({
userId: {

3
projects/app/src/service/models/promotionRecord.ts
View File

@@ -1,4 +1,5 @@
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { PromotionRecordSchema as PromotionRecordType } from '@/types/mongoSchema';
const PromotionRecordSchema = new Schema({

5
projects/app/src/service/models/trainingData.ts
View File

@@ -1,5 +1,6 @@
/* 模型的知识库 */
import { Schema, model, models, Model as MongoModel } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { TrainingDataSchema as TrainingDateType } from '@/types/mongoSchema';
import { TrainingTypeMap } from '@/constants/plugin';
@@ -68,5 +69,5 @@ try {
console.log(error);
}
export const TrainingData: MongoModel<TrainingDateType> =
export const TrainingData: Model<TrainingDateType> =
models['trainingData'] || model('trainingData', TrainingDataSchema);

59
projects/app/src/service/models/user.ts
View File

@@ -1,59 +0,0 @@
import { Schema, model, models, Model } from 'mongoose';
import { hashPassword } from '@/service/utils/tools';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { UserModelSchema } from '@/types/mongoSchema';
const UserSchema = new Schema({
username: {
// 可以是手机/邮箱，新的验证都只用手机
type: String,
required: true,
unique: true // 唯一
},
password: {
type: String,
required: true,
set: (val: string) => hashPassword(val),
get: (val: string) => hashPassword(val),
select: false
},
createTime: {
type: Date,
default: () => new Date()
},
avatar: {
type: String,
default: '/icon/human.svg'
},
balance: {
type: Number,
default: 2 * PRICE_SCALE
},
inviterId: {
// 谁邀请注册的
type: Schema.Types.ObjectId,
ref: 'user'
},
promotionRate: {
type: Number,
default: 15
},
limit: {
exportKbTime: {
// Every half hour
type: Date
}
},
openaiAccount: {
type: {
key: String,
baseUrl: String
}
},
timezone: {
type: String,
default: 'Asia/Shanghai'
}
});
export const User: Model<UserModelSchema> = models['user'] || model('user', UserSchema);

2
projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts
View File

@@ -105,7 +105,7 @@ async function functionCall({
required: ['type']
}
};
const ai = getAIApi(user.openaiAccount);
const ai = getAIApi(user.openaiAccount, 48000);
const response = await ai.chat.completions.create({
model: cqModel.model,

2
projects/app/src/service/moduleDispatch/agent/extract.ts
View File

@@ -126,7 +126,7 @@ async function functionCall({
}
};
const ai = getAIApi(user.openaiAccount);
const ai = getAIApi(user.openaiAccount, 480000);
const response = await ai.chat.completions.create({
model: extractModel.model,

4
projects/app/src/service/moduleDispatch/chat/oneapi.ts
View File

@@ -10,7 +10,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat';
import { getChatModel } from '@/service/utils/data';
import { countModelPrice } from '@/service/common/bill/push';
import { ChatModelItemType } from '@/types/model';
import { postTextCensor } from '@/service/common/api/plugins';
import { postTextCensor } from '@fastgpt/common/plusApi/censor';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/core/ai/constant';
import { AppModuleItemType } from '@/types/app';
import { countMessagesTokens, sliceMessagesTB } from '@/utils/common/tiktoken';
@@ -151,7 +151,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
};
} else {
const unStreamResponse = response as ChatCompletion;
const answer = unStreamResponse.choices?.[0].message?.content || '';
const answer = unStreamResponse.choices?.[0]?.message?.content || '';
const totalTokens = unStreamResponse.usage?.total_tokens || 0;
const completeMessages = filterMessages.concat({

5
projects/app/src/service/moduleDispatch/kb/search.ts
View File

@@ -24,6 +24,7 @@ export type KBSearchResponse = {
export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSearchResponse> {
const {
moduleName,
user,
inputs: { kbList = [], similarity = 0.4, limit = 5, userChatInput }
} = props as KBSearchProps;
@@ -45,10 +46,10 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
// search kb
const res: any = await PgClient.query(
`BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 40};
select id, kb_id, q, a, source, file_id, (vector <#> '[${
vectors[0]
}]') * -1 AS score from ${PgDatasetTableName} where kb_id IN (${kbList
}]') * -1 AS score from ${PgDatasetTableName} where user_id='${user._id}' AND kb_id IN (${kbList
.map((item) => `'${item.kbId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0]

109
projects/app/src/service/mongo.ts
View File

@@ -1,110 +1,48 @@
import mongoose from 'mongoose';
import { startQueue } from './utils/tools';
import { getInitConfig } from '@/pages/api/system/getInitData';
import { User } from './models/user';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { initPg } from './pg';
import { createHashPassword } from '@/utils/tools';
import { createLogger, format, transports } from 'winston';
import 'winston-mongodb';
import { getTikTokenEnc } from '@/utils/common/tiktoken';
import { initHttpAgent } from '@fastgpt/core/init';
import { MongoUser } from '@fastgpt/support/user/schema';
import { connectMongo } from '@fastgpt/common/mongo/init';
import { hashStr } from '@fastgpt/common/tools/str';
import { getInitConfig, initGlobal } from '@/pages/api/system/getInitData';
/**
* connect MongoDB and init data
*/
export async function connectToDatabase(): Promise<void> {
if (global.mongodb) {
return;
}
global.mongodb = 'connecting';
// init global data
global.qaQueueLen = 0;
global.vectorQueueLen = 0;
global.sendInformQueue = [];
global.sendInformQueueLen = 0;
// logger
initLogger();
// init function
getInitConfig();
// init tikToken
getTikTokenEnc();
initHttpAgent();
try {
mongoose.set('strictQuery', true);
global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
bufferCommands: true,
maxConnecting: Number(process.env.DB_MAX_LINK || 5),
maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
minPoolSize: 2
});
await initRootUser();
initPg();
console.log('mongo connected');
} catch (error) {
console.log('error->', 'mongo connect error');
global.mongodb = null;
}
// init function
startQueue();
}
function initLogger() {
global.logger = createLogger({
transports: [
new transports.MongoDB({
db: process.env.MONGODB_URI as string,
collection: 'server_logs',
options: {
useUnifiedTopology: true
},
cappedSize: 500000000,
tryReconnect: true,
metaKey: 'meta',
format: format.combine(format.timestamp(), format.json())
}),
new transports.Console({
format: format.combine(
format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
format.printf((info) => {
if (info.level === 'error') {
console.log(info.meta);
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
}
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
info.meta ? `: ${JSON.stringify(info.meta)}` : ''
}`;
})
)
})
]
await connectMongo({
beforeHook: () => {
initGlobal();
getInitConfig();
},
afterHook: async () => {
await initRootUser();
initPg();
// start queue
startQueue();
}
});
}
async function initRootUser() {
try {
const rootUser = await User.findOne({
const rootUser = await MongoUser.findOne({
username: 'root'
});
const psw = process.env.DEFAULT_ROOT_PSW || '123456';
if (rootUser) {
await User.findOneAndUpdate(
await MongoUser.findOneAndUpdate(
{ username: 'root' },
{
password: createHashPassword(psw),
password: hashStr(psw),
balance: 999999 * PRICE_SCALE
}
);
} else {
await User.create({
await MongoUser.create({
username: 'root',
password: createHashPassword(psw),
password: hashStr(psw),
balance: 999999 * PRICE_SCALE
});
}
@@ -121,15 +59,10 @@ async function initRootUser() {
export * from './models/chat';
export * from './models/chatItem';
export * from './models/app';
export * from './models/user';
export * from './common/bill/schema';
export * from './models/pay';
export * from './models/trainingData';
export * from './models/promotionRecord';
export * from './models/collection';
export * from './models/kb';
export * from './models/inform';
export * from './models/image';
export * from './support/outLink/schema';
export * from './support/openapi/schema';

1
projects/app/src/service/pg.ts
View File

@@ -228,6 +228,7 @@ export async function initPg() {
q TEXT NOT NULL,
a TEXT
);
CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 16, ef_construction = 64);
`);
console.log('init pg successful');
} catch (error) {

2
projects/app/src/service/response.ts
View File

@@ -1,6 +1,6 @@
import { sseResponseEventEnum } from '@/constants/chat';
import { NextApiResponse } from 'next';
import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode';
import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { clearCookie, sseResponse, addLog } from './utils/tools';
export interface ResponseType<T = any> {

38
projects/app/src/service/support/openapi/auth.ts
View File

@@ -1,38 +0,0 @@
import { ERROR_ENUM } from '@/service/errorCode';
import { updateApiKeyUsedTime } from './index';
import { OpenApi } from './schema';
export async function authOpenApiKey({ apikey }: { apikey: string }) {
if (!apikey) {
return Promise.reject(ERROR_ENUM.unAuthApiKey);
}
try {
const openApi = await OpenApi.findOne({ apiKey: apikey });
if (!openApi) {
return Promise.reject(ERROR_ENUM.unAuthApiKey);
}
const userId = String(openApi.userId);
// auth limit
if (global.feConfigs?.isPlus) {
if (openApi?.limit?.expiredTime && openApi.limit.expiredTime.getTime() < Date.now()) {
return Promise.reject(`Key ${openApi.apiKey} is expired`);
}
if (
openApi?.limit?.credit &&
openApi.limit.credit > -1 &&
openApi.usage > openApi.limit.credit
) {
return Promise.reject(`Key ${openApi.apiKey} is over usage`);
}
}
updateApiKeyUsedTime(openApi._id);
return { apikey, userId, appId: openApi.appId };
} catch (error) {
return Promise.reject(error);
}
}

18
projects/app/src/service/support/openapi/index.ts
View File

@@ -1,18 +0,0 @@
import { OpenApi } from './schema';
export async function updateApiKeyUsedTime(id: string) {
await OpenApi.findByIdAndUpdate(id, {
lastUsedTime: new Date()
});
}
export async function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) {
await OpenApi.findOneAndUpdate(
{ apiKey: apikey },
{
$inc: {
usage
}
}
);
}

57
projects/app/src/service/support/openapi/schema.ts
View File

@@ -1,57 +0,0 @@
import { Schema, model, models, Model } from 'mongoose';
import { OpenApiSchema } from '@/types/support/openapi';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { formatPrice } from '@fastgpt/common/bill/index';
const OpenApiSchema = new Schema(
{
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
apiKey: {
type: String,
required: true,
get: (val: string) => `******${val.substring(val.length - 4)}`
},
createTime: {
type: Date,
default: () => new Date()
},
lastUsedTime: {
type: Date
},
appId: {
type: String,
required: false
},
name: {
type: String,
default: 'Api Key'
},
usage: {
// total usage. value from bill total
type: Number,
default: 0,
get: (val: number) => formatPrice(val)
},
limit: {
expiredTime: {
type: Date
},
credit: {
// value from user settings
type: Number,
default: -1,
set: (val: number) => val * PRICE_SCALE,
get: (val: number) => formatPrice(val)
}
}
},
{
toObject: { getters: true }
}
);
export const OpenApi: Model<OpenApiSchema> = models['openapi'] || model('openapi', OpenApiSchema);

171
projects/app/src/service/support/outLink/auth.ts
View File

@@ -1,171 +0,0 @@
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { IpLimit } from '@/service/common/ipLimit/schema';
import { authBalanceByUid, AuthUserTypeEnum } from '@/service/utils/auth';
import { OutLinkSchema } from '@/types/support/outLink';
import { OutLink } from './schema';
import axios from 'axios';
type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
export async function authOutLinkChat({
shareId,
ip,
authToken,
question
}: AuthLinkProps & {
shareId: string;
}) {
// get outLink
const outLink = await OutLink.findOne({
shareId
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
const uid = String(outLink.userId);
const [user] = await Promise.all([
authBalanceByUid(uid), // authBalance
...(global.feConfigs?.isPlus ? [authOutLinkLimit({ outLink, ip, authToken, question })] : []) // limit auth
]);
return {
user,
userId: String(outLink.userId),
appId: String(outLink.appId),
authType: AuthUserTypeEnum.token,
responseDetail: outLink.responseDetail
};
}
export async function authOutLinkLimit({
outLink,
ip,
authToken,
question
}: AuthLinkProps & {
outLink: OutLinkSchema;
}) {
if (!ip || !outLink.limit) {
return;
}
if (outLink.limit.expiredTime && outLink.limit.expiredTime.getTime() < Date.now()) {
return Promise.reject('分享链接已过期');
}
if (outLink.limit.credit > -1 && outLink.total > outLink.limit.credit * PRICE_SCALE) {
return Promise.reject('链接超出使用限制');
}
// ip limit
await (async () => {
if (!outLink.limit) {
return;
}
try {
const ipLimit = await IpLimit.findOne({ ip, eventId: outLink._id });
// first request
if (!ipLimit) {
return await IpLimit.create({
eventId: outLink._id,
ip,
account: outLink.limit.QPM - 1
});
}
// over one minute
const diffTime = Date.now() - ipLimit.lastMinute.getTime();
if (diffTime >= 60 * 1000) {
ipLimit.account = outLink.limit.QPM - 1;
ipLimit.lastMinute = new Date();
return await ipLimit.save();
}
// over limit
if (ipLimit.account <= 0) {
return Promise.reject(
`每分钟仅能请求 ${outLink.limit.QPM} 次, ${60 - Math.round(diffTime / 1000)}s 后重试~`
);
}
// update limit
ipLimit.account = ipLimit.account - 1;
await ipLimit.save();
} catch (error) {}
})();
// url auth. send request
await authShareStart({ authToken, tokenUrl: outLink.limit.hookUrl, question });
}
export async function authOutLinkId({ id }: { id: string }) {
const outLink = await OutLink.findOne({
shareId: id
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
return {
userId: String(outLink.userId)
};
}
type TokenAuthResponseType = {
success: boolean;
msg?: string;
message?: string;
};
export const authShareChatInit = async (authToken?: string, tokenUrl?: string) => {
if (!tokenUrl || !global.feConfigs?.isPlus) return;
try {
const { data } = await axios<TokenAuthResponseType>({
baseURL: tokenUrl,
url: '/shareAuth/init',
method: 'POST',
data: {
token: authToken
}
});
if (data?.success !== true) {
return Promise.reject(data?.message || data?.msg || '身份校验失败');
}
} catch (error) {
return Promise.reject('身份校验失败');
}
};
export const authShareStart = async ({
tokenUrl,
authToken,
question
}: {
authToken?: string;
question: string;
tokenUrl?: string;
}) => {
if (!tokenUrl || !global.feConfigs?.isPlus) return;
try {
const { data } = await axios<TokenAuthResponseType>({
baseURL: tokenUrl,
url: '/shareAuth/start',
method: 'POST',
data: {
token: authToken,
question
}
});
if (data?.success !== true) {
return Promise.reject(data?.message || data?.msg || '身份校验失败');
}
} catch (error) {
return Promise.reject('身份校验失败');
}
};

52
projects/app/src/service/support/outLink/index.ts
View File

@@ -1,52 +0,0 @@
import { addLog } from '@/service/utils/tools';
import { ChatHistoryItemResType } from '@/types/chat';
import axios from 'axios';
import { OutLink } from './schema';
export const updateOutLinkUsage = async ({
shareId,
total
}: {
shareId: string;
total: number;
}) => {
try {
await OutLink.findOneAndUpdate(
{ shareId },
{
$inc: { total },
lastTime: new Date()
}
);
} catch (err) {
addLog.error('update shareChat error', err);
}
};
export const pushResult2Remote = async ({
authToken,
shareId,
responseData
}: {
authToken?: string;
shareId?: string;
responseData?: ChatHistoryItemResType[];
}) => {
if (!shareId || !authToken) return;
try {
const outLink = await OutLink.findOne({
shareId
});
if (!outLink?.limit?.hookUrl) return;
axios({
method: 'post',
baseURL: outLink.limit.hookUrl,
url: '/shareAuth/finish',
data: {
token: authToken,
responseData
}
});
} catch (error) {}
};

58
projects/app/src/service/support/outLink/schema.ts
View File

@@ -1,58 +0,0 @@
import { Schema, model, models, Model } from 'mongoose';
import { OutLinkSchema as SchemaType } from '@/types/support/outLink';
import { OutLinkTypeEnum } from '@/constants/chat';
const OutLinkSchema = new Schema({
shareId: {
type: String,
required: true
},
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
appId: {
type: Schema.Types.ObjectId,
ref: 'model',
required: true
},
type: {
type: String,
default: OutLinkTypeEnum.share
},
name: {
type: String,
required: true
},
total: {
// total amount
type: Number,
default: 0
},
lastTime: {
type: Date
},
responseDetail: {
type: Boolean,
default: false
},
limit: {
expiredTime: {
type: Date
},
QPM: {
type: Number,
default: 1000
},
credit: {
type: Number,
default: -1
},
hookUrl: {
type: String
}
}
});
export const OutLink: Model<SchemaType> = models['outlinks'] || model('outlinks', OutLinkSchema);

175
projects/app/src/service/utils/auth.ts
View File

@@ -1,170 +1,7 @@
import type { NextApiRequest } from 'next';
import Cookie from 'cookie';
import { App, User, KB } from '../mongo';
import type { AppSchema, UserModelSchema } from '@/types/mongoSchema';
import { ERROR_ENUM } from '../errorCode';
import { authJWT } from './tools';
import { authOpenApiKey } from '../support/openapi/auth';
import { authOutLinkId } from '../support/outLink/auth';
export enum AuthUserTypeEnum {
token = 'token',
root = 'root',
apikey = 'apikey',
outLink = 'outLink'
}
/* auth balance */
export const authBalanceByUid = async (uid: string) => {
const user = await User.findById<UserModelSchema>(
uid,
'_id username balance openaiAccount timezone'
);
if (!user) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
if (user.balance <= 0) {
return Promise.reject(ERROR_ENUM.insufficientQuota);
}
return user;
};
/* uniform auth user */
export const authUser = async ({
req,
authToken = false,
authRoot = false,
authApiKey = false,
authBalance = false,
authOutLink
}: {
req: NextApiRequest;
authToken?: boolean;
authRoot?: boolean;
authApiKey?: boolean;
authBalance?: boolean;
authOutLink?: boolean;
}) => {
const authCookieToken = async (cookie?: string, token?: string): Promise<string> => {
// 获取 cookie
const cookies = Cookie.parse(cookie || '');
const cookieToken = cookies.token || token;
if (!cookieToken) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return await authJWT(cookieToken);
};
// from authorization get apikey
const parseAuthorization = async (authorization?: string) => {
if (!authorization) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// Bearer fastgpt-xxxx-appId
const auth = authorization.split(' ')[1];
if (!auth) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
const { apikey, appId: authorizationAppid = '' } = await (async () => {
const arr = auth.split('-');
// abandon
if (arr.length === 3) {
return {
apikey: `${arr[0]}-${arr[1]}`,
appId: arr[2]
};
}
if (arr.length === 2) {
return {
apikey: auth
};
}
return Promise.reject(ERROR_ENUM.unAuthorization);
})();
// auth apikey
const { userId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
return {
uid: userId,
apikey,
appId: apiKeyAppId || authorizationAppid
};
};
// root user
const parseRootKey = async (rootKey?: string, userId = '') => {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return userId;
};
const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers || {}) as {
cookie?: string;
token?: string;
apikey?: string;
rootkey?: string; // abandon
userid?: string;
authorization?: string;
};
const { shareId } = (req?.body || {}) as { shareId?: string };
let uid = '';
let appId = '';
let openApiKey = apikey;
let authType: `${AuthUserTypeEnum}` = AuthUserTypeEnum.token;
if (authOutLink && shareId) {
const res = await authOutLinkId({ id: shareId });
uid = res.userId;
authType = AuthUserTypeEnum.outLink;
} else if (authToken && (cookie || token)) {
// user token(from fastgpt web)
uid = await authCookieToken(cookie, token);
authType = AuthUserTypeEnum.token;
} else if (authRoot && rootkey) {
// root user
uid = await parseRootKey(rootkey, userid);
authType = AuthUserTypeEnum.root;
} else if (authApiKey && apikey) {
// apikey
const parseResult = await authOpenApiKey({ apikey });
uid = parseResult.userId;
authType = AuthUserTypeEnum.apikey;
openApiKey = parseResult.apikey;
} else if (authApiKey && authorization) {
// apikey from authorization
const authResponse = await parseAuthorization(authorization);
uid = authResponse.uid;
appId = authResponse.appId;
openApiKey = authResponse.apikey;
authType = AuthUserTypeEnum.apikey;
}
// not rootUser and no uid, reject request
if (!rootkey && !uid) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// balance check
const user = await (() => {
if (authBalance) {
return authBalanceByUid(uid);
}
})();
return {
userId: String(uid),
appId,
authType,
user,
apikey: openApiKey
};
};
import { App } from '../mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import type { AppSchema } from '@/types/mongoSchema';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
// 模型使用权校验
export const authApp = async ({
@@ -200,8 +37,8 @@ export const authApp = async ({
};
// 知识库操作权限
export const authKb = async ({ kbId, userId }: { kbId: string; userId: string }) => {
const kb = await KB.findOne({
export const authDataset = async ({ kbId, userId }: { kbId: string; userId: string }) => {
const kb = await MongoDataset.findOne({
_id: kbId,
userId
});

34
projects/app/src/service/utils/tools.ts
View File

@@ -1,41 +1,8 @@
import type { NextApiResponse, NextApiHandler, NextApiRequest } from 'next';
import NextCors from 'nextjs-cors';
import crypto from 'crypto';
import jwt from 'jsonwebtoken';
import { generateQA } from '../events/generateQA';
import { generateVector } from '../events/generateVector';
import { ERROR_ENUM } from '../errorCode';
/* 密码加密 */
export const hashPassword = (psw: string) => {
return crypto.createHash('sha256').update(psw).digest('hex');
};
/* 生成 token */
export const generateToken = (userId: string) => {
const key = process.env.TOKEN_KEY as string;
const token = jwt.sign(
{
userId,
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
},
key
);
return token;
};
// auth token
export const authJWT = (token: string) =>
new Promise<string>((resolve, reject) => {
const key = process.env.TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId) {
reject(ERROR_ENUM.unAuthorization);
return;
}
resolve(decoded.userId);
});
});
/* set cookie */
export const setCookie = (res: NextApiResponse, token: string) => {
res.setHeader(
@@ -67,6 +34,7 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
/* start task */
export const startQueue = () => {
if (!global.systemEnv) return;
for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) {
generateQA();
}