4.8.12 test fix (#2988)

* perf: qps limit

* perf: http response data

* perf: json path check

* fix: ts

* loop support reference parent variable
This commit is contained in:
Archer
2024-10-25 16:34:26 +08:00
committed by GitHub
parent 165fe077bc
commit 74d58d562b
14 changed files with 144 additions and 122 deletions

View File

@@ -41,7 +41,7 @@ export enum ERROR_ENUM {
unAuthModel = 'unAuthModel',
unAuthApiKey = 'unAuthApiKey',
unAuthFile = 'unAuthFile',
QPSLimitExceed = 'QPSLimitExceed'
tooManyRequest = 'tooManyRequest'
}
export type ErrType<T> = Record<
@@ -69,10 +69,10 @@ export const ERROR_RESPONSE: Record<
message: i18nT('common:code_error.error_message.403'),
data: null
},
[ERROR_ENUM.QPSLimitExceed]: {
[ERROR_ENUM.tooManyRequest]: {
code: 429,
statusText: ERROR_ENUM.QPSLimitExceed,
message: i18nT('common:code_error.error_code.429'),
statusText: ERROR_ENUM.tooManyRequest,
message: 'Too many request',
data: null
},
[ERROR_ENUM.insufficientQuota]: {

View File

@@ -8,7 +8,7 @@
"weight": 10,
"courseUrl": "https://fael3z0zfze.feishu.cn/wiki/LsKAwOmtniA4vkkC259cmfxXnAc?fromScene=spaceOverview",
"isTool": true,
"templateType": "tools",
"templateType": "search",
"workflow": {
"nodes": [
@@ -37,7 +37,7 @@
"required": true
},
{
"renderTypeList": ["reference"],
"renderTypeList": ["input", "reference"],
"selectedTypeIndex": 0,
"valueType": "string",
"canEdit": true,

View File

@@ -19,8 +19,11 @@ export const NextEntry = ({ beforeCallback = [] }: { beforeCallback?: Promise<an
await Promise.all([withNextCors(req, res), ...beforeCallback]);
let response = null;
for (const handler of args) {
for await (const handler of args) {
response = await handler(req, res);
if (res.writableFinished) {
break;
}
}
// Get request duration

View File

@@ -1,26 +0,0 @@
import { ApiRequestProps } from 'type/next';
import requestIp from 'request-ip';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { authFrequencyLimit } from 'common/system/frequencyLimit/utils';
import { addSeconds } from 'date-fns';
// unit: times/s
// how to use?
// export default NextAPI(useQPSLimit(10), handler); // limit 10 times per second for a ip
export function useQPSLimit(limit: number) {
return async (req: ApiRequestProps) => {
const ip = requestIp.getClientIp(req);
if (!ip) {
return;
}
try {
await authFrequencyLimit({
eventId: 'ip-qps-limit' + ip,
maxAmount: limit,
expiredTime: addSeconds(new Date(), 1)
});
} catch (_) {
return Promise.reject(ERROR_ENUM.QPSLimitExceed);
}
};
}

View File

@@ -0,0 +1,32 @@
import { ApiRequestProps } from '../../type/next';
import requestIp from 'request-ip';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { authFrequencyLimit } from '../system/frequencyLimit/utils';
import { addSeconds } from 'date-fns';
import { NextApiResponse } from 'next';
import { jsonRes } from '../response';
// unit: times/s
// how to use?
// export default NextAPI(useQPSLimit(10), handler); // limit 10 times per second for a ip
export function useReqFrequencyLimit(seconds: number, limit: number) {
return async (req: ApiRequestProps, res: NextApiResponse) => {
const ip = requestIp.getClientIp(req);
if (!ip && process.env.USE_IP_LIMIT !== 'true') {
return;
}
try {
await authFrequencyLimit({
eventId: 'ip-qps-limit' + ip,
maxAmount: limit,
expiredTime: addSeconds(new Date(), seconds)
});
} catch (_) {
res.status(429);
jsonRes(res, {
code: 429,
message: ERROR_ENUM.tooManyRequest
});
}
};
}

View File

@@ -17,7 +17,7 @@ const FrequencyLimitSchema = new Schema({
});
try {
FrequencyLimitSchema.index({ eventId: 1 }, { unique: true });
FrequencyLimitSchema.index({ eventId: 1, expiredTime: 1 });
FrequencyLimitSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 0 });
} catch (error) {}

View File

@@ -1,6 +1,5 @@
import { AuthFrequencyLimitProps } from '@fastgpt/global/common/frequenctLimit/type';
import { MongoFrequencyLimit } from './schema';
import { readFromSecondary } from '../../mongo/utils';
export const authFrequencyLimit = async ({
eventId,
@@ -11,22 +10,24 @@ export const authFrequencyLimit = async ({
// 对应 eventId 的 account+1, 不存在的话,则创建一个
const result = await MongoFrequencyLimit.findOneAndUpdate(
{
eventId
eventId,
expiredTime: { $gte: new Date() }
},
{
$inc: { amount: 1 },
// If not exist, set the expiredTime
$setOnInsert: { expiredTime }
},
{
upsert: true,
new: true,
...readFromSecondary
new: true
}
);
).lean();
// 因为始终会返回+1的结果所以这里不能直接等需要多一个。
if (result.amount > maxAmount) {
return Promise.reject(result);
}
} catch (error) {}
} catch (error) {
console.log(error);
}
};

View File

@@ -32,8 +32,6 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
return Promise.reject('Input array length cannot be greater than 50');
}
const runNodes = runtimeNodes.filter((node) => childrenNodeIdList.includes(node.nodeId));
const outputValueArr = [];
const loopDetail: ChatHistoryItemResType[] = [];
let assistantResponses: AIChatItemValueItemType[] = [];
@@ -43,7 +41,7 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
for await (const item of loopInputArray) {
const response = await dispatchWorkFlow({
...props,
runtimeNodes: runNodes.map((node) =>
runtimeNodes: runtimeNodes.map((node) =>
node.flowNodeType === FlowNodeTypeEnum.loopStart
? {
...node,

View File

@@ -239,7 +239,15 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
)
.forEach((item) => {
const key = item.key.startsWith('$') ? item.key : `$.${item.key}`;
results[item.key] = JSONPath({ path: key, json: formatResponse })[0];
results[item.key] = (() => {
// 检查是否是简单的属性访问或单一索引访问
if (/^\$(\.[a-zA-Z_][a-zA-Z0-9_]*)+$/.test(key) || /^\$(\[\d+\])+$/.test(key)) {
return JSONPath({ path: key, json: formatResponse })[0];
}
// 如果无法确定,默认返回数组
return JSONPath({ path: key, json: formatResponse });
})();
});
if (typeof formatResponse[NodeOutputKeyEnum.answerText] === 'string') {
@@ -252,6 +260,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
}
return {
...results,
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints: 0,
params: Object.keys(params).length > 0 ? params : undefined,
@@ -261,8 +270,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
},
[DispatchNodeResponseKeyEnum.toolResponses]:
Object.keys(results).length > 0 ? results : rawResponse,
[NodeOutputKeyEnum.httpRawResponse]: rawResponse,
...results
[NodeOutputKeyEnum.httpRawResponse]: rawResponse
};
} catch (error) {
addLog.error('Http request error', error);

View File

@@ -37,5 +37,6 @@ FE_DOMAIN=http://localhost:3000
LOG_LEVEL=debug
STORE_LOG_LEVEL=warn
# 安全配置
# 工作流最大运行次数,避免极端的死循环情况
WORKFLOW_MAX_RUN_TIMES=500

View File

@@ -33,79 +33,6 @@ enum CronJobTypeEnum {
}
type CronType = 'month' | 'week' | 'day' | 'interval';
const get24HoursOptions = () => {
return Array.from({ length: 24 }, (_, i) => ({
label: `${i < 10 ? '0' : ''}${i}:00`,
value: i
}));
};
const getRoute = (i: number) => {
switch (i) {
case 0:
return 'app:week.Sunday';
case 1:
return 'app:week.Monday';
case 2:
return 'app:week.Tuesday';
case 3:
return 'app:week.Wednesday';
case 4:
return 'app:week.Thursday';
case 5:
return 'app:week.Friday';
case 6:
return 'app:week.Saturday';
default:
return 'app:week.Sunday';
}
};
const getWeekOptions = () => {
return Array.from({ length: 7 }, (_, i) => {
return {
label: i18nT(getRoute(i)),
value: i,
children: get24HoursOptions()
};
});
};
const getMonthOptions = () => {
return Array.from({ length: 28 }, (_, i) => ({
label: `${i + 1}` + i18nT('app:month.unit'),
value: i,
children: get24HoursOptions()
}));
};
const getInterValOptions = () => {
// 每n小时
return [
{
label: i18nT('app:interval.per_hour'),
value: 1
},
{
label: i18nT('app:interval.2_hours'),
value: 2
},
{
label: i18nT('app:interval.3_hours'),
value: 3
},
{
label: i18nT('app:interval.4_hours'),
value: 4
},
{
label: i18nT('app:interval.6_hours'),
value: 6
},
{
label: i18nT('app:interval.12_hours'),
value: 12
}
];
};
const defaultValue = ['day', 0, 0];
const defaultCronString = '0 0 * * *';
@@ -125,6 +52,81 @@ const ScheduledTriggerConfig = ({
const cronString = value?.cronString;
const defaultPrompt = value?.defaultPrompt;
const get24HoursOptions = () => {
return Array.from({ length: 24 }, (_, i) => ({
label: `${i < 10 ? '0' : ''}${i}:00`,
value: i
}));
};
const getRoute = (i: number) => {
const { t } = useTranslation();
switch (i) {
case 0:
return t('app:week.Sunday');
case 1:
return t('app:week.Monday');
case 2:
return t('app:week.Tuesday');
case 3:
return t('app:week.Wednesday');
case 4:
return t('app:week.Thursday');
case 5:
return t('app:week.Friday');
case 6:
return t('app:week.Saturday');
default:
return t('app:week.Sunday');
}
};
const getWeekOptions = () => {
return Array.from({ length: 7 }, (_, i) => {
return {
label: getRoute(i),
value: i,
children: get24HoursOptions()
};
});
};
const getMonthOptions = () => {
return Array.from({ length: 28 }, (_, i) => ({
label: `${i + 1}` + t('app:month.unit'),
value: i,
children: get24HoursOptions()
}));
};
const getInterValOptions = () => {
// 每n小时
return [
{
label: t('app:interval.per_hour'),
value: 1
},
{
label: t('app:interval.2_hours'),
value: 2
},
{
label: t('app:interval.3_hours'),
value: 3
},
{
label: t('app:interval.4_hours'),
value: 4
},
{
label: t('app:interval.6_hours'),
value: 6
},
{
label: t('app:interval.12_hours'),
value: 12
}
];
};
const cronSelectList = useRef<MultipleSelectProps['list']>([
{
label: t('app:cron.every_day'),

View File

@@ -14,6 +14,7 @@ import {
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { useReqFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
async function handler(req: NextApiRequest) {
const {
@@ -98,4 +99,4 @@ async function handler(req: NextApiRequest) {
};
}
export default NextAPI(handler);
export default NextAPI(useReqFrequencyLimit(1, 2), handler);

View File

@@ -3,6 +3,7 @@ export const getDocPath = (path: string) => {
const feConfigs = useSystemStore.getState().feConfigs;
if (!feConfigs?.docUrl) return '';
if (!path.startsWith('/')) return path;
if (feConfigs.docUrl.endsWith('/')) return feConfigs.docUrl.slice(0, -1);
return feConfigs.docUrl + path;
};

View File

@@ -193,10 +193,11 @@ export const computedNodeInputReference = ({
if (!node) {
return;
}
const parentId = node.parentNodeId;
let sourceNodes: FlowNodeItemType[] = [];
// 根据 edge 获取所有的 source 节点source节点会继续向前递归获取
const findSourceNode = (nodeId: string) => {
const targetEdges = edges.filter((item) => item.target === nodeId);
const targetEdges = edges.filter((item) => item.target === nodeId || item.target === parentId);
targetEdges.forEach((edge) => {
const sourceNode = nodes.find((item) => item.nodeId === edge.source);
if (!sourceNode) return;