From 74d58d562b391205f4bcb0d031743456dedc1bd9 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 25 Oct 2024 16:34:26 +0800 Subject: [PATCH] 4.8.12 test fix (#2988) * perf: qps limit * perf: http response data * perf: json path check * fix: ts * loop support reference parent variable --- packages/global/common/error/errorCode.ts | 8 +- packages/plugins/src/bing/template.json | 4 +- packages/service/common/middle/entry.ts | 5 +- packages/service/common/middle/qpsLimit.ts | 26 --- .../common/middle/reqFrequencyLimit.ts | 32 ++++ .../common/system/frequencyLimit/schema.ts | 2 +- .../common/system/frequencyLimit/utils.ts | 15 +- .../core/workflow/dispatch/loop/runLoop.ts | 4 +- .../core/workflow/dispatch/tools/http468.ts | 14 +- projects/app/.env.template | 1 + .../core/app/ScheduledTriggerConfig.tsx | 148 +++++++++--------- .../src/pages/api/core/dataset/searchTest.ts | 3 +- projects/app/src/web/common/system/doc.ts | 1 + projects/app/src/web/core/workflow/utils.ts | 3 +- 14 files changed, 144 insertions(+), 122 deletions(-) delete mode 100644 packages/service/common/middle/qpsLimit.ts create mode 100644 packages/service/common/middle/reqFrequencyLimit.ts diff --git a/packages/global/common/error/errorCode.ts b/packages/global/common/error/errorCode.ts index ff5a64c76..a11bc974b 100644 --- a/packages/global/common/error/errorCode.ts +++ b/packages/global/common/error/errorCode.ts @@ -41,7 +41,7 @@ export enum ERROR_ENUM { unAuthModel = 'unAuthModel', unAuthApiKey = 'unAuthApiKey', unAuthFile = 'unAuthFile', - QPSLimitExceed = 'QPSLimitExceed' + tooManyRequest = 'tooManyRequest' } export type ErrType = Record< @@ -69,10 +69,10 @@ export const ERROR_RESPONSE: Record< message: i18nT('common:code_error.error_message.403'), data: null }, - [ERROR_ENUM.QPSLimitExceed]: { + [ERROR_ENUM.tooManyRequest]: { code: 429, - statusText: ERROR_ENUM.QPSLimitExceed, - message: i18nT('common:code_error.error_code.429'), + statusText: ERROR_ENUM.tooManyRequest, + message: 'Too many request', data: null }, [ERROR_ENUM.insufficientQuota]: { diff --git a/packages/plugins/src/bing/template.json b/packages/plugins/src/bing/template.json index cf0170960..65780248c 100644 --- a/packages/plugins/src/bing/template.json +++ b/packages/plugins/src/bing/template.json @@ -8,7 +8,7 @@ "weight": 10, "courseUrl": "https://fael3z0zfze.feishu.cn/wiki/LsKAwOmtniA4vkkC259cmfxXnAc?fromScene=spaceOverview", "isTool": true, - "templateType": "tools", + "templateType": "search", "workflow": { "nodes": [ @@ -37,7 +37,7 @@ "required": true }, { - "renderTypeList": ["reference"], + "renderTypeList": ["input", "reference"], "selectedTypeIndex": 0, "valueType": "string", "canEdit": true, diff --git a/packages/service/common/middle/entry.ts b/packages/service/common/middle/entry.ts index 62bb2a4fe..2f49b543b 100644 --- a/packages/service/common/middle/entry.ts +++ b/packages/service/common/middle/entry.ts @@ -19,8 +19,11 @@ export const NextEntry = ({ beforeCallback = [] }: { beforeCallback?: Promise { - const ip = requestIp.getClientIp(req); - if (!ip) { - return; - } - try { - await authFrequencyLimit({ - eventId: 'ip-qps-limit' + ip, - maxAmount: limit, - expiredTime: addSeconds(new Date(), 1) - }); - } catch (_) { - return Promise.reject(ERROR_ENUM.QPSLimitExceed); - } - }; -} diff --git a/packages/service/common/middle/reqFrequencyLimit.ts b/packages/service/common/middle/reqFrequencyLimit.ts new file mode 100644 index 000000000..e2beb1a5e --- /dev/null +++ b/packages/service/common/middle/reqFrequencyLimit.ts @@ -0,0 +1,32 @@ +import { ApiRequestProps } from '../../type/next'; +import requestIp from 'request-ip'; +import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode'; +import { authFrequencyLimit } from '../system/frequencyLimit/utils'; +import { addSeconds } from 'date-fns'; +import { NextApiResponse } from 'next'; +import { jsonRes } from '../response'; + +// unit: times/s +// how to use? +// export default NextAPI(useQPSLimit(10), handler); // limit 10 times per second for a ip +export function useReqFrequencyLimit(seconds: number, limit: number) { + return async (req: ApiRequestProps, res: NextApiResponse) => { + const ip = requestIp.getClientIp(req); + if (!ip && process.env.USE_IP_LIMIT !== 'true') { + return; + } + try { + await authFrequencyLimit({ + eventId: 'ip-qps-limit' + ip, + maxAmount: limit, + expiredTime: addSeconds(new Date(), seconds) + }); + } catch (_) { + res.status(429); + jsonRes(res, { + code: 429, + message: ERROR_ENUM.tooManyRequest + }); + } + }; +} diff --git a/packages/service/common/system/frequencyLimit/schema.ts b/packages/service/common/system/frequencyLimit/schema.ts index 6ee8751c8..95bb9bd33 100644 --- a/packages/service/common/system/frequencyLimit/schema.ts +++ b/packages/service/common/system/frequencyLimit/schema.ts @@ -17,7 +17,7 @@ const FrequencyLimitSchema = new Schema({ }); try { - FrequencyLimitSchema.index({ eventId: 1 }, { unique: true }); + FrequencyLimitSchema.index({ eventId: 1, expiredTime: 1 }); FrequencyLimitSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 0 }); } catch (error) {} diff --git a/packages/service/common/system/frequencyLimit/utils.ts b/packages/service/common/system/frequencyLimit/utils.ts index b07ae85a4..9774ad450 100644 --- a/packages/service/common/system/frequencyLimit/utils.ts +++ b/packages/service/common/system/frequencyLimit/utils.ts @@ -1,6 +1,5 @@ import { AuthFrequencyLimitProps } from '@fastgpt/global/common/frequenctLimit/type'; import { MongoFrequencyLimit } from './schema'; -import { readFromSecondary } from '../../mongo/utils'; export const authFrequencyLimit = async ({ eventId, @@ -11,22 +10,24 @@ export const authFrequencyLimit = async ({ // 对应 eventId 的 account+1, 不存在的话，则创建一个 const result = await MongoFrequencyLimit.findOneAndUpdate( { - eventId + eventId, + expiredTime: { $gte: new Date() } }, { $inc: { amount: 1 }, + // If not exist, set the expiredTime $setOnInsert: { expiredTime } }, { upsert: true, - new: true, - ...readFromSecondary + new: true } - ); - + ).lean(); // 因为始终会返回+1的结果，所以这里不能直接等，需要多一个。 if (result.amount > maxAmount) { return Promise.reject(result); } - } catch (error) {} + } catch (error) { + console.log(error); + } }; diff --git a/packages/service/core/workflow/dispatch/loop/runLoop.ts b/packages/service/core/workflow/dispatch/loop/runLoop.ts index 350ba6c68..d6c25b761 100644 --- a/packages/service/core/workflow/dispatch/loop/runLoop.ts +++ b/packages/service/core/workflow/dispatch/loop/runLoop.ts @@ -32,8 +32,6 @@ export const dispatchLoop = async (props: Props): Promise => { return Promise.reject('Input array length cannot be greater than 50'); } - const runNodes = runtimeNodes.filter((node) => childrenNodeIdList.includes(node.nodeId)); - const outputValueArr = []; const loopDetail: ChatHistoryItemResType[] = []; let assistantResponses: AIChatItemValueItemType[] = []; @@ -43,7 +41,7 @@ export const dispatchLoop = async (props: Props): Promise => { for await (const item of loopInputArray) { const response = await dispatchWorkFlow({ ...props, - runtimeNodes: runNodes.map((node) => + runtimeNodes: runtimeNodes.map((node) => node.flowNodeType === FlowNodeTypeEnum.loopStart ? { ...node, diff --git a/packages/service/core/workflow/dispatch/tools/http468.ts b/packages/service/core/workflow/dispatch/tools/http468.ts index fb2f58c97..0fb9a9229 100644 --- a/packages/service/core/workflow/dispatch/tools/http468.ts +++ b/packages/service/core/workflow/dispatch/tools/http468.ts @@ -239,7 +239,15 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise { const key = item.key.startsWith('$') ? item.key : `$.${item.key}`; - results[item.key] = JSONPath({ path: key, json: formatResponse })[0]; + results[item.key] = (() => { + // 检查是否是简单的属性访问或单一索引访问 + if (/^\$(\.[a-zA-Z_][a-zA-Z0-9_]*)+$/.test(key) || /^\$(\[\d+\])+$/.test(key)) { + return JSONPath({ path: key, json: formatResponse })[0]; + } + + // 如果无法确定，默认返回数组 + return JSONPath({ path: key, json: formatResponse }); + })(); }); if (typeof formatResponse[NodeOutputKeyEnum.answerText] === 'string') { @@ -252,6 +260,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise 0 ? params : undefined, @@ -261,8 +270,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise 0 ? results : rawResponse, - [NodeOutputKeyEnum.httpRawResponse]: rawResponse, - ...results + [NodeOutputKeyEnum.httpRawResponse]: rawResponse }; } catch (error) { addLog.error('Http request error', error); diff --git a/projects/app/.env.template b/projects/app/.env.template index 81cecfa93..4d8d8175c 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -37,5 +37,6 @@ FE_DOMAIN=http://localhost:3000 LOG_LEVEL=debug STORE_LOG_LEVEL=warn +# 安全配置 # 工作流最大运行次数，避免极端的死循环情况 WORKFLOW_MAX_RUN_TIMES=500 \ No newline at end of file diff --git a/projects/app/src/components/core/app/ScheduledTriggerConfig.tsx b/projects/app/src/components/core/app/ScheduledTriggerConfig.tsx index 45cc9ce5c..ddef626ec 100644 --- a/projects/app/src/components/core/app/ScheduledTriggerConfig.tsx +++ b/projects/app/src/components/core/app/ScheduledTriggerConfig.tsx @@ -33,79 +33,6 @@ enum CronJobTypeEnum { } type CronType = 'month' | 'week' | 'day' | 'interval'; -const get24HoursOptions = () => { - return Array.from({ length: 24 }, (_, i) => ({ - label: `${i < 10 ? '0' : ''}${i}:00`, - value: i - })); -}; - -const getRoute = (i: number) => { - switch (i) { - case 0: - return 'app:week.Sunday'; - case 1: - return 'app:week.Monday'; - case 2: - return 'app:week.Tuesday'; - case 3: - return 'app:week.Wednesday'; - case 4: - return 'app:week.Thursday'; - case 5: - return 'app:week.Friday'; - case 6: - return 'app:week.Saturday'; - default: - return 'app:week.Sunday'; - } -}; - -const getWeekOptions = () => { - return Array.from({ length: 7 }, (_, i) => { - return { - label: i18nT(getRoute(i)), - value: i, - children: get24HoursOptions() - }; - }); -}; -const getMonthOptions = () => { - return Array.from({ length: 28 }, (_, i) => ({ - label: `${i + 1}` + i18nT('app:month.unit'), - value: i, - children: get24HoursOptions() - })); -}; -const getInterValOptions = () => { - // 每n小时 - return [ - { - label: i18nT('app:interval.per_hour'), - value: 1 - }, - { - label: i18nT('app:interval.2_hours'), - value: 2 - }, - { - label: i18nT('app:interval.3_hours'), - value: 3 - }, - { - label: i18nT('app:interval.4_hours'), - value: 4 - }, - { - label: i18nT('app:interval.6_hours'), - value: 6 - }, - { - label: i18nT('app:interval.12_hours'), - value: 12 - } - ]; -}; const defaultValue = ['day', 0, 0]; const defaultCronString = '0 0 * * *'; @@ -125,6 +52,81 @@ const ScheduledTriggerConfig = ({ const cronString = value?.cronString; const defaultPrompt = value?.defaultPrompt; + const get24HoursOptions = () => { + return Array.from({ length: 24 }, (_, i) => ({ + label: `${i < 10 ? '0' : ''}${i}:00`, + value: i + })); + }; + + const getRoute = (i: number) => { + const { t } = useTranslation(); + switch (i) { + case 0: + return t('app:week.Sunday'); + case 1: + return t('app:week.Monday'); + case 2: + return t('app:week.Tuesday'); + case 3: + return t('app:week.Wednesday'); + case 4: + return t('app:week.Thursday'); + case 5: + return t('app:week.Friday'); + case 6: + return t('app:week.Saturday'); + default: + return t('app:week.Sunday'); + } + }; + + const getWeekOptions = () => { + return Array.from({ length: 7 }, (_, i) => { + return { + label: getRoute(i), + value: i, + children: get24HoursOptions() + }; + }); + }; + const getMonthOptions = () => { + return Array.from({ length: 28 }, (_, i) => ({ + label: `${i + 1}` + t('app:month.unit'), + value: i, + children: get24HoursOptions() + })); + }; + const getInterValOptions = () => { + // 每n小时 + return [ + { + label: t('app:interval.per_hour'), + value: 1 + }, + { + label: t('app:interval.2_hours'), + value: 2 + }, + { + label: t('app:interval.3_hours'), + value: 3 + }, + { + label: t('app:interval.4_hours'), + value: 4 + }, + { + label: t('app:interval.6_hours'), + value: 6 + }, + { + label: t('app:interval.12_hours'), + value: 12 + } + ]; + }; + const cronSelectList = useRef([ { label: t('app:cron.every_day'), diff --git a/projects/app/src/pages/api/core/dataset/searchTest.ts b/projects/app/src/pages/api/core/dataset/searchTest.ts index 05f234400..559aaa51c 100644 --- a/projects/app/src/pages/api/core/dataset/searchTest.ts +++ b/projects/app/src/pages/api/core/dataset/searchTest.ts @@ -14,6 +14,7 @@ import { import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; +import { useReqFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit'; async function handler(req: NextApiRequest) { const { @@ -98,4 +99,4 @@ async function handler(req: NextApiRequest) { }; } -export default NextAPI(handler); +export default NextAPI(useReqFrequencyLimit(1, 2), handler); diff --git a/projects/app/src/web/common/system/doc.ts b/projects/app/src/web/common/system/doc.ts index 1d1ccd44b..b64b85881 100644 --- a/projects/app/src/web/common/system/doc.ts +++ b/projects/app/src/web/common/system/doc.ts @@ -3,6 +3,7 @@ export const getDocPath = (path: string) => { const feConfigs = useSystemStore.getState().feConfigs; if (!feConfigs?.docUrl) return ''; + if (!path.startsWith('/')) return path; if (feConfigs.docUrl.endsWith('/')) return feConfigs.docUrl.slice(0, -1); return feConfigs.docUrl + path; }; diff --git a/projects/app/src/web/core/workflow/utils.ts b/projects/app/src/web/core/workflow/utils.ts index 864b1cc65..9e416f367 100644 --- a/projects/app/src/web/core/workflow/utils.ts +++ b/projects/app/src/web/core/workflow/utils.ts @@ -193,10 +193,11 @@ export const computedNodeInputReference = ({ if (!node) { return; } + const parentId = node.parentNodeId; let sourceNodes: FlowNodeItemType[] = []; // 根据 edge 获取所有的 source 节点（source节点会继续向前递归获取） const findSourceNode = (nodeId: string) => { - const targetEdges = edges.filter((item) => item.target === nodeId); + const targetEdges = edges.filter((item) => item.target === nodeId || item.target === parentId); targetEdges.forEach((edge) => { const sourceNode = nodes.find((item) => item.nodeId === edge.source); if (!sourceNode) return;