ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-24 13:53:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Check the url to avoid ssrf attacks (#3965)

Browse Source 
* fix: Check the url to avoid ssrf attacks

* Delete docSite/content/zh-cn/docs/development/upgrading/490.md
This commit is contained in:
Archer
2025-03-04 14:45:29 +08:00
committed by GitHub
parent e860c56b77
commit b4dda6a41b
4 changed files with 208 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
packages/plugins/src/fetchUrl/template.json
View File

@@ -16,7 +16,7 @@
"nodeId": "lmpb9v2lo2lk",
"name": "插件开始",
"intro": "自定义配置外部输入，使用插件时，仅暴露自定义配置的输入",
"avatar": "/imgs/workflow/input.png",
"avatar": "core/workflow/template/workflowStart",
"flowNodeType": "pluginInput",
"showStatus": false,
"position": {
@@ -26,14 +26,16 @@
"version": "481",
"inputs": [
{
"renderTypeList": ["reference"],
"renderTypeList": ["input", "reference"],
"selectedTypeIndex": 0,
"valueType": "string",
"key": "url",
"label": "url",
"description": "需要读取的网页链接",
"required": true,
"toolDescription": "需要读取的网页链接"
"toolDescription": "需要读取的网页链接",
"list": [],
"defaultValue": ""
}
],
"outputs": [
@@ -50,12 +52,12 @@
"nodeId": "i7uow4wj2wdp",
"name": "插件输出",
"intro": "自定义配置外部输出，使用插件时，仅暴露自定义配置的输出",
"avatar": "/imgs/workflow/output.png",
"avatar": "core/workflow/template/pluginOutput",
"flowNodeType": "pluginOutput",
"showStatus": false,
"position": {
"x": 1607.7142331269129,
"y": -150.8808596935447
"x": 1853.935047606551,
"y": -154.13661665265613
},
"version": "481",
"inputs": [
@@ -81,12 +83,12 @@
"nodeId": "ebLCxU43hHuZ",
"name": "HTTP 请求",
"intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）",
"avatar": "/imgs/workflow/http.png",
"avatar": "core/workflow/template/httpRequest",
"flowNodeType": "httpRequest468",
"showStatus": true,
"position": {
"x": 1050.9890727421412,
"y": -415.2085119990912
"x": 1054.2940501177068,
"y": -503.13661665265613
},
"version": "481",
"inputs": [
@@ -96,7 +98,7 @@
"valueType": "dynamic",
"label": "",
"required": false,
"description": "core.module.input.description.HTTP Dynamic Input",
"description": "common:core.module.input.description.HTTP Dynamic Input",
"customInputConfig": {
"selectValueTypeList": [
"string",
@@ -107,16 +109,19 @@
"arrayNumber",
"arrayBoolean",
"arrayObject",
"arrayAny",
"any",
"chatHistory",
"datasetQuote",
"dynamic",
"selectApp",
"selectDataset"
"selectDataset",
"selectApp"
],
"showDescription": false,
"showDefaultValue": true
}
},
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpMethod",
@@ -124,17 +129,33 @@
"valueType": "string",
"label": "",
"value": "POST",
"required": true
"required": true,
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpTimeout",
"renderTypeList": ["custom"],
"valueType": "number",
"label": "",
"value": 30,
"min": 5,
"max": 600,
"required": true,
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpReqUrl",
"renderTypeList": ["hidden"],
"valueType": "string",
"label": "",
"description": "core.module.input.description.Http Request Url",
"description": "common:core.module.input.description.Http Request Url",
"placeholder": "https://api.ai.com/getInventory",
"required": false,
"value": "fetchUrl"
"value": "fetchUrl",
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpHeader",
@@ -142,9 +163,11 @@
"valueType": "any",
"value": [],
"label": "",
"description": "core.module.input.description.Http Request Header",
"placeholder": "core.module.input.description.Http Request Header",
"required": false
"description": "common:core.module.input.description.Http Request Header",
"placeholder": "common:core.module.input.description.Http Request Header",
"required": false,
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpParams",
@@ -152,7 +175,9 @@
"valueType": "any",
"value": [],
"label": "",
"required": false
"required": false,
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpJsonBody",
@@ -160,7 +185,29 @@
"valueType": "any",
"value": "{\n \"url\": \"{{url}}\"\n}",
"label": "",
"required": false
"required": false,
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpFormBody",
"renderTypeList": ["hidden"],
"valueType": "any",
"value": [],
"label": "",
"required": false,
"debugLabel": "",
"toolDescription": ""
},
{
"key": "system_httpContentType",
"renderTypeList": ["hidden"],
"valueType": "string",
"value": "json",
"label": "",
"required": false,
"debugLabel": "",
"toolDescription": ""
},
{
"renderTypeList": ["reference"],
@@ -178,12 +225,13 @@
"arrayNumber",
"arrayBoolean",
"arrayObject",
"arrayAny",
"any",
"chatHistory",
"datasetQuote",
"dynamic",
"selectApp",
"selectDataset"
"selectDataset",
"selectApp"
],
"showDescription": false,
"showDefaultValue": true
@@ -193,6 +241,23 @@
}
],
"outputs": [
{
"id": "error",
"key": "error",
"label": "workflow:request_error",
"description": "HTTP请求错误信息，成功时返回空",
"valueType": "object",
"type": "static"
},
{
"id": "httpRawResponse",
"key": "httpRawResponse",
"required": true,
"label": "workflow:raw_response",
"description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
"valueType": "any",
"type": "static"
},
{
"id": "system_addOutputParam",
"key": "system_addOutputParam",
@@ -220,23 +285,6 @@
"showDefaultValue": true
}
},
{
"id": "error",
"key": "error",
"label": "请求错误",
"description": "HTTP请求错误信息，成功时返回空",
"valueType": "object",
"type": "static"
},
{
"id": "httpRawResponse",
"key": "httpRawResponse",
"label": "原始响应",
"required": true,
"description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
"valueType": "any",
"type": "static"
},
{
"id": "rH4tMV02robs",
"valueType": "string",
@@ -260,6 +308,34 @@
"sourceHandle": "ebLCxU43hHuZ-source-right",
"targetHandle": "i7uow4wj2wdp-target-left"
}
]
],
"chatConfig": {
"welcomeText": "",
"variables": [],
"questionGuide": {
"open": false,
"model": "gpt-4o-mini",
"customPrompt": "You are an AI assistant tasked with predicting the user's next question based on the conversation history. Your goal is to generate 3 potential questions that will guide the user to continue the conversation. When generating these questions, adhere to the following rules:\n\n1. Use the same language as the user's last question in the conversation history.\n2. Keep each question under 20 characters in length.\n\nAnalyze the conversation history provided to you and use it as context to generate relevant and engaging follow-up questions. Your predictions should be logical extensions of the current topic or related areas that the user might be interested in exploring further.\n\nRemember to maintain consistency in tone and style with the existing conversation while providing diverse options for the user to choose from. Your goal is to keep the conversation flowing naturally and help the user delve deeper into the subject matter or explore related topics."
},
"ttsConfig": {
"type": "web"
},
"whisperConfig": {
"open": false,
"autoSend": false,
"autoTTSResponse": false
},
"chatInputGuide": {
"open": false,
"textList": [],
"customUrl": ""
},
"instruction": "",
"autoExecute": {
"open": false,
"defaultPrompt": ""
},
"_id": "677b59849d672185a5671b45"
}
}
}

11
packages/service/common/string/cheerio.ts
View File

@@ -2,6 +2,7 @@ import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/ap
import * as cheerio from 'cheerio';
import axios from 'axios';
import { htmlToMarkdown } from './utils';
import { isInternalAddress } from '../system/utils';
export const cheerioToHtml = ({
fetchUrl,
@@ -75,6 +76,16 @@ export const urlsFetch = async ({
const response = await Promise.all(
urlList.map(async (url) => {
const isInternal = isInternalAddress(url);
if (isInternal) {
return {
url,
title: '',
content: 'Cannot fetch internal url',
selector: ''
};
}
try {
const fetchRes = await axios.get(url, {
timeout: 30000

63
packages/service/common/system/utils.ts Normal file
View File

@@ -0,0 +1,63 @@
import { SERVICE_LOCAL_HOST } from './tools';
export const isInternalAddress = (url: string): boolean => {
try {
const parsedUrl = new URL(url);
const hostname = parsedUrl.hostname;
const fullUrl = parsedUrl.toString();
// Check for localhost and common internal domains
if (hostname === SERVICE_LOCAL_HOST) {
return true;
}
// Metadata endpoints whitelist
const metadataEndpoints = [
// AWS
'http://169.254.169.254/latest/meta-data/',
// Azure
'http://169.254.169.254/metadata/instance?api-version=2021-02-01',
// GCP
'http://metadata.google.internal/computeMetadata/v1/',
// Alibaba Cloud
'http://100.100.100.200/latest/meta-data/',
// Tencent Cloud
'http://metadata.tencentyun.com/latest/meta-data/',
// Huawei Cloud
'http://169.254.169.254/latest/meta-data/'
];
if (metadataEndpoints.some((endpoint) => fullUrl.startsWith(endpoint))) {
return true;
}
// For non-metadata URLs, check if it's a domain name
const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/;
if (!ipv4Pattern.test(hostname)) {
return true;
}
// ... existing IP validation code ...
const parts = hostname.split('.').map(Number);
if (parts.length !== 4 || parts.some((part) => part < 0 || part > 255)) {
return false;
}
// Only allow public IP ranges
return (
parts[0] !== 0 &&
parts[0] !== 10 &&
parts[0] !== 127 &&
!(parts[0] === 169 && parts[1] === 254) &&
!(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) &&
!(parts[0] === 192 && parts[1] === 168) &&
!(parts[0] >= 224 && parts[0] <= 239) &&
!(parts[0] >= 240 && parts[0] <= 255) &&
!(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) &&
!(parts[0] === 9 && parts[1] === 0) &&
!(parts[0] === 11 && parts[1] === 0)
);
} catch {
return false; // If URL parsing fails, reject it as potentially unsafe
}
};