fix: Check the url to avoid ssrf attacks (#3965)

* fix: Check the url to avoid ssrf attacks

* Delete docSite/content/zh-cn/docs/development/upgrading/490.md
This commit is contained in:
Archer
2025-03-04 14:45:29 +08:00
committed by GitHub
parent e860c56b77
commit b4dda6a41b
4 changed files with 208 additions and 53 deletions

View File

@@ -2,6 +2,7 @@ import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/ap
import * as cheerio from 'cheerio';
import axios from 'axios';
import { htmlToMarkdown } from './utils';
import { isInternalAddress } from '../system/utils';
export const cheerioToHtml = ({
fetchUrl,
@@ -75,6 +76,16 @@ export const urlsFetch = async ({
const response = await Promise.all(
urlList.map(async (url) => {
const isInternal = isInternalAddress(url);
if (isInternal) {
return {
url,
title: '',
content: 'Cannot fetch internal url',
selector: ''
};
}
try {
const fetchRes = await axios.get(url, {
timeout: 30000