ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-27 08:25:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

add env proxypool (#3939)

Browse Source
This commit is contained in:
gggaaallleee
2025-03-02 17:50:03 +08:00
committed by GitHub
parent abc6dffb41
commit 113e8f711f
2 changed files with 13 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
plugins/webcrawler/SPIDER/.env.example
View File

@@ -19,3 +19,5 @@ BLACKLIST = [".gov.cn",".edu.cn"]
STD_TTL=3600
EXPIRE_AFTER_SECONDS=9000
#VALIDATE_PROXY=[{"ip":"","port":},{"ip":"","port":}]

37
plugins/webcrawler/SPIDER/src/utils/setupPage.ts
View File

@@ -1,6 +1,8 @@
import { Page } from 'puppeteer';
import randomUseragent from 'random-useragent';
import dotenv from 'dotenv';
dotenv.config();
const getRandomUserAgent = () => {
return randomUseragent.getRandom();
};
@@ -11,31 +13,10 @@ const getRandomPlatform = () => {
};
//代理池
const validateproxy = [
{ ip: "39.102.210.222", port: 8080 },
{ ip: "8.130.71.75", port: 8080 },
{ ip: "39.102.214.208", port: 9999 },
{ ip: "39.104.59.56", port: 8080 },
{ ip: "8.130.37.235", port: 3128 },
{ ip: "8.138.131.110", port: 8080 },
{ ip: "8.140.105.75", port: 8009 },
{ ip: "114.80.38.120", port: 3081 },
{ ip: "8.148.23.165", port: 8081 },
{ ip: "119.96.72.199", port: 59394 },
{ ip: "120.55.14.137", port: 80 },
{ ip: "47.116.181.146", port: 5060 },
{ ip: "39.102.214.199", port: 3128 },
{ ip: "47.121.183.107", port: 8080 },
{ ip: "39.104.16.201", port: 8080 },
{ ip: "39.102.209.163", port: 10002 },
{ ip: "101.201.76.157", port: 9090 },
{ ip: "122.224.124.26", port: 12080 },
{ ip: "180.105.244.199", port: 1080 },
{ ip: "119.3.113.150", port: 9094 }
];
const validateproxy = process.env.VALIDATE_PROXY ? JSON.parse(process.env.VALIDATE_PROXY) : [];
const getRandomProxy = () => {
return validateproxy[Math.floor(Math.random() * validateproxy.length)];
return validateproxy.length > 0 ? validateproxy[Math.floor(Math.random() * validateproxy.length)] : null;
};
const getRandomLanguages = () => {
@@ -49,10 +30,12 @@ const getRandomLanguages = () => {
export const setupPage = async (page: Page): Promise<void> => {
const proxy = getRandomProxy();
await page.authenticate({
username: proxy.ip,
password: proxy.port.toString()
});
if (proxy) {
await page.authenticate({
username: proxy.ip,
password: proxy.port.toString()
});
}
await page.evaluateOnNewDocument(() => {
const newProto = (navigator as any).__proto__;