perf: url fetch

This commit is contained in:
archer
2023-08-17 23:26:44 +08:00
parent 59ccc8565b
commit c5b5c440ca
5 changed files with 388 additions and 53 deletions

View File

@@ -59,7 +59,7 @@
"request-ip": "^3.3.0",
"sass": "^1.58.3",
"tunnel": "^0.0.6",
"jsdom": "^22.1.0"
"jsdom": "^22.1.0",
"winston": "^3.10.0",
"winston-mongodb": "^5.1.1",
"zustand": "^4.3.5"

324
client/pnpm-lock.yaml generated
View File

@@ -23,6 +23,9 @@ dependencies:
'@emotion/styled':
specifier: ^11.10.6
version: registry.npmmirror.com/@emotion/styled@11.10.6(@emotion/react@11.10.6)(@types/react@18.0.28)(react@18.2.0)
'@mozilla/readability':
specifier: ^0.4.4
version: registry.npmmirror.com/@mozilla/readability@0.4.4
'@next/font':
specifier: 13.1.6
version: registry.npmmirror.com/@next/font@13.1.6
@@ -68,6 +71,9 @@ dependencies:
js-cookie:
specifier: ^3.0.5
version: registry.npmmirror.com/js-cookie@3.0.5
jsdom:
specifier: ^22.1.0
version: registry.npmmirror.com/jsdom@22.1.0
jsonwebtoken:
specifier: ^9.0.0
version: registry.npmmirror.com/jsonwebtoken@9.0.0
@@ -175,6 +181,9 @@ devDependencies:
'@types/js-cookie':
specifier: ^3.0.3
version: registry.npmmirror.com/@types/js-cookie@3.0.3
'@types/jsdom':
specifier: ^21.1.1
version: registry.npmmirror.com/@types/jsdom@21.1.1
'@types/jsonwebtoken':
specifier: ^9.0.1
version: registry.npmmirror.com/@types/jsonwebtoken@9.0.1
@@ -4490,6 +4499,13 @@ packages:
tslib: registry.npmmirror.com/tslib@2.5.0
dev: false
registry.npmmirror.com/@mozilla/readability@0.4.4:
resolution: {integrity: sha512-MCgZyANpJ6msfvVMi6+A0UAsvZj//4OHREYUB9f2087uXHVoU+H+SWhuihvb1beKpM323bReQPRio0WNk2+V6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@mozilla/readability/-/readability-0.4.4.tgz}
name: '@mozilla/readability'
version: 0.4.4
engines: {node: '>=14.0.0'}
dev: false
registry.npmmirror.com/@next/env@13.1.6:
resolution: {integrity: sha512-s+W9Fdqh5MFk6ECrbnVmmAOwxKQuhGMT7xXHrkYIBMBcTiOqNWhv5KbJIboKR5STXxNXl32hllnvKaffzFaWQg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@next/env/-/env-13.1.6.tgz}
name: '@next/env'
@@ -5078,6 +5094,13 @@ packages:
use-sync-external-store: registry.npmmirror.com/use-sync-external-store@1.2.0(react@18.2.0)
dev: false
registry.npmmirror.com/@tootallnate/once@2.0.0:
resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@tootallnate/once/-/once-2.0.0.tgz}
name: '@tootallnate/once'
version: 2.0.0
engines: {node: '>= 10'}
dev: false
registry.npmmirror.com/@trysound/sax@0.2.0:
resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@trysound/sax/-/sax-0.2.0.tgz}
name: '@trysound/sax'
@@ -5377,6 +5400,16 @@ packages:
version: 3.0.3
dev: true
registry.npmmirror.com/@types/jsdom@21.1.1:
resolution: {integrity: sha512-cZFuoVLtzKP3gmq9eNosUL1R50U+USkbLtUQ1bYVgl/lKp0FZM7Cq4aIHAL8oIvQ17uSHi7jXPtfDOdjPwBE7A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/jsdom/-/jsdom-21.1.1.tgz}
name: '@types/jsdom'
version: 21.1.1
dependencies:
'@types/node': registry.npmmirror.com/@types/node@18.14.0
'@types/tough-cookie': registry.npmmirror.com/@types/tough-cookie@4.0.2
parse5: registry.npmmirror.com/parse5@7.1.2
dev: true
registry.npmmirror.com/@types/json5@0.0.29:
resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/json5/-/json5-0.0.29.tgz}
name: '@types/json5'
@@ -5507,6 +5540,12 @@ packages:
name: '@types/scheduler'
version: 0.16.3
registry.npmmirror.com/@types/tough-cookie@4.0.2:
resolution: {integrity: sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz}
name: '@types/tough-cookie'
version: 4.0.2
dev: true
registry.npmmirror.com/@types/triple-beam@1.3.2:
resolution: {integrity: sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/triple-beam/-/triple-beam-1.3.2.tgz}
name: '@types/triple-beam'
@@ -5628,6 +5667,12 @@ packages:
version: 0.2.2
dev: false
registry.npmmirror.com/abab@2.0.6:
resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abab/-/abab-2.0.6.tgz}
name: abab
version: 2.0.6
dev: false
registry.npmmirror.com/acorn-jsx@5.3.2(acorn@8.8.2):
resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz}
id: registry.npmmirror.com/acorn-jsx/5.3.2
@@ -5647,6 +5692,17 @@ packages:
hasBin: true
dev: true
registry.npmmirror.com/agent-base@6.0.2:
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agent-base/-/agent-base-6.0.2.tgz}
name: agent-base
version: 6.0.2
engines: {node: '>= 6.0.0'}
dependencies:
debug: registry.npmmirror.com/debug@4.3.4
transitivePeerDependencies:
- supports-color
dev: false
registry.npmmirror.com/ajv@6.12.6:
resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/ajv/-/ajv-6.12.6.tgz}
name: ajv
@@ -6410,6 +6466,15 @@ packages:
css-tree: registry.npmmirror.com/css-tree@1.1.3
dev: true
registry.npmmirror.com/cssstyle@3.0.0:
resolution: {integrity: sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cssstyle/-/cssstyle-3.0.0.tgz}
name: cssstyle
version: 3.0.0
engines: {node: '>=14'}
dependencies:
rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
dev: false
registry.npmmirror.com/csstype@3.1.2:
resolution: {integrity: sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/csstype/-/csstype-3.1.2.tgz}
name: csstype
@@ -6775,6 +6840,17 @@ packages:
version: 1.0.8
dev: true
registry.npmmirror.com/data-urls@4.0.0:
resolution: {integrity: sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/data-urls/-/data-urls-4.0.0.tgz}
name: data-urls
version: 4.0.0
engines: {node: '>=14'}
dependencies:
abab: registry.npmmirror.com/abab@2.0.6
whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
dev: false
registry.npmmirror.com/date-fns@2.30.0:
resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/date-fns/-/date-fns-2.30.0.tgz}
name: date-fns
@@ -6816,6 +6892,12 @@ packages:
dependencies:
ms: registry.npmmirror.com/ms@2.1.2
registry.npmmirror.com/decimal.js@10.4.3:
resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decimal.js/-/decimal.js-10.4.3.tgz}
name: decimal.js
version: 10.4.3
dev: false
registry.npmmirror.com/decode-named-character-reference@1.0.2:
resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz}
name: decode-named-character-reference
@@ -7001,6 +7083,15 @@ packages:
version: 2.3.0
dev: true
registry.npmmirror.com/domexception@4.0.0:
resolution: {integrity: sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domexception/-/domexception-4.0.0.tgz}
name: domexception
version: 4.0.0
engines: {node: '>=12'}
dependencies:
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
dev: false
registry.npmmirror.com/domhandler@4.3.1:
resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domhandler/-/domhandler-4.3.1.tgz}
name: domhandler
@@ -7095,7 +7186,6 @@ packages:
name: entities
version: 4.5.0
engines: {node: '>=0.12'}
dev: true
registry.npmmirror.com/error-ex@1.3.2:
resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/error-ex/-/error-ex-1.3.2.tgz}
@@ -8161,6 +8251,15 @@ packages:
react-is: registry.npmmirror.com/react-is@16.13.1
dev: false
registry.npmmirror.com/html-encoding-sniffer@3.0.0:
resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz}
name: html-encoding-sniffer
version: 3.0.0
engines: {node: '>=12'}
dependencies:
whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
dev: false
registry.npmmirror.com/html-parse-stringify@3.0.1:
resolution: {integrity: sha512-KknJ50kTInJ7qIScF3jeaFRpMpE8/lfiTdzf/twXyPBLAGrLRTmkz3AdTnKeh40X8k9L2fdYwEp/42WGXIRGcg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-parse-stringify/-/html-parse-stringify-3.0.1.tgz}
name: html-parse-stringify
@@ -8169,6 +8268,31 @@ packages:
void-elements: registry.npmmirror.com/void-elements@3.1.0
dev: false
registry.npmmirror.com/http-proxy-agent@5.0.0:
resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz}
name: http-proxy-agent
version: 5.0.0
engines: {node: '>= 6'}
dependencies:
'@tootallnate/once': registry.npmmirror.com/@tootallnate/once@2.0.0
agent-base: registry.npmmirror.com/agent-base@6.0.2
debug: registry.npmmirror.com/debug@4.3.4
transitivePeerDependencies:
- supports-color
dev: false
registry.npmmirror.com/https-proxy-agent@5.0.1:
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz}
name: https-proxy-agent
version: 5.0.1
engines: {node: '>= 6'}
dependencies:
agent-base: registry.npmmirror.com/agent-base@6.0.2
debug: registry.npmmirror.com/debug@4.3.4
transitivePeerDependencies:
- supports-color
dev: false
registry.npmmirror.com/human-signals@2.1.0:
resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/human-signals/-/human-signals-2.1.0.tgz}
name: human-signals
@@ -8508,6 +8632,12 @@ packages:
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/is-potential-custom-element-name@1.0.1:
resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz}
name: is-potential-custom-element-name
version: 1.0.1
dev: false
registry.npmmirror.com/is-regex@1.1.4:
resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-regex/-/is-regex-1.1.4.tgz}
name: is-regex
@@ -8653,6 +8783,46 @@ packages:
argparse: registry.npmmirror.com/argparse@2.0.1
dev: true
registry.npmmirror.com/jsdom@22.1.0:
resolution: {integrity: sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsdom/-/jsdom-22.1.0.tgz}
name: jsdom
version: 22.1.0
engines: {node: '>=16'}
peerDependencies:
canvas: ^2.5.0
peerDependenciesMeta:
canvas:
optional: true
dependencies:
abab: registry.npmmirror.com/abab@2.0.6
cssstyle: registry.npmmirror.com/cssstyle@3.0.0
data-urls: registry.npmmirror.com/data-urls@4.0.0
decimal.js: registry.npmmirror.com/decimal.js@10.4.3
domexception: registry.npmmirror.com/domexception@4.0.0
form-data: registry.npmmirror.com/form-data@4.0.0
html-encoding-sniffer: registry.npmmirror.com/html-encoding-sniffer@3.0.0
http-proxy-agent: registry.npmmirror.com/http-proxy-agent@5.0.0
https-proxy-agent: registry.npmmirror.com/https-proxy-agent@5.0.1
is-potential-custom-element-name: registry.npmmirror.com/is-potential-custom-element-name@1.0.1
nwsapi: registry.npmmirror.com/nwsapi@2.2.7
parse5: registry.npmmirror.com/parse5@7.1.2
rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
saxes: registry.npmmirror.com/saxes@6.0.0
symbol-tree: registry.npmmirror.com/symbol-tree@3.2.4
tough-cookie: registry.npmmirror.com/tough-cookie@4.1.3
w3c-xmlserializer: registry.npmmirror.com/w3c-xmlserializer@4.0.0
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
ws: registry.npmmirror.com/ws@8.13.0
xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
transitivePeerDependencies:
- bufferutil
- supports-color
- utf-8-validate
dev: false
registry.npmmirror.com/jsesc@0.5.0:
resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsesc/-/jsesc-0.5.0.tgz}
name: jsesc
@@ -9835,6 +10005,12 @@ packages:
boolbase: registry.npmmirror.com/boolbase@1.0.0
dev: true
registry.npmmirror.com/nwsapi@2.2.7:
resolution: {integrity: sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/nwsapi/-/nwsapi-2.2.7.tgz}
name: nwsapi
version: 2.2.7
dev: false
registry.npmmirror.com/object-assign@4.1.1:
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz}
name: object-assign
@@ -10076,6 +10252,13 @@ packages:
version: 6.0.1
dev: false
registry.npmmirror.com/parse5@7.1.2:
resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parse5/-/parse5-7.1.2.tgz}
name: parse5
version: 7.1.2
dependencies:
entities: registry.npmmirror.com/entities@4.5.0
registry.npmmirror.com/path-exists@4.0.0:
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz}
name: path-exists
@@ -10286,6 +10469,12 @@ packages:
version: 1.1.0
dev: false
registry.npmmirror.com/psl@1.9.0:
resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/psl/-/psl-1.9.0.tgz}
name: psl
version: 1.9.0
dev: false
registry.npmmirror.com/punycode@2.3.0:
resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/punycode/-/punycode-2.3.0.tgz}
name: punycode
@@ -10301,6 +10490,12 @@ packages:
side-channel: registry.npmmirror.com/side-channel@1.0.4
dev: false
registry.npmmirror.com/querystringify@2.2.0:
resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/querystringify/-/querystringify-2.2.0.tgz}
name: querystringify
version: 2.2.0
dev: false
registry.npmmirror.com/queue-microtask@1.2.3:
resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/queue-microtask/-/queue-microtask-1.2.3.tgz}
name: queue-microtask
@@ -10765,6 +10960,12 @@ packages:
engines: {node: '>=4'}
dev: false
registry.npmmirror.com/requires-port@1.0.0:
resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/requires-port/-/requires-port-1.0.0.tgz}
name: requires-port
version: 1.0.0
dev: false
registry.npmmirror.com/resolve-from@4.0.0:
resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz}
name: resolve-from
@@ -10820,6 +11021,12 @@ packages:
version: 3.0.2
dev: false
registry.npmmirror.com/rrweb-cssom@0.6.0:
resolution: {integrity: sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz}
name: rrweb-cssom
version: 0.6.0
dev: false
registry.npmmirror.com/run-applescript@5.0.0:
resolution: {integrity: sha512-XcT5rBksx1QdIhlFOCtgZkB99ZEouFZ1E2Kc2LHqNW13U3/74YGdkQRmThTwxy4QIyookibDKYZOPqX//6BlAg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/run-applescript/-/run-applescript-5.0.0.tgz}
name: run-applescript
@@ -10916,6 +11123,15 @@ packages:
version: 1.1.6
dev: false
registry.npmmirror.com/saxes@6.0.0:
resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/saxes/-/saxes-6.0.0.tgz}
name: saxes
version: 6.0.0
engines: {node: '>=v12.22.7'}
dependencies:
xmlchars: registry.npmmirror.com/xmlchars@2.2.0
dev: false
registry.npmmirror.com/scheduler@0.23.0:
resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/scheduler/-/scheduler-0.23.0.tgz}
name: scheduler
@@ -11277,6 +11493,12 @@ packages:
stable: registry.npmmirror.com/stable@0.1.8
dev: true
registry.npmmirror.com/symbol-tree@3.2.4:
resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/symbol-tree/-/symbol-tree-3.2.4.tgz}
name: symbol-tree
version: 3.2.4
dev: false
registry.npmmirror.com/synckit@0.8.5:
resolution: {integrity: sha512-L1dapNV6vu2s/4Sputv8xGsCdAVlb5nRDMFU/E27D44l5U6cw1g0dGd45uLc+OXjNMmF4ntiMdCimzcjFKQI8Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/synckit/-/synckit-0.8.5.tgz}
name: synckit
@@ -11339,6 +11561,18 @@ packages:
version: 1.0.6
dev: false
registry.npmmirror.com/tough-cookie@4.1.3:
resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tough-cookie/-/tough-cookie-4.1.3.tgz}
name: tough-cookie
version: 4.1.3
engines: {node: '>=6'}
dependencies:
psl: registry.npmmirror.com/psl@1.9.0
punycode: registry.npmmirror.com/punycode@2.3.0
universalify: registry.npmmirror.com/universalify@0.2.0
url-parse: registry.npmmirror.com/url-parse@1.5.10
dev: false
registry.npmmirror.com/tr46@3.0.0:
resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-3.0.0.tgz}
name: tr46
@@ -11348,6 +11582,15 @@ packages:
punycode: registry.npmmirror.com/punycode@2.3.0
dev: false
registry.npmmirror.com/tr46@4.1.1:
resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-4.1.1.tgz}
name: tr46
version: 4.1.1
engines: {node: '>=14'}
dependencies:
punycode: registry.npmmirror.com/punycode@2.3.0
dev: false
registry.npmmirror.com/trim-lines@3.0.1:
resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/trim-lines/-/trim-lines-3.0.1.tgz}
name: trim-lines
@@ -11590,6 +11833,13 @@ packages:
unist-util-visit-parents: registry.npmmirror.com/unist-util-visit-parents@5.1.3
dev: false
registry.npmmirror.com/universalify@0.2.0:
resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/universalify/-/universalify-0.2.0.tgz}
name: universalify
version: 0.2.0
engines: {node: '>= 4.0.0'}
dev: false
registry.npmmirror.com/untildify@4.0.0:
resolution: {integrity: sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/untildify/-/untildify-4.0.0.tgz}
name: untildify
@@ -11618,6 +11868,15 @@ packages:
punycode: registry.npmmirror.com/punycode@2.3.0
dev: true
registry.npmmirror.com/url-parse@1.5.10:
resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz}
name: url-parse
version: 1.5.10
dependencies:
querystringify: registry.npmmirror.com/querystringify@2.2.0
requires-port: registry.npmmirror.com/requires-port@1.0.0
dev: false
registry.npmmirror.com/use-callback-ref@1.3.0(@types/react@18.0.28)(react@18.2.0):
resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/use-callback-ref/-/use-callback-ref-1.3.0.tgz}
id: registry.npmmirror.com/use-callback-ref/1.3.0
@@ -11743,6 +12002,15 @@ packages:
engines: {node: '>=0.10.0'}
dev: false
registry.npmmirror.com/w3c-xmlserializer@4.0.0:
resolution: {integrity: sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz}
name: w3c-xmlserializer
version: 4.0.0
engines: {node: '>=14'}
dependencies:
xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
dev: false
registry.npmmirror.com/web-namespaces@2.0.1:
resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/web-namespaces/-/web-namespaces-2.0.1.tgz}
name: web-namespaces
@@ -11762,6 +12030,22 @@ packages:
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/whatwg-encoding@2.0.0:
resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz}
name: whatwg-encoding
version: 2.0.0
engines: {node: '>=12'}
dependencies:
iconv-lite: registry.npmmirror.com/iconv-lite@0.6.3
dev: false
registry.npmmirror.com/whatwg-mimetype@3.0.0:
resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz}
name: whatwg-mimetype
version: 3.0.0
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/whatwg-url@11.0.0:
resolution: {integrity: sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-11.0.0.tgz}
name: whatwg-url
@@ -11772,6 +12056,16 @@ packages:
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
dev: false
registry.npmmirror.com/whatwg-url@12.0.1:
resolution: {integrity: sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-12.0.1.tgz}
name: whatwg-url
version: 12.0.1
engines: {node: '>=14'}
dependencies:
tr46: registry.npmmirror.com/tr46@4.1.1
webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
dev: false
registry.npmmirror.com/which-boxed-primitive@1.0.2:
resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz}
name: which-boxed-primitive
@@ -11882,6 +12176,28 @@ packages:
name: wrappy
version: 1.0.2
registry.npmmirror.com/ws@8.13.0:
resolution: {integrity: sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/ws/-/ws-8.13.0.tgz}
name: ws
version: 8.13.0
engines: {node: '>=10.0.0'}
peerDependencies:
bufferutil: ^4.0.1
utf-8-validate: '>=5.0.2'
peerDependenciesMeta:
bufferutil:
optional: true
utf-8-validate:
optional: true
dev: false
registry.npmmirror.com/xml-name-validator@4.0.0:
resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz}
name: xml-name-validator
version: 4.0.0
engines: {node: '>=12'}
dev: false
registry.npmmirror.com/xmlbuilder@10.1.1:
resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz}
name: xmlbuilder
@@ -11889,6 +12205,12 @@ packages:
engines: {node: '>=4.0'}
dev: false
registry.npmmirror.com/xmlchars@2.2.0:
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlchars/-/xmlchars-2.2.0.tgz}
name: xmlchars
version: 2.2.0
dev: false
registry.npmmirror.com/xtend@4.0.2:
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz}
name: xtend

View File

@@ -1,51 +0,0 @@
// pages/api/fetchContent.ts
import { NextApiRequest, NextApiResponse } from 'next';
import axios from 'axios';
import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import { jsonRes } from '@/service/response';
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
const { url } = req.body;
if (!url) {
return res.status(400).json({ error: 'URL is required' });
}
try {
const response = await axios.get(url, {
httpsAgent: new (require('https').Agent)({
rejectUnauthorized: false,
}),
});
const dom = new JSDOM(response.data, {
url,
contentType: 'text/html',
});
const reader = new Readability(dom.window.document);
const article = reader.parse();
if (!article) {
jsonRes(res, {
code: 500,
error: '页面获取失败或页面为空'
});
return;
}
jsonRes(res, {
code: 200,
data: article.content
});
} catch (error:any) {
jsonRes(res, {
code: 500,
error: error
});
}
};
export default fetchContent;

View File

@@ -0,0 +1,64 @@
// pages/api/fetchContent.ts
import { NextApiRequest, NextApiResponse } from 'next';
import axios from 'axios';
import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
type FetchResultItem = {
url: string;
title: string;
content: string;
};
export type UrlFetchResponse = FetchResultItem[];
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try {
const { urlList = [] } = req.body as { urlList: string[] };
if (!urlList || urlList.length === 0) {
throw new Error('urlList is empty');
}
await authUser({ req });
const response = (
await Promise.allSettled(
urlList.map(async (url) => {
const fetchRes = await axios.get(url, {
timeout: 30000
});
const dom = new JSDOM(fetchRes.data, {
url,
contentType: 'text/html'
});
const reader = new Readability(dom.window.document);
const article = reader.parse();
return {
url,
title: article?.title || '',
content: article?.textContent || ''
};
})
)
)
.filter((item) => item.status === 'fulfilled')
.map((item: any) => item.value)
.filter((item) => item.content);
jsonRes<UrlFetchResponse>(res, {
data: response
});
} catch (error: any) {
jsonRes(res, {
code: 500,
error: error
});
}
};
export default fetchContent;

Binary file not shown.

Before

Width:  |  Height:  |  Size: 100 KiB