ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-24 13:53:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.6.7-alpha commit (#743)

Browse Source 
Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-01-19 11:17:28 +08:00
committed by GitHub
parent 8ee7407c4c
commit c031e6dcc9
324 changed files with 8509 additions and 4757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
packages/service/common/string/cheerio.ts
View File

@@ -64,41 +64,39 @@ export const urlsFetch = async ({
}: UrlFetchParams): Promise<UrlFetchResponse> => {
urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
const response = (
await Promise.all(
urlList.map(async (url) => {
try {
const fetchRes = await axios.get(url, {
timeout: 30000
});
const response = await Promise.all(
urlList.map(async (url) => {
try {
const fetchRes = await axios.get(url, {
timeout: 30000
});
const $ = cheerio.load(fetchRes.data);
const { title, html, usedSelector } = cheerioToHtml({
fetchUrl: url,
$,
selector
});
const md = await htmlToMarkdown(html);
const $ = cheerio.load(fetchRes.data);
const { title, html, usedSelector } = cheerioToHtml({
fetchUrl: url,
$,
selector
});
const md = await htmlToMarkdown(html);
return {
url,
title,
content: md,
selector: usedSelector
};
} catch (error) {
console.log(error, 'fetch error');
return {
url,
title,
content: md,
selector: usedSelector
};
} catch (error) {
console.log(error, 'fetch error');
return {
url,
title: '',
content: '',
selector: ''
};
}
})
)
).filter((item) => item.content);
return {
url,
title: '',
content: '',
selector: ''
};
}
})
);
return response;
};