页面抓取 (#185)

* Create GLM2对接教程.md

* 添加GLM2接入教程

* Delete GLM2对接教程.md

* Delete image.png

* Delete openai_api.py

* Delete openai_api_int4.py

* Delete openai_api_int8.py

* Create GLM2对接教程.md

* 添加ChatGLM2接口

* Delete openai_api_int4.py

* Delete openai_api_int8.py

* Update openai_api.py

* Update GLM2对接教程.md

* 页面抓取接口

* Update package.json

* Update fetchContent.ts

* Delete GLM2对接教程.md

* Delete openai_api.py

---------

Co-authored-by: Archer <545436317@qq.com>

client/src/pages/api/plugins/kb/fetchContent.ts

@@ -0,0 +1,51 @@
+// pages/api/fetchContent.ts
+import { NextApiRequest, NextApiResponse } from 'next';
+import axios from 'axios';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+import { jsonRes } from '@/service/response';
+
+const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
+  const { url } = req.body;
+  if (!url) {
+    return res.status(400).json({ error: 'URL is required' });
+  }
+
+  try {
+    const response = await axios.get(url, {
+      httpsAgent: new (require('https').Agent)({
+        rejectUnauthorized: false,
+      }),
+    });
+
+    const dom = new JSDOM(response.data, {
+      url,
+      contentType: 'text/html',
+    });
+
+    const reader = new Readability(dom.window.document);
+    const article = reader.parse();
+
+    if (!article) {
+        jsonRes(res, {
+            code: 500,
+            error: '页面获取失败或页面为空'
+          });
+          return;
+    }
+
+    jsonRes(res, {
+        code: 200,
+        data: article.content
+      });
+    
+  } catch (error:any) {
+    jsonRes(res, {
+        code: 500,
+        error: error
+      });
+    }
+  
+};
+
+export default fetchContent;