v4.6.5 (#620)

packages/service/common/api/plusRequest.ts

@@ -71,7 +71,7 @@ instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err)
 
 export function request(url: string, data: any, config: ConfigType, method: Method): any {
   if (!global.systemEnv || !global.systemEnv?.pluginBaseUrl) {
-    console.log('未部署商业版接口');
+    console.log('未部署商业版接口', url);
     return Promise.reject('The The request was denied...');
   }
 

packages/service/common/string/cheerio.ts

@@ -0,0 +1,95 @@
+import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api';
+import * as cheerio from 'cheerio';
+import axios from 'axios';
+import { htmlToMarkdown } from './markdown';
+
+export const cheerioToHtml = ({
+  fetchUrl,
+  $,
+  selector
+}: {
+  fetchUrl: string;
+  $: cheerio.CheerioAPI;
+  selector?: string;
+}) => {
+  // get origin url
+  const originUrl = new URL(fetchUrl).origin;
+
+  const selectDom = $(selector || 'body');
+
+  // remove i element
+  selectDom.find('i,script').remove();
+
+  // remove empty a element
+  selectDom
+    .find('a')
+    .filter((i, el) => {
+      return $(el).text().trim() === '' && $(el).children().length === 0;
+    })
+    .remove();
+
+  // if link,img startWith /, add origin url
+  selectDom.find('a').each((i, el) => {
+    const href = $(el).attr('href');
+    if (href && href.startsWith('/')) {
+      $(el).attr('href', originUrl + href);
+    }
+  });
+  selectDom.find('img').each((i, el) => {
+    const src = $(el).attr('src');
+    if (src && src.startsWith('/')) {
+      $(el).attr('src', originUrl + src);
+    }
+  });
+
+  const html = selectDom
+    .map((item, dom) => {
+      return $(dom).html();
+    })
+    .get()
+    .join('\n');
+
+  return html;
+};
+export const urlsFetch = async ({
+  urlList,
+  selector
+}: UrlFetchParams): Promise<UrlFetchResponse> => {
+  urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
+
+  const response = (
+    await Promise.all(
+      urlList.map(async (url) => {
+        try {
+          const fetchRes = await axios.get(url, {
+            timeout: 30000
+          });
+
+          const $ = cheerio.load(fetchRes.data);
+
+          const md = await htmlToMarkdown(
+            cheerioToHtml({
+              fetchUrl: url,
+              $,
+              selector
+            })
+          );
+
+          return {
+            url,
+            content: md
+          };
+        } catch (error) {
+          console.log(error, 'fetch error');
+
+          return {
+            url,
+            content: ''
+          };
+        }
+      })
+    )
+  ).filter((item) => item.content);
+
+  return response;
+};

packages/service/common/string/markdown.ts

@@ -0,0 +1,23 @@
+import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
+import { Worker } from 'worker_threads';
+import { getWorkerPath } from './utils';
+
+/* html string to markdown */
+export const htmlToMarkdown = (html?: string | null) =>
+  new Promise<string>((resolve, reject) => {
+    if (!html) return resolve('');
+
+    const start = Date.now();
+
+    // worker
+    const worker = new Worker(getWorkerPath('html2md'));
+
+    worker.on('message', (md: string) => {
+      resolve(simpleMarkdownText(md));
+    });
+    worker.on('error', (err) => {
+      reject(err);
+    });
+
+    worker.postMessage(html);
+  });

packages/service/common/string/utils.ts

@@ -0,0 +1,9 @@
+export const getWorkerPath = (name: string) => {
+  // @ts-ignore
+  const isSubModule = !!global?.systemConfig;
+
+  const isProd = process.env.NODE_ENV === 'production';
+  return isProd
+    ? `/app/worker/${name}.js`
+    : `../../${isSubModule ? 'FastGPT/' : ''}/worker/${name}.js`;
+};

packages/service/core/chat/chatItemSchema.ts

@@ -61,8 +61,8 @@ const ChatItemSchema = new Schema({
   userBadFeedback: {
     type: String
   },
-  robotBadFeedback: {
-    type: String
+  customFeedbacks: {
+    type: [String]
   },
   adminFeedback: {
     type: {
@@ -86,7 +86,7 @@ try {
   ChatItemSchema.index({ chatId: 1 });
   ChatItemSchema.index({ userGoodFeedback: 1 });
   ChatItemSchema.index({ userBadFeedback: 1 });
-  ChatItemSchema.index({ robotBadFeedback: 1 });
+  ChatItemSchema.index({ customFeedbacks: 1 });
   ChatItemSchema.index({ adminFeedback: 1 });
 } catch (error) {
   console.log(error);

packages/service/core/chat/controller.ts

@@ -1,5 +1,6 @@
 import type { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { MongoChatItem } from './chatItemSchema';
+import { addLog } from '../../common/system/log';
 
 export async function getChatItems({
   chatId,
@@ -20,3 +21,29 @@ export async function getChatItems({
 
   return { history };
 }
+
+export const addCustomFeedbacks = async ({
+  chatId,
+  chatItemId,
+  feedbacks
+}: {
+  chatId?: string;
+  chatItemId?: string;
+  feedbacks: string[];
+}) => {
+  if (!chatId || !chatItemId) return;
+
+  try {
+    await MongoChatItem.findOneAndUpdate(
+      {
+        chatId,
+        dataId: chatItemId
+      },
+      {
+        $push: { customFeedbacks: { $each: feedbacks } }
+      }
+    );
+  } catch (error) {
+    addLog.error('addCustomFeedbacks error', error);
+  }
+};

packages/service/core/dataset/collection/utils.ts

@@ -4,7 +4,7 @@ import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { MongoDatasetTraining } from '../training/schema';
-import { urlsFetch } from '@fastgpt/global/common/file/tools';
+import { urlsFetch } from '../../../common/string/cheerio';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
 
 /**
@@ -105,7 +105,8 @@ export const loadingOneChunkCollection = async ({
   // split data
   const { chunks } = splitText2Chunks({
     text: newRawText,
-    chunkLen: collection.chunkSize || 512
+    chunkLen: collection.chunkSize || 512,
+    countTokens: false
   });
 
   // insert to training queue

packages/service/core/plugin/controller.ts

@@ -44,6 +44,7 @@ const getPluginTemplateById = async (id: string): Promise<PluginTemplateType> =>
       name: item.name,
       avatar: item.avatar,
       intro: item.intro,
+      showStatus: true,
       source: PluginSourceEnum.personal,
       modules: item.modules,
       templateType: ModuleTemplateTypeEnum.personalPlugin
@@ -67,7 +68,7 @@ export async function getPluginPreviewModule({
     avatar: plugin.avatar,
     name: plugin.name,
     intro: plugin.intro,
-    showStatus: true,
+    showStatus: plugin.showStatus,
     ...plugin2ModuleIO(plugin.id, plugin.modules)
   };
 }

packages/service/package.json

@@ -3,22 +3,24 @@
   "version": "1.0.0",
   "dependencies": {
     "@fastgpt/global": "workspace:*",
-    "axios": "^1.5.1",
     "cookie": "^0.5.0",
     "encoding": "^0.1.13",
     "jsonwebtoken": "^9.0.2",
     "mongoose": "^7.0.2",
     "nanoid": "^4.0.1",
+    "dayjs": "^1.11.7",
     "next": "13.5.2",
+    "multer": "1.4.5-lts.1",
+    "axios": "^1.5.1",
+    "cheerio": "1.0.0-rc.12",
     "nextjs-cors": "^2.1.2",
     "pg": "^8.10.0",
-    "tunnel": "^0.0.6",
-    "dayjs": "^1.11.7"
+    "tunnel": "^0.0.6"
   },
   "devDependencies": {
     "@types/cookie": "^0.5.2",
     "@types/jsonwebtoken": "^9.0.3",
-    "@types/node": "^20.8.5",
+    "@types/multer": "^1.4.10",
     "@types/pg": "^8.6.6",
     "@types/tunnel": "^0.0.4"
   }

packages/service/support/permission/auth/common.ts

@@ -31,3 +31,11 @@ export async function authCertOrShareId({
     canWrite: false
   };
 }
+
+/* auth the request from local service */
+export const authRequestFromLocal = ({ req }: AuthModeType) => {
+  const host = `${process.env.HOSTNAME || 'localhost'}:${process.env.PORT || 3000}`;
+  if (host !== req.headers.host) {
+    return Promise.reject('Invalid request');
+  }
+};