feat: Text check before synchronization (#689)

* fix: icon

* fix: web selector

* fix: web selector

* perf: link sync

* dev doc

* chomd doc

* perf: git intro

* 466 intro

* intro img

* add json editor (#5)

* team limit

* websync limit

* json editor

* text editor

* perf: search test

* change cq value type

* doc

* intro img

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/file/api.d.ts

@@ -12,4 +12,5 @@ export type UrlFetchParams = {
 export type UrlFetchResponse = {
   url: string;
   content: string;
+  selector?: string;
 }[];

packages/global/common/string/tiktoken/index.ts

@@ -35,7 +35,7 @@ export function countPromptTokens(
   const text = `${role}\n${prompt}`;
   try {
     const encodeText = enc.encode(text);
-    return encodeText.length + 3; // 补充 role 估算值
+    return encodeText.length + role.length; // 补充 role 估算值
   } catch (error) {
     return text.length;
   }

packages/global/common/system/types/index.d.ts

@@ -44,7 +44,8 @@ export type FastGPTFeConfigsType = {
     google?: string;
   };
   limit?: {
-    exportLimitMinutes?: number;
+    exportDatasetLimitMinutes?: number;
+    websiteSyncLimitMinuted?: number;
   };
   scripts?: { [key: string]: string }[];
   favicon?: string;