feat: Text check before synchronization (#689)

* fix: icon

* fix: web selector

* fix: web selector

* perf: link sync

* dev doc

* chomd doc

* perf: git intro

* 466 intro

* intro img

* add json editor (#5)

* team limit

* websync limit

* json editor

* text editor

* perf: search test

* change cq value type

* doc

* intro img

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/global/common/file/api.d.ts

@@ -12,4 +12,5 @@ export type UrlFetchParams = {
 export type UrlFetchResponse = {
   url: string;
   content: string;
+  selector?: string;
 }[];

packages/global/common/string/tiktoken/index.ts

@@ -35,7 +35,7 @@ export function countPromptTokens(
   const text = `${role}\n${prompt}`;
   try {
     const encodeText = enc.encode(text);
-    return encodeText.length + 3; // 补充 role 估算值
+    return encodeText.length + role.length; // 补充 role 估算值
   } catch (error) {
     return text.length;
   }

packages/global/common/system/types/index.d.ts

@@ -44,7 +44,8 @@ export type FastGPTFeConfigsType = {
     google?: string;
   };
   limit?: {
-    exportLimitMinutes?: number;
+    exportDatasetLimitMinutes?: number;
+    websiteSyncLimitMinuted?: number;
   };
   scripts?: { [key: string]: string }[];
   favicon?: string;

packages/global/core/dataset/constant.ts

@@ -73,6 +73,19 @@ export const DatasetCollectionTrainingTypeMap = {
   }
 };
 
+export enum DatasetCollectionSyncResultEnum {
+  sameRaw = 'sameRaw',
+  success = 'success'
+}
+export const DatasetCollectionSyncResultMap = {
+  [DatasetCollectionSyncResultEnum.sameRaw]: {
+    label: 'core.dataset.collection.sync.result.sameRaw'
+  },
+  [DatasetCollectionSyncResultEnum.success]: {
+    label: 'core.dataset.collection.sync.result.success'
+  }
+};
+
 /* ------------ data -------------- */
 export enum DatasetDataIndexTypeEnum {
   chunk = 'chunk',
@@ -150,16 +163,24 @@ export enum SearchScoreTypeEnum {
 }
 export const SearchScoreTypeMap = {
   [SearchScoreTypeEnum.embedding]: {
-    label: 'core.dataset.search.score.embedding'
+    label: 'core.dataset.search.score.embedding',
+    desc: 'core.dataset.search.score.embedding desc',
+    showScore: true
   },
   [SearchScoreTypeEnum.fullText]: {
-    label: 'core.dataset.search.score.fullText'
+    label: 'core.dataset.search.score.fullText',
+    desc: 'core.dataset.search.score.fullText desc',
+    showScore: false
   },
   [SearchScoreTypeEnum.reRank]: {
-    label: 'core.dataset.search.score.reRank'
+    label: 'core.dataset.search.score.reRank',
+    desc: 'core.dataset.search.score.reRank desc',
+    showScore: true
   },
   [SearchScoreTypeEnum.rrf]: {
-    label: 'core.dataset.search.score.rrf'
+    label: 'core.dataset.search.score.rrf',
+    desc: 'core.dataset.search.score.rrf desc',
+    showScore: false
   }
 };
 

packages/global/core/dataset/type.d.ts

@@ -49,7 +49,10 @@ export type DatasetCollectionSchemaType = {
   qaPrompt?: string;
   rawTextLength?: number;
   hashRawText?: string;
-  metadata?: Record<string, any>;
+  metadata?: {
+    webPageSelector?: string;
+    [key: string]: any;
+  };
 };
 
 export type DatasetDataIndexItemType = {

packages/global/core/module/node/constant.ts

@@ -7,7 +7,10 @@ export enum FlowNodeInputTypeEnum {
   slider = 'slider',
   target = 'target', // data input
   switch = 'switch',
+
+  // editor
   textarea = 'textarea',
+  JSONEditor = 'JSONEditor',
 
   addInputParam = 'addInputParam', // params input
 

packages/global/core/module/template/system/http.ts

@@ -55,7 +55,7 @@ export const HttpModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.httpHeader,
-      type: FlowNodeInputTypeEnum.textarea,
+      type: FlowNodeInputTypeEnum.JSONEditor,
       valueType: ModuleIOValueTypeEnum.string,
       label: 'core.module.input.label.Http Request Header',
       description: 'core.module.input.description.Http Request Header',

packages/global/support/user/team/type.d.ts

@@ -10,6 +10,10 @@ export type TeamSchema = {
   balance: number;
   maxSize: number;
   lastDatasetBillTime: Date;
+  limit: {
+    lastExportDatasetTime: Date;
+    lastWebsiteSyncTime: Date;
+  };
 };
 
 export type TeamMemberSchema = {

packages/global/support/user/type.d.ts

@@ -17,10 +17,6 @@ export type UserModelSchema = {
     key: string;
     baseUrl: string;
   };
-  limit: {
-    exportKbTime?: Date;
-    datasetMaxCount?: number;
-  };
 };
 
 export type UserType = {