Add image index and pdf parse (#3956)

* feat: think tag parse

* feat: parse think tag test

* feat: pdf parse ux

* feat: doc2x parse

* perf: rewrite training mode setting

* feat: image parse queue

* perf: image index

* feat: image parse process

* feat: add init sh

* fix: ts

packages/global/common/system/types/index.d.ts

@@ -43,10 +43,14 @@ export type FastGPTConfigFileType = {
 export type FastGPTFeConfigsType = {
   show_workorder?: boolean;
   show_emptyChat?: boolean;
+  isPlus?: boolean;
   register_method?: ['email' | 'phone' | 'sync'];
   login_method?: ['email' | 'phone']; // Attention: login method is diffrent with oauth
   find_password_method?: ['email' | 'phone'];
   bind_notification_method?: ['email' | 'phone'];
+  googleClientVerKey?: string;
+
+  show_emptyChat?: boolean;
   show_appStore?: boolean;
   show_git?: boolean;
   show_pay?: boolean;
@@ -57,15 +61,19 @@ export type FastGPTFeConfigsType = {
   show_aiproxy?: boolean;
   concatMd?: string;
 
+  concatMd?: string;
   docUrl?: string;
   openAPIDocUrl?: string;
   systemPluginCourseUrl?: string;
   appTemplateCourse?: string;
+  customApiDomain?: string;
+  customSharePageDomain?: string;
 
   systemTitle?: string;
   systemDescription?: string;
-  googleClientVerKey?: string;
-  isPlus?: boolean;
+  scripts?: { [key: string]: string }[];
+  favicon?: string;
+
   sso?: {
     icon?: string;
     title?: string;
@@ -91,13 +99,14 @@ export type FastGPTFeConfigsType = {
     exportDatasetLimitMinutes?: number;
     websiteSyncLimitMinuted?: number;
   };
-  scripts?: { [key: string]: string }[];
-  favicon?: string;
-  customApiDomain?: string;
-  customSharePageDomain?: string;
 
   uploadFileMaxAmount?: number;
   uploadFileMaxSize?: number;
+
+  // Compute by systemEnv.customPdfParse
+  showCustomPdfParse?: boolean;
+  customPdfParsePrice?: number;
+
   lafEnv?: string;
   navbarItems?: NavbarItemType[];
   externalProviderWorkflowVariables?: ExternalProviderWorkflowVarType[];
@@ -107,9 +116,18 @@ export type SystemEnvType = {
   openapiPrefix?: string;
   vectorMaxProcess: number;
   qaMaxProcess: number;
+  vlmMaxProcess: number;
   pgHNSWEfSearch: number;
   tokenWorkers: number; // token count max worker
 
   oneapiUrl?: string;
   chatApiKey?: string;
+
+  customPdfParse?: {
+    url?: string;
+    key?: string;
+
+    doc2xKey?: string;
+    price?: number; // n points/1 page
+  };
 };

packages/global/common/system/utils.ts

@@ -16,3 +16,24 @@ export const retryFn = async <T>(fn: () => Promise<T>, retryTimes = 3): Promise<
     return Promise.reject(error);
   }
 };
+
+export const batchRun = async <T>(arr: T[], fn: (arr: T) => any, batchSize = 10) => {
+  const batchArr = new Array(batchSize).fill(null);
+  const result: any[] = [];
+
+  const batchFn = async () => {
+    const data = arr.shift();
+    if (data) {
+      result.push(await fn(data));
+      return batchFn();
+    }
+  };
+
+  await Promise.all(
+    batchArr.map(async () => {
+      await batchFn();
+    })
+  );
+
+  return result;
+};