perf: load plugin groups code;perf: system plugin schema;fix: special variables replace;perf: retry cron app job (#3347)

* perf: load plugin groups code

* perf: system plugin schema

* feat: retry cron app job

* fix: special variables replace

docSite/content/zh-cn/docs/development/upgrading/4815.md

@@ -45,4 +45,5 @@ weight: 809
 13. 修复 - 对话页面切换自动执行应用时，会误触发非自动执行应用。
 14. 修复 - 语言播放鉴权问题。
 15. 修复 - 插件应用知识库引用上限始终为 3000
-16. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。
+16. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。
+17. 修复 - 工作流特殊变量替换问题。（$开头的字符串无法替换）

packages/global/common/system/utils.ts

@@ -4,3 +4,15 @@ export const delay = (ms: number) =>
       resolve('');
     }, ms);
   });
+
+export const retryFn = async <T>(fn: () => Promise<T>, retryTimes = 3): Promise<T> => {
+  try {
+    return fn();
+  } catch (error) {
+    if (retryTimes > 0) {
+      await delay(500);
+      return retryFn(fn, retryTimes - 1);
+    }
+    return Promise.reject(error);
+  }
+};

packages/global/core/dataset/type.d.ts

@@ -34,6 +34,9 @@ export type DatasetSchemaType = {
   inheritPermission: boolean;
   apiServer?: APIFileServer;
 
+  syncSchedule?: { cronString: string; timezone: string };
+  syncNextTime?: Date;
+
   // abandon
   externalReadUrl?: string;
   defaultPermission?: number;

packages/global/core/workflow/runtime/utils.ts

@@ -321,7 +321,7 @@ export function replaceEditorVariable({
     })();
 
     const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g');
-    text = text.replace(regex, formatVal);
+    text = text.replace(regex, () => formatVal);
   });
 
   return text || '';

packages/service/core/app/plugin/systemPluginSchema.ts

@@ -10,8 +10,7 @@ const SystemPluginSchema = new Schema({
     required: true
   },
   isActive: {
-    type: Boolean,
+    type: Boolean
-    required: true
   },
   inputConfig: {
     type: Array,

packages/service/core/app/plugin/type.d.ts

@@ -13,7 +13,7 @@ export type SystemPluginConfigSchemaType = {
   hasTokenFee: boolean;
   isActive: boolean;
   pluginOrder: number;
-  inputConfig: SystemPluginTemplateItemType['inputConfig'];
+  inputConfig?: SystemPluginTemplateItemType['inputConfig'];
 
   customConfig?: {
     name: string;

packages/service/core/app/schema.ts

@@ -121,6 +121,6 @@ const AppSchema = new Schema({
 
 AppSchema.index({ teamId: 1, updateTime: -1 });
 AppSchema.index({ teamId: 1, type: 1 });
-AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
+AppSchema.index({ scheduledTriggerConfig: 1, scheduledTriggerNextTime: -1 });
 
 export const MongoApp = getMongoModel<AppType>(AppCollectionName, AppSchema);

packages/service/core/dataset/schema.ts

@@ -91,6 +91,18 @@ const DatasetSchema = new Schema({
     type: Object
   },
 
+  syncSchedule: {
+    cronString: {
+      type: String
+    },
+    timezone: {
+      type: String
+    }
+  },
+  syncNextTime: {
+    type: Date
+  },
+
   // abandoned
   externalReadUrl: {
     type: String
@@ -100,6 +112,7 @@ const DatasetSchema = new Schema({
 
 try {
   DatasetSchema.index({ teamId: 1 });
+  DatasetSchema.index({ syncSchedule: 1, syncNextTime: -1 });
 } catch (error) {
   console.log(error);
 }

packages/web/core/workflow/constants.ts

@@ -1,5 +1,6 @@
 import { FlowNodeTemplateTypeEnum } from '@fastgpt/global/core/workflow/constants';
 import { i18nT } from '../../i18n/utils';
+import type { PluginGroupSchemaType, TGroupType } from '../../../service/core/app/plugin/type';
 
 export const workflowNodeTemplateList = [
   {
@@ -49,10 +50,7 @@ export const workflowNodeTemplateList = [
   }
 ];
 
-export const systemPluginTemplateList: {
+export const systemPluginTemplateList: TGroupType[] = [
-  typeId: string;
-  typeName: string;
-}[] = [
   {
     typeId: FlowNodeTemplateTypeEnum.tools,
     typeName: i18nT('common:navbar.Tools')
@@ -74,7 +72,7 @@ export const systemPluginTemplateList: {
     typeName: i18nT('common:common.Other')
   }
 ];
-export const defaultGroup = {
+export const defaultGroup: PluginGroupSchemaType = {
   groupId: 'systemPlugin',
   groupAvatar: 'common/navbar/pluginLight',
   groupName: i18nT('common:core.module.template.System Plugin'),

projects/app/src/components/Layout/navbar.tsx

@@ -115,7 +115,7 @@ const Navbar = ({ unread }: { unread: number }) => {
         borderRadius={'50%'}
         overflow={'hidden'}
         cursor={'pointer'}
-        onClick={() => router.push('/account')}
+        onClick={() => router.push('/account/info')}
       >
         <Avatar w={'2rem'} h={'2rem'} src={userInfo?.avatar} borderRadius={'50%'} />
       </Box>

projects/app/src/pages/account/components/AccountContainer.tsx

@@ -161,6 +161,7 @@ const AccountContainer = ({
             <Box mb={3}>
               <LightRowTabs<TabEnum>
                 m={'auto'}
+                w={'100%'}
                 size={isPc ? 'md' : 'sm'}
                 list={tabList.map((item) => ({
                   value: item.value,

projects/app/src/pages/account/info/index.tsx

@@ -118,7 +118,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => {
   const standardPlan = teamPlanStatus?.standardConstants;
   const { isPc } = useSystem();
   const { toast } = useToast();
-  const router = useRouter();
 
   const {
     isOpen: isOpenConversionModal,

projects/app/src/pages/api/core/app/plugin/getPluginGroups.ts

@@ -1,29 +0,0 @@
-import { NextAPI } from '@/service/middleware/entry';
-import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
-import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type';
-import { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
-
-export type getPluginGroupsQuery = {};
-
-export type getPluginGroupsBody = {};
-
-export type getPluginGroupsResponse = PluginGroupSchemaType[];
-
-async function handler(
-  req: ApiRequestProps<getPluginGroupsBody, getPluginGroupsQuery>,
-  res: ApiResponseType<any>
-): Promise<getPluginGroupsResponse> {
-  const pluginGroups = await MongoPluginGroups.find().sort({ groupOrder: 1 });
-
-  const result = pluginGroups.map((item) => ({
-    groupId: item.groupId,
-    groupName: item.groupName,
-    groupAvatar: item.groupAvatar,
-    groupTypes: item.groupTypes,
-    groupOrder: item.groupOrder
-  }));
-
-  return result;
-}
-
-export default NextAPI(handler);

projects/app/src/service/common/system/cron.ts

@@ -12,12 +12,14 @@ import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants'
 import { addHours } from 'date-fns';
 import { getScheduleTriggerApp } from '@/service/core/app/utils';
 
+// Try to run train every minute
 const setTrainingQueueCron = () => {
   setCron('*/1 * * * *', () => {
     startTrainingQueue();
   });
 };
 
+// Clear tmp upload files every ten minutes
 const setClearTmpUploadFilesCron = () => {
   // Clear tmp upload files every ten minutes
   setCron('*/10 * * * *', () => {
@@ -61,6 +63,7 @@ const clearInvalidDataCron = () => {
   });
 };
 
+// Run app timer trigger every hour
 const scheduleTriggerAppCron = () => {
   setCron('0 */1 * * *', async () => {
     if (

projects/app/src/service/core/app/utils.ts

@@ -3,7 +3,7 @@ import { pushChatUsage } from '@/service/support/wallet/usage/push';
 import { defaultApp } from '@/web/core/app/constants';
 import { getNextTimeByCronStringAndTimezone } from '@fastgpt/global/common/string/time';
 import { getNanoid } from '@fastgpt/global/common/string/tools';
-import { delay } from '@fastgpt/global/common/system/utils';
+import { delay, retryFn } from '@fastgpt/global/common/system/utils';
 import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
 import {
   getWorkflowEntryNodeIds,
@@ -18,63 +18,72 @@ import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
 
 export const getScheduleTriggerApp = async () => {
   // 1. Find all the app
-  const apps = await MongoApp.find({
+  const apps = await retryFn(() => {
-    scheduledTriggerConfig: { $ne: null },
+    return MongoApp.find({
-    scheduledTriggerNextTime: { $lte: new Date() }
+      scheduledTriggerConfig: { $ne: null },
+      scheduledTriggerNextTime: { $lte: new Date() }
+    });
   });
 
   // 2. Run apps
   await Promise.allSettled(
     apps.map(async (app) => {
-      if (!app.scheduledTriggerConfig) return;
-      // random delay 0 ~ 60s
-      await delay(Math.floor(Math.random() * 60 * 1000));
-      const { user } = await getUserChatInfoAndAuthTeamPoints(app.tmbId);
-
       try {
-        const { flowUsages } = await dispatchWorkFlow({
+        if (!app.scheduledTriggerConfig) return;
-          chatId: getNanoid(),
+        // random delay 0 ~ 60s
-          user,
+        await delay(Math.floor(Math.random() * 60 * 1000));
-          mode: 'chat',
+        const { user } = await getUserChatInfoAndAuthTeamPoints(app.tmbId);
-          runningAppInfo: {
+
-            id: String(app._id),
+        await retryFn(async () => {
-            teamId: String(app.teamId),
+          if (!app.scheduledTriggerConfig) return;
-            tmbId: String(app.tmbId)
+
-          },
+          const { flowUsages } = await dispatchWorkFlow({
-          uid: String(app.tmbId),
+            chatId: getNanoid(),
-          runtimeNodes: storeNodes2RuntimeNodes(app.modules, getWorkflowEntryNodeIds(app.modules)),
+            user,
-          runtimeEdges: initWorkflowEdgeStatus(app.edges),
+            mode: 'chat',
-          variables: {},
+            runningAppInfo: {
-          query: [
+              id: String(app._id),
-            {
+              teamId: String(app.teamId),
-              type: ChatItemValueTypeEnum.text,
+              tmbId: String(app.tmbId)
-              text: {
+            },
-                content: app.scheduledTriggerConfig?.defaultPrompt
+            uid: String(app.tmbId),
+            runtimeNodes: storeNodes2RuntimeNodes(
+              app.modules,
+              getWorkflowEntryNodeIds(app.modules)
+            ),
+            runtimeEdges: initWorkflowEdgeStatus(app.edges),
+            variables: {},
+            query: [
+              {
+                type: ChatItemValueTypeEnum.text,
+                text: {
+                  content: app.scheduledTriggerConfig?.defaultPrompt
+                }
               }
-            }
+            ],
-          ],
+            chatConfig: defaultApp.chatConfig,
-          chatConfig: defaultApp.chatConfig,
+            histories: [],
-          histories: [],
+            stream: false,
-          stream: false,
+            maxRunTimes: WORKFLOW_MAX_RUN_TIMES
-          maxRunTimes: WORKFLOW_MAX_RUN_TIMES
+          });
-        });
+          pushChatUsage({
-        pushChatUsage({
+            appName: app.name,
-          appName: app.name,
+            appId: app._id,
-          appId: app._id,
+            teamId: String(app.teamId),
-          teamId: String(app.teamId),
+            tmbId: String(app.tmbId),
-          tmbId: String(app.tmbId),
+            source: UsageSourceEnum.cronJob,
-          source: UsageSourceEnum.cronJob,
+            flowUsages
-          flowUsages
+          });
         });
+
+        // update next time
+        app.scheduledTriggerNextTime = getNextTimeByCronStringAndTimezone(
+          app.scheduledTriggerConfig
+        );
+        await app.save();
       } catch (error) {
-        addLog.error('Schedule trigger error', error);
+        addLog.warn('Schedule trigger error', { error });
       }
-
-      // update next time
-      app.scheduledTriggerNextTime = getNextTimeByCronStringAndTimezone(app.scheduledTriggerConfig);
-      await app.save();
-
-      return;
     })
   );
 };

projects/app/src/web/core/app/api/plugin.ts

@@ -14,6 +14,8 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 import { ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
 import { GetSystemPluginTemplatesBody } from '@/pages/api/core/app/plugin/getSystemPluginTemplates';
 import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import { defaultGroup } from '@fastgpt/web/core/workflow/constants';
 
 /* ============ team plugin ============== */
 export const getTeamPlugTemplates = (data?: ListAppBody) =>
@@ -41,8 +43,11 @@ export const getTeamPlugTemplates = (data?: ListAppBody) =>
 export const getSystemPlugTemplates = (data: GetSystemPluginTemplatesBody) =>
   POST<NodeTemplateListItemType[]>('/core/app/plugin/getSystemPluginTemplates', data);
 
-export const getPluginGroups = () =>
+export const getPluginGroups = () => {
-  GET<PluginGroupSchemaType[]>('/core/app/plugin/getPluginGroups');
+  return useSystemStore.getState()?.feConfigs?.isPlus
+    ? GET<PluginGroupSchemaType[]>('/proApi/core/app/plugin/getPluginGroups')
+    : Promise.resolve([defaultGroup]);
+};
 
 export const getSystemPluginPaths = (parentId: ParentIdType) => {
   if (!parentId) return Promise.resolve<ParentTreePathItemType[]>([]);