ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-07-21 11:43:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: load plugin groups code;perf: system plugin schema;fix: special variables replace;perf: retry cron app job (#3347)

Browse Source 
* perf: load plugin groups code

* perf: system plugin schema

* feat: retry cron app job

* fix: special variables replace
This commit is contained in:
Archer
2024-12-09 17:18:07 +08:00
committed by GitHub
parent a4f6128a89
commit 1a294c1fd3
16 changed files with 106 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
docSite/content/zh-cn/docs/development/upgrading/4815.md
View File

@@ -45,4 +45,5 @@ weight: 809
13. 修复 - 对话页面切换自动执行应用时，会误触发非自动执行应用。
14. 修复 - 语言播放鉴权问题。
15. 修复 - 插件应用知识库引用上限始终为 3000
16. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。
16. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。
17. 修复 - 工作流特殊变量替换问题。（$开头的字符串无法替换）

12
packages/global/common/system/utils.ts
View File

@@ -4,3 +4,15 @@ export const delay = (ms: number) =>
resolve('');
}, ms);
});
export const retryFn = async <T>(fn: () => Promise<T>, retryTimes = 3): Promise<T> => {
try {
return fn();
} catch (error) {
if (retryTimes > 0) {
await delay(500);
return retryFn(fn, retryTimes - 1);
}
return Promise.reject(error);
}
};

3
packages/global/core/dataset/type.d.ts vendored
View File

@@ -34,6 +34,9 @@ export type DatasetSchemaType = {
inheritPermission: boolean;
apiServer?: APIFileServer;
syncSchedule?: { cronString: string; timezone: string };
syncNextTime?: Date;
// abandon
externalReadUrl?: string;
defaultPermission?: number;

2
packages/global/core/workflow/runtime/utils.ts
View File

@@ -321,7 +321,7 @@ export function replaceEditorVariable({
})();
const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g');
text = text.replace(regex, formatVal);
text = text.replace(regex, () => formatVal);
});
return text || '';

3
packages/service/core/app/plugin/systemPluginSchema.ts
View File

@@ -10,8 +10,7 @@ const SystemPluginSchema = new Schema({
required: true
},
isActive: {
type: Boolean,
required: true
type: Boolean
},
inputConfig: {
type: Array,

2
packages/service/core/app/plugin/type.d.ts vendored
View File

@@ -13,7 +13,7 @@ export type SystemPluginConfigSchemaType = {
hasTokenFee: boolean;
isActive: boolean;
pluginOrder: number;
inputConfig: SystemPluginTemplateItemType['inputConfig'];
inputConfig?: SystemPluginTemplateItemType['inputConfig'];
customConfig?: {
name: string;

2
packages/service/core/app/schema.ts
View File

@@ -121,6 +121,6 @@ const AppSchema = new Schema({
AppSchema.index({ teamId: 1, updateTime: -1 });
AppSchema.index({ teamId: 1, type: 1 });
AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
AppSchema.index({ scheduledTriggerConfig: 1, scheduledTriggerNextTime: -1 });
export const MongoApp = getMongoModel<AppType>(AppCollectionName, AppSchema);

13
packages/service/core/dataset/schema.ts
View File

@@ -91,6 +91,18 @@ const DatasetSchema = new Schema({
type: Object
},
syncSchedule: {
cronString: {
type: String
},
timezone: {
type: String
}
},
syncNextTime: {
type: Date
},
// abandoned
externalReadUrl: {
type: String
@@ -100,6 +112,7 @@ const DatasetSchema = new Schema({
try {
DatasetSchema.index({ teamId: 1 });
DatasetSchema.index({ syncSchedule: 1, syncNextTime: -1 });
} catch (error) {
console.log(error);
}

8
packages/web/core/workflow/constants.ts
View File

@@ -1,5 +1,6 @@
import { FlowNodeTemplateTypeEnum } from '@fastgpt/global/core/workflow/constants';
import { i18nT } from '../../i18n/utils';
import type { PluginGroupSchemaType, TGroupType } from '../../../service/core/app/plugin/type';
export const workflowNodeTemplateList = [
{
@@ -49,10 +50,7 @@ export const workflowNodeTemplateList = [
}
];
export const systemPluginTemplateList: {
typeId: string;
typeName: string;
}[] = [
export const systemPluginTemplateList: TGroupType[] = [
{
typeId: FlowNodeTemplateTypeEnum.tools,
typeName: i18nT('common:navbar.Tools')
@@ -74,7 +72,7 @@ export const systemPluginTemplateList: {
typeName: i18nT('common:common.Other')
}
];
export const defaultGroup = {
export const defaultGroup: PluginGroupSchemaType = {
groupId: 'systemPlugin',
groupAvatar: 'common/navbar/pluginLight',
groupName: i18nT('common:core.module.template.System Plugin'),

2
projects/app/src/components/Layout/navbar.tsx
View File

@@ -115,7 +115,7 @@ const Navbar = ({ unread }: { unread: number }) => {
borderRadius={'50%'}
overflow={'hidden'}
cursor={'pointer'}
onClick={() => router.push('/account')}
onClick={() => router.push('/account/info')}
>
<Avatar w={'2rem'} h={'2rem'} src={userInfo?.avatar} borderRadius={'50%'} />
</Box>

1
projects/app/src/pages/account/components/AccountContainer.tsx
View File

@@ -161,6 +161,7 @@ const AccountContainer = ({
<Box mb={3}>
<LightRowTabs<TabEnum>
m={'auto'}
w={'100%'}
size={isPc ? 'md' : 'sm'}
list={tabList.map((item) => ({
value: item.value,

1
projects/app/src/pages/account/info/index.tsx
View File

@@ -118,7 +118,6 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => {
const standardPlan = teamPlanStatus?.standardConstants;
const { isPc } = useSystem();
const { toast } = useToast();
const router = useRouter();
const {
isOpen: isOpenConversionModal,

29
projects/app/src/pages/api/core/app/plugin/getPluginGroups.ts
View File

@@ -1,29 +0,0 @@
import { NextAPI } from '@/service/middleware/entry';
import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type';
import { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
export type getPluginGroupsQuery = {};
export type getPluginGroupsBody = {};
export type getPluginGroupsResponse = PluginGroupSchemaType[];
async function handler(
req: ApiRequestProps<getPluginGroupsBody, getPluginGroupsQuery>,
res: ApiResponseType<any>
): Promise<getPluginGroupsResponse> {
const pluginGroups = await MongoPluginGroups.find().sort({ groupOrder: 1 });
const result = pluginGroups.map((item) => ({
groupId: item.groupId,
groupName: item.groupName,
groupAvatar: item.groupAvatar,
groupTypes: item.groupTypes,
groupOrder: item.groupOrder
}));
return result;
}
export default NextAPI(handler);

3
projects/app/src/service/common/system/cron.ts
View File

@@ -12,12 +12,14 @@ import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants'
import { addHours } from 'date-fns';
import { getScheduleTriggerApp } from '@/service/core/app/utils';
// Try to run train every minute
const setTrainingQueueCron = () => {
setCron('*/1 * * * *', () => {
startTrainingQueue();
});
};
// Clear tmp upload files every ten minutes
const setClearTmpUploadFilesCron = () => {
// Clear tmp upload files every ten minutes
setCron('*/10 * * * *', () => {
@@ -61,6 +63,7 @@ const clearInvalidDataCron = () => {
});
};
// Run app timer trigger every hour
const scheduleTriggerAppCron = () => {
setCron('0 */1 * * *', async () => {
if (

105
projects/app/src/service/core/app/utils.ts
View File

@@ -3,7 +3,7 @@ import { pushChatUsage } from '@/service/support/wallet/usage/push';
import { defaultApp } from '@/web/core/app/constants';
import { getNextTimeByCronStringAndTimezone } from '@fastgpt/global/common/string/time';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { delay } from '@fastgpt/global/common/system/utils';
import { delay, retryFn } from '@fastgpt/global/common/system/utils';
import { ChatItemValueTypeEnum } from '@fastgpt/global/core/chat/constants';
import {
getWorkflowEntryNodeIds,
@@ -18,63 +18,72 @@ import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
export const getScheduleTriggerApp = async () => {
// 1. Find all the app
const apps = await MongoApp.find({
scheduledTriggerConfig: { $ne: null },
scheduledTriggerNextTime: { $lte: new Date() }
const apps = await retryFn(() => {
return MongoApp.find({
scheduledTriggerConfig: { $ne: null },
scheduledTriggerNextTime: { $lte: new Date() }
});
});
// 2. Run apps
await Promise.allSettled(
apps.map(async (app) => {
if (!app.scheduledTriggerConfig) return;
// random delay 0 ~ 60s
await delay(Math.floor(Math.random() * 60 * 1000));
const { user } = await getUserChatInfoAndAuthTeamPoints(app.tmbId);
try {
const { flowUsages } = await dispatchWorkFlow({
chatId: getNanoid(),
user,
mode: 'chat',
runningAppInfo: {
id: String(app._id),
teamId: String(app.teamId),
tmbId: String(app.tmbId)
},
uid: String(app.tmbId),
runtimeNodes: storeNodes2RuntimeNodes(app.modules, getWorkflowEntryNodeIds(app.modules)),
runtimeEdges: initWorkflowEdgeStatus(app.edges),
variables: {},
query: [
{
type: ChatItemValueTypeEnum.text,
text: {
content: app.scheduledTriggerConfig?.defaultPrompt
if (!app.scheduledTriggerConfig) return;
// random delay 0 ~ 60s
await delay(Math.floor(Math.random() * 60 * 1000));
const { user } = await getUserChatInfoAndAuthTeamPoints(app.tmbId);
await retryFn(async () => {
if (!app.scheduledTriggerConfig) return;
const { flowUsages } = await dispatchWorkFlow({
chatId: getNanoid(),
user,
mode: 'chat',
runningAppInfo: {
id: String(app._id),
teamId: String(app.teamId),
tmbId: String(app.tmbId)
},
uid: String(app.tmbId),
runtimeNodes: storeNodes2RuntimeNodes(
app.modules,
getWorkflowEntryNodeIds(app.modules)
),
runtimeEdges: initWorkflowEdgeStatus(app.edges),
variables: {},
query: [
{
type: ChatItemValueTypeEnum.text,
text: {
content: app.scheduledTriggerConfig?.defaultPrompt
}
}
}
],
chatConfig: defaultApp.chatConfig,
histories: [],
stream: false,
maxRunTimes: WORKFLOW_MAX_RUN_TIMES
});
pushChatUsage({
appName: app.name,
appId: app._id,
teamId: String(app.teamId),
tmbId: String(app.tmbId),
source: UsageSourceEnum.cronJob,
flowUsages
],
chatConfig: defaultApp.chatConfig,
histories: [],
stream: false,
maxRunTimes: WORKFLOW_MAX_RUN_TIMES
});
pushChatUsage({
appName: app.name,
appId: app._id,
teamId: String(app.teamId),
tmbId: String(app.tmbId),
source: UsageSourceEnum.cronJob,
flowUsages
});
});
// update next time
app.scheduledTriggerNextTime = getNextTimeByCronStringAndTimezone(
app.scheduledTriggerConfig
);
await app.save();
} catch (error) {
addLog.error('Schedule trigger error', error);
addLog.warn('Schedule trigger error', { error });
}
// update next time
app.scheduledTriggerNextTime = getNextTimeByCronStringAndTimezone(app.scheduledTriggerConfig);
await app.save();
return;
})
);
};

9
projects/app/src/web/core/app/api/plugin.ts
View File

@@ -14,6 +14,8 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import { GetSystemPluginTemplatesBody } from '@/pages/api/core/app/plugin/getSystemPluginTemplates';
import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { defaultGroup } from '@fastgpt/web/core/workflow/constants';
/* ============ team plugin ============== */
export const getTeamPlugTemplates = (data?: ListAppBody) =>
@@ -41,8 +43,11 @@ export const getTeamPlugTemplates = (data?: ListAppBody) =>
export const getSystemPlugTemplates = (data: GetSystemPluginTemplatesBody) =>
POST<NodeTemplateListItemType[]>('/core/app/plugin/getSystemPluginTemplates', data);
export const getPluginGroups = () =>
GET<PluginGroupSchemaType[]>('/core/app/plugin/getPluginGroups');
export const getPluginGroups = () => {
return useSystemStore.getState()?.feConfigs?.isPlus
? GET<PluginGroupSchemaType[]>('/proApi/core/app/plugin/getPluginGroups')
: Promise.resolve([defaultGroup]);
};
export const getSystemPluginPaths = (parentId: ParentIdType) => {
if (!parentId) return Promise.resolve<ParentTreePathItemType[]>([]);