V4.8.15 feature (#3331)

* feat: add customize toolkit (#3205)

* chaoyang

* fix-auth

* add toolkit

* add order

* plugin usage

* fix

* delete console:

* Fix: Fix fullscreen preview top positioning and improve Markdown rendering logic (#3247)

* 完成任务：修复全屏预览顶部固定问题，优化 Markdown 渲染逻辑

* 有问题修改

* 问题再修改

* 修正问题

* fix: plugin standalone display issue (#3254)

* 4.8.15 test (#3246)

* o1 config

* perf: system plugin code

* 调整系统插件代码。增加html 渲染安全配置。 (#3258)

* perf: base64 picker

* perf: list app or dataset

* perf: plugin config code

* 小窗适配等问题 (#3257)

* 小窗适配等问题

* git问题

* 小窗剩余问题

* feat: system plugin auth and lock version (#3265)

* feat: system plugin auth and lock version

* update comment

* 4.8.15 test (#3267)

* tmp log

* perf: login direct

* perf: iframe html code

* remove log

* fix: plugin standalone display (#3277)

* refactor: 页面拆分&i18n拆分 (#3281)

* refactor: account组件拆成独立页面

* script: 新增i18n json文件创建脚本

* refactor: 页面i18n拆分

* i18n: add en&hant

* 4.8.15 test (#3285)

* tmp log

* remove log

* fix: watch avatar refresh

* perf: i18n code

* fix(plugin): use intro instead of userguide (#3290)

* Universal SSO (#3292)

* tmp log

* remove log

* feat: common oauth

* readme

* perf: sso provider

* remove sso code

* perf: refresh plugins

* feat: add api dataset (#3272)

* add api-dataset

* fix api-dataset

* fix api dataset

* fix ts

* perf: create collection code (#3301)

* tmp log

* remove log

* perf: i18n change

* update version doc

* feat: question guide from chatId

* perf: create collection code

* fix: request api

* fix: request api

* fix: tts auth and response type (#3303)

* perf: md splitter

* fix: tts auth and response type

* fix: api file dataset (#3307)

* perf: api dataset init (#3310)

* perf: collection schema

* perf: api dataset init

* refactor: 团队管理独立页面 (#3302)

* ui: 团队管理独立页面

* 代码优化

* fix

* perf: sync collection and ui check (#3314)

* perf: sync collection

* remove script

* perf: update api server

* perf: api dataset parent

* perf: team ui

* perf: team 18n

* update team ui

* perf: ui check

* perf: i18n

* fix: debug variables & cronjob & system plugin callback load (#3315)

* fix: debug variables & cronjob & system plugin callback load

* fix type

* fix

* fix

* fix: plugin dataset quote;perf: system variables init (#3316)

* fix: plugin dataset quote

* perf: system variables init

* perf: node templates ui;fix: dataset import ui (#3318)

* fix: dataset import ui

* perf: node templates ui

* perf: ui refresh

* feat:套餐改名和套餐跳转配置 (#3309)

* fixing:except Sidebar

* 去除了多余的代码

* 修正了套餐说明的代码

* 修正了误删除的show_git代码

* 修正了名字部分等代码

* 修正了问题,遗留了其他和ui讨论不一致的部分

* 4.8.15 test (#3319)

* remove log

* pref: bill ui

* pref: bill ui

* perf: log

* html渲染文档 (#3270)

* html渲染文档

* 文档有点小问题

* feat: doc (#3322)

* 集合重训练 (#3282)

* rebaser

* 一点补充

* 小问题

* 其他问题修正，删除集合保留文件的参数还没找到...

* reTraining

* delete uesless

* 删除了一行错误代码

* 集合重训练部分

* fixing

* 删除console代码

* feat: navbar item config (#3326)

* perf: custom navbar code;perf: retraining code;feat: api dataset and dataset api doc (#3329)

* feat: api dataset and dataset api doc

* perf: retraining code

* perf: custom navbar code

* fix: ts (#3330)

* fix: ts

* fix: ts

* retraining ui

* perf: api collection filter

* perf: retrining button

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>

projects/app/src/service/common/system/index.ts

@@ -9,6 +9,8 @@ import { FastGPTProUrl, isProduction } from '@fastgpt/service/common/system/cons
 import { initFastGPTConfig } from '@fastgpt/service/common/system/tools';
 import json5 from 'json5';
 import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type';
+import { defaultGroup } from '@fastgpt/web/core/workflow/constants';
+import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
 
 export const readConfigData = (name: string) => {
   const splitName = name.split('.');
@@ -185,3 +187,22 @@ function getSystemPluginV1() {
 
   global.communityPluginsV1 = fileTemplates;
 }
+
+export async function initSystemPlugins() {
+  try {
+    const { groupOrder, ...restDefaultGroup } = defaultGroup;
+    await MongoPluginGroups.updateOne(
+      {
+        groupId: defaultGroup.groupId
+      },
+      {
+        $set: restDefaultGroup
+      },
+      {
+        upsert: true
+      }
+    );
+  } catch (error) {
+    console.error('Error initializing system plugins:', error);
+  }
+}

projects/app/src/service/common/system/volumnMongoWatch.ts

@@ -2,7 +2,8 @@ import { getSystemPluginCb } from '@/service/core/app/plugin';
 import { initSystemConfig } from '.';
 import { createDatasetTrainingMongoWatch } from '@/service/core/dataset/training/utils';
 import { MongoSystemConfigs } from '@fastgpt/service/common/system/config/schema';
-import { MongoSystemPluginSchema } from '@fastgpt/service/core/app/plugin/systemPluginSchema';
+import { MongoSystemPlugin } from '@fastgpt/service/core/app/plugin/systemPluginSchema';
+import { debounce } from 'lodash';
 
 export const startMongoWatch = async () => {
   reloadConfigWatch();
@@ -24,13 +25,16 @@ const reloadConfigWatch = () => {
 };
 
 const refetchSystemPlugins = () => {
-  const changeStream = MongoSystemPluginSchema.watch();
+  const changeStream = MongoSystemPlugin.watch();
 
-  changeStream.on('change', async (change) => {
-    setTimeout(() => {
-      try {
-        getSystemPluginCb(true);
-      } catch (error) {}
-    }, 5000);
-  });
+  changeStream.on(
+    'change',
+    debounce(async (change) => {
+      setTimeout(() => {
+        try {
+          getSystemPluginCb(true);
+        } catch (error) {}
+      }, 5000);
+    }, 500)
+  );
 };

projects/app/src/service/core/app/plugin.ts

@@ -10,6 +10,7 @@ import { SystemPluginResponseType } from '@fastgpt/plugins/type';
 const getCommercialPlugins = () => {
   return GET<SystemPluginTemplateItemType[]>('/core/app/plugin/getSystemPlugins');
 };
+
 export const getSystemPlugins = async (refresh = false) => {
   if (isProduction && global.systemPlugins && global.systemPlugins.length > 0 && !refresh)
     return cloneDeep(global.systemPlugins);

projects/app/src/service/events/utils.ts

@@ -19,7 +19,7 @@ export const checkTeamAiPointsAndLock = async (teamId: string) => {
           teamId
         });
         console.log('余额不足，暂停【向量】生成任务');
-        lockTrainingDataByTeamId(teamId);
+        await lockTrainingDataByTeamId(teamId);
       } catch (error) {}
     }
     return false;

projects/app/src/service/support/permission/auth/chat.ts

@@ -1,15 +1,10 @@
 import { ChatSchema } from '@fastgpt/global/core/chat/type';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import { AuthModeType } from '@fastgpt/service/support/permission/type';
-import { authOutLink, authOutLinkInit } from './outLink';
+import { authOutLink } from './outLink';
 import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
-import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
 import { authTeamSpaceToken } from './team';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { authOutLinkValid } from '@fastgpt/service/support/permission/publish/authLink';
 import { AuthUserTypeEnum, ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
-import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
-import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
 import { authApp } from '@fastgpt/service/support/permission/app/auth';
 
 /* 
@@ -184,62 +179,3 @@ export async function authChatCrud({
 
   return Promise.reject(ChatErrEnum.unAuthChat);
 }
-
-/* 
-  Abandoned
-  Different chat source
-  1. token (header)
-  2. apikey (header)
-  3. share page (body: shareId outLinkUid)
-  4. team chat page (body: teamId teamToken)
-*/
-export async function authChatCert(props: AuthModeType): Promise<{
-  teamId: string;
-  tmbId: string;
-  authType: AuthUserTypeEnum;
-  apikey: string;
-  isOwner: boolean;
-  canWrite: boolean;
-  outLinkUid?: string;
-}> {
-  const { teamId, teamToken, shareId, outLinkUid } = props.req.body as OutLinkChatAuthProps;
-
-  if (shareId && outLinkUid) {
-    const { outLinkConfig } = await authOutLinkValid({ shareId });
-    const { uid } = await authOutLinkInit({
-      outLinkUid,
-      tokenUrl: outLinkConfig.limit?.hookUrl
-    });
-
-    return {
-      teamId: String(outLinkConfig.teamId),
-      tmbId: String(outLinkConfig.tmbId),
-      authType: AuthUserTypeEnum.outLink,
-      apikey: '',
-      isOwner: false,
-      canWrite: false,
-      outLinkUid: uid
-    };
-  }
-  if (teamId && teamToken) {
-    const { uid } = await authTeamSpaceToken({ teamId, teamToken });
-    const tmb = await MongoTeamMember.findOne(
-      { teamId, role: TeamMemberRoleEnum.owner },
-      'tmbId'
-    ).lean();
-
-    if (!tmb) return Promise.reject(ChatErrEnum.unAuthChat);
-
-    return {
-      teamId,
-      tmbId: String(tmb._id),
-      authType: AuthUserTypeEnum.teamDomain,
-      apikey: '',
-      isOwner: false,
-      canWrite: false,
-      outLinkUid: uid
-    };
-  }
-
-  return authCert(props);
-}

projects/app/src/service/support/wallet/usage/push.ts

@@ -4,6 +4,7 @@ import { addLog } from '@fastgpt/service/common/system/log';
 import { createUsage, concatUsage } from './controller';
 import { formatModelChars2Points } from '@fastgpt/service/support/wallet/usage/utils';
 import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
+import { i18nT } from '@fastgpt/web/i18n/utils';
 
 export const pushChatUsage = ({
   appName,
@@ -198,7 +199,7 @@ export const pushQuestionGuideUsage = ({
 };
 
 export function pushAudioSpeechUsage({
-  appName = 'support.wallet.usage.Audio Speech',
+  appName = i18nT('common:support.wallet.usage.Audio Speech'),
   model,
   charsLength,
   teamId,