V4.8.15 feature (#3331)

* feat: add customize toolkit (#3205)

* chaoyang

* fix-auth

* add toolkit

* add order

* plugin usage

* fix

* delete console:

* Fix: Fix fullscreen preview top positioning and improve Markdown rendering logic (#3247)

* 完成任务：修复全屏预览顶部固定问题，优化 Markdown 渲染逻辑

* 有问题修改

* 问题再修改

* 修正问题

* fix: plugin standalone display issue (#3254)

* 4.8.15 test (#3246)

* o1 config

* perf: system plugin code

* 调整系统插件代码。增加html 渲染安全配置。 (#3258)

* perf: base64 picker

* perf: list app or dataset

* perf: plugin config code

* 小窗适配等问题 (#3257)

* 小窗适配等问题

* git问题

* 小窗剩余问题

* feat: system plugin auth and lock version (#3265)

* feat: system plugin auth and lock version

* update comment

* 4.8.15 test (#3267)

* tmp log

* perf: login direct

* perf: iframe html code

* remove log

* fix: plugin standalone display (#3277)

* refactor: 页面拆分&i18n拆分 (#3281)

* refactor: account组件拆成独立页面

* script: 新增i18n json文件创建脚本

* refactor: 页面i18n拆分

* i18n: add en&hant

* 4.8.15 test (#3285)

* tmp log

* remove log

* fix: watch avatar refresh

* perf: i18n code

* fix(plugin): use intro instead of userguide (#3290)

* Universal SSO (#3292)

* tmp log

* remove log

* feat: common oauth

* readme

* perf: sso provider

* remove sso code

* perf: refresh plugins

* feat: add api dataset (#3272)

* add api-dataset

* fix api-dataset

* fix api dataset

* fix ts

* perf: create collection code (#3301)

* tmp log

* remove log

* perf: i18n change

* update version doc

* feat: question guide from chatId

* perf: create collection code

* fix: request api

* fix: request api

* fix: tts auth and response type (#3303)

* perf: md splitter

* fix: tts auth and response type

* fix: api file dataset (#3307)

* perf: api dataset init (#3310)

* perf: collection schema

* perf: api dataset init

* refactor: 团队管理独立页面 (#3302)

* ui: 团队管理独立页面

* 代码优化

* fix

* perf: sync collection and ui check (#3314)

* perf: sync collection

* remove script

* perf: update api server

* perf: api dataset parent

* perf: team ui

* perf: team 18n

* update team ui

* perf: ui check

* perf: i18n

* fix: debug variables & cronjob & system plugin callback load (#3315)

* fix: debug variables & cronjob & system plugin callback load

* fix type

* fix

* fix

* fix: plugin dataset quote;perf: system variables init (#3316)

* fix: plugin dataset quote

* perf: system variables init

* perf: node templates ui;fix: dataset import ui (#3318)

* fix: dataset import ui

* perf: node templates ui

* perf: ui refresh

* feat:套餐改名和套餐跳转配置 (#3309)

* fixing:except Sidebar

* 去除了多余的代码

* 修正了套餐说明的代码

* 修正了误删除的show_git代码

* 修正了名字部分等代码

* 修正了问题,遗留了其他和ui讨论不一致的部分

* 4.8.15 test (#3319)

* remove log

* pref: bill ui

* pref: bill ui

* perf: log

* html渲染文档 (#3270)

* html渲染文档

* 文档有点小问题

* feat: doc (#3322)

* 集合重训练 (#3282)

* rebaser

* 一点补充

* 小问题

* 其他问题修正，删除集合保留文件的参数还没找到...

* reTraining

* delete uesless

* 删除了一行错误代码

* 集合重训练部分

* fixing

* 删除console代码

* feat: navbar item config (#3326)

* perf: custom navbar code;perf: retraining code;feat: api dataset and dataset api doc (#3329)

* feat: api dataset and dataset api doc

* perf: retraining code

* perf: custom navbar code

* fix: ts (#3330)

* fix: ts

* fix: ts

* retraining ui

* perf: api collection filter

* perf: retrining button

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>

packages/service/core/workflow/dispatch/index.ts

@@ -72,7 +72,6 @@ import { dispatchLoopEnd } from './loop/runLoopEnd';
 import { dispatchLoopStart } from './loop/runLoopStart';
 import { dispatchFormInput } from './interactive/formInput';
 import { dispatchToolParams } from './agent/runTool/toolParams';
-import { responseWrite } from '../../../common/response';
 
 const callbackMap: Record<FlowNodeTypeEnum, Function> = {
   [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart,
@@ -500,8 +499,7 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
       value = replaceEditorVariable({
         text: value,
         nodes: runtimeNodes,
-        variables,
-        runningNode: node
+        variables
       });
 
       // replace reference variables
@@ -693,9 +691,17 @@ export function getSystemVariable({
   chatId,
   responseChatItemId,
   histories = [],
-  uid
+  uid,
+  chatConfig
 }: Props): SystemVariablesType {
+  const variables = chatConfig?.variables || [];
+  const variablesMap = variables.reduce<Record<string, any>>((acc, item) => {
+    acc[item.key] = valueTypeFormat(item.defaultValue, item.valueType);
+    return acc;
+  }, {});
+
   return {
+    ...variablesMap,
     userId: uid,
     appId: String(runningAppInfo.id),
     chatId,

packages/service/core/workflow/dispatch/plugin/run.ts

@@ -23,7 +23,6 @@ type RunPluginProps = ModuleDispatchProps<{
   [key: string]: any;
 }>;
 type RunPluginResponse = DispatchNodeResultType<{}>;
-
 export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPluginResponse> => {
   const {
     node: { pluginId, version },
@@ -31,7 +30,6 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
     query,
     params: { system_forbid_stream = false, ...data } // Plugin input
   } = props;
-
   if (!pluginId) {
     return Promise.reject('pluginId can not find');
   }
@@ -54,7 +52,6 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
         acc[cur.key] = cur.isToolOutput === false ? false : true;
         return acc;
       }, {}) ?? {};
-
   const runtimeNodes = storeNodes2RuntimeNodes(
     plugin.nodes,
     getWorkflowEntryNodeIds(plugin.nodes)
@@ -79,7 +76,6 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
     ...filterSystemVariables(props.variables),
     appId: String(plugin.id)
   };
-
   const { flowResponses, flowUsages, assistantResponses, runTimes } = await dispatchWorkFlow({
     ...props,
     // Rewrite stream mode
@@ -105,9 +101,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
     runtimeNodes,
     runtimeEdges: initWorkflowEdgeStatus(plugin.edges)
   });
-
   const output = flowResponses.find((item) => item.moduleType === FlowNodeTypeEnum.pluginOutput);
-
   if (output) {
     output.moduleLogo = plugin.avatar;
   }
@@ -117,7 +111,6 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
     childrenUsage: flowUsages,
     error: !!output?.pluginOutput?.error
   });
-
   return {
     // 嵌套运行时，如果 childApp stream=false，实际上不会有任何内容输出给用户，所以不需要存储
     assistantResponses: system_forbid_stream ? [] : assistantResponses,

packages/service/core/workflow/dispatch/tools/answer.ts

@@ -20,7 +20,7 @@ export const dispatchAnswer = (props: Record<string, any>): AnswerResponse => {
   } = props as AnswerProps;
 
   const formatText = typeof text === 'string' ? text : JSON.stringify(text, null, 2);
-  const responseText = `\n${formatText}`.replaceAll('\\n', '\n');
+  const responseText = `\n${formatText}`;
 
   workflowStreamResponse?.({
     event: SseResponseEventEnum.fastAnswer,

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -110,8 +110,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
       replaceEditorVariable({
         text,
         nodes: runtimeNodes,
-        variables: allVariables,
-        runningNode: node
+        variables: allVariables
       }),
       allVariables
     );

packages/service/core/workflow/dispatch/tools/readFiles.ts

@@ -12,6 +12,7 @@ import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
+import { addLog } from '../../../../common/system/log';
 
 type Props = ModuleDispatchProps<{
   [NodeInputKeyEnum.fileUrlList]: string[];
@@ -138,7 +139,7 @@ export const getFileContentFromLinks = async ({
 
         return url;
       } catch (error) {
-        console.log(error);
+        addLog.warn(`Parse url error`, { error });
         return '';
       }
     })

packages/service/core/workflow/dispatch/tools/runUpdateVar.ts

@@ -47,8 +47,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
           ? replaceEditorVariable({
               text: formatValue,
               nodes: runtimeNodes,
-              variables,
-              runningNode: node
+              variables
             })
           : formatValue;
       } else {

packages/service/core/workflow/dispatch/utils.ts

@@ -134,7 +134,7 @@ export const checkQuoteQAValue = (quoteQA?: SearchDataResponseItemType[]) => {
   if (quoteQA.length === 0) {
     return [];
   }
-  if (quoteQA.some((item) => !item.q || !item.datasetId)) {
+  if (quoteQA.some((item) => !item.q)) {
     return undefined;
   }
   return quoteQA;