From 565a966d1990b8750e40971dd7c3dcc2292822a6 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 28 Mar 2025 13:45:09 +0800 Subject: [PATCH] Python Sandbox (#4380) * Python3 Sandbox (#3944) * update python box (#4251) * update python box * Adjust the height of the NodeCode border. * update python sandbox and add test systemcall bash * update sandbox * add VERSION_RELEASE (#4376) * save empty docx * fix pythonbox log error * fix: js template --------- Co-authored-by: dogfar <37035781+dogfar@users.noreply.github.com> Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> Co-authored-by: gggaaallleee <1293587368@qq.com> --- .../template/system/sandbox/constants.ts | 24 +++- .../workflow/template/system/sandbox/index.ts | 2 + .../core/workflow/dispatch/code/run.ts | 17 ++- .../service/core/workflow/dispatch/utils.ts | 1 + .../service/worker/readFile/extension/docx.ts | 1 + .../common/Textarea/CodeEditor/Editor.tsx | 75 ++++++---- .../common/Textarea/CodeEditor/index.tsx | 24 +++- .../CodeEditor/usePythonCompletion.ts | 83 +++++++++++ packages/web/i18n/en/workflow.json | 2 + packages/web/i18n/zh-CN/workflow.json | 2 + packages/web/i18n/zh-Hant/workflow.json | 2 + plugins/webcrawler/deploy/docker-compose.yaml | 80 +++++++++++ .../webcrawler/deploy/searxng/limiter.toml | 6 + .../webcrawler/deploy/searxng/settings.yml | 122 ++++++++++++++++ projects/README.md | 14 +- .../Flow/nodes/NodeCode.tsx | 84 ++++++++--- projects/sandbox/Dockerfile | 23 +++- projects/sandbox/requirements.txt | 2 + projects/sandbox/src/sandbox/constants.ts | 130 ++++++++++++++++++ .../sandbox/src/sandbox/sandbox.controller.ts | 10 +- projects/sandbox/src/sandbox/utils.ts | 102 ++++++++++---- projects/sandbox/testSystemCall.sh | 41 ++++++ .../global/common/string/textSplitter.test.ts | 22 +++ 23 files changed, 777 insertions(+), 92 deletions(-) create mode 100644 packages/web/components/common/Textarea/CodeEditor/usePythonCompletion.ts create mode 100644 plugins/webcrawler/deploy/docker-compose.yaml create mode 100644 plugins/webcrawler/deploy/searxng/limiter.toml create mode 100644 plugins/webcrawler/deploy/searxng/settings.yml create mode 100644 projects/sandbox/requirements.txt create mode 100644 projects/sandbox/src/sandbox/constants.ts create mode 100644 projects/sandbox/testSystemCall.sh diff --git a/packages/global/core/workflow/template/system/sandbox/constants.ts b/packages/global/core/workflow/template/system/sandbox/constants.ts index 7fc1dda7a..c954bc380 100644 --- a/packages/global/core/workflow/template/system/sandbox/constants.ts +++ b/packages/global/core/workflow/template/system/sandbox/constants.ts @@ -1,7 +1,23 @@ export const JS_TEMPLATE = `function main({data1, data2}){ - return { - result: data1, - data2 - } + return { + result: data1, + data2 + } }`; + +export const PY_TEMPLATE = `def main(data1, data2): + return { + "result": data1, + "data2": data2 + } +`; + +export enum SandboxCodeTypeEnum { + js = 'js', + py = 'py' +} +export const SNADBOX_CODE_TEMPLATE = { + [SandboxCodeTypeEnum.js]: JS_TEMPLATE, + [SandboxCodeTypeEnum.py]: PY_TEMPLATE +}; diff --git a/packages/global/core/workflow/template/system/sandbox/index.ts b/packages/global/core/workflow/template/system/sandbox/index.ts index 5325c5e3b..f1ba750b2 100644 --- a/packages/global/core/workflow/template/system/sandbox/index.ts +++ b/packages/global/core/workflow/template/system/sandbox/index.ts @@ -68,12 +68,14 @@ export const CodeNode: FlowNodeTemplateType = { key: NodeInputKeyEnum.codeType, renderTypeList: [FlowNodeInputTypeEnum.hidden], label: '', + valueType: WorkflowIOValueTypeEnum.string, value: 'js' }, { key: NodeInputKeyEnum.code, renderTypeList: [FlowNodeInputTypeEnum.custom], label: '', + valueType: WorkflowIOValueTypeEnum.string, value: JS_TEMPLATE } ], diff --git a/packages/service/core/workflow/dispatch/code/run.ts b/packages/service/core/workflow/dispatch/code/run.ts index 154941060..7d3491762 100644 --- a/packages/service/core/workflow/dispatch/code/run.ts +++ b/packages/service/core/workflow/dispatch/code/run.ts @@ -4,9 +4,10 @@ import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/ty import axios from 'axios'; import { formatHttpError } from '../utils'; import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants'; +import { SandboxCodeTypeEnum } from '@fastgpt/global/core/workflow/template/system/sandbox/constants'; type RunCodeType = ModuleDispatchProps<{ - [NodeInputKeyEnum.codeType]: 'js'; + [NodeInputKeyEnum.codeType]: string; [NodeInputKeyEnum.code]: string; [NodeInputKeyEnum.addInputParam]: Record; }>; @@ -16,6 +17,14 @@ type RunCodeResponse = DispatchNodeResultType<{ [key: string]: any; }>; +function getURL(codeType: string): string { + if (codeType == SandboxCodeTypeEnum.py) { + return `${process.env.SANDBOX_URL}/sandbox/python`; + } else { + return `${process.env.SANDBOX_URL}/sandbox/js`; + } +} + export const dispatchRunCode = async (props: RunCodeType): Promise => { const { params: { codeType, code, [NodeInputKeyEnum.addInputParam]: customVariables } @@ -27,7 +36,7 @@ export const dispatchRunCode = async (props: RunCodeType): Promise { if (value === undefined) return; + if (!type) return value; if (type === 'string') { if (typeof value !== 'object') return String(value); diff --git a/packages/service/worker/readFile/extension/docx.ts b/packages/service/worker/readFile/extension/docx.ts index 43d6f66fe..66df4df03 100644 --- a/packages/service/worker/readFile/extension/docx.ts +++ b/packages/service/worker/readFile/extension/docx.ts @@ -13,6 +13,7 @@ export const readDocsFile = async ({ buffer }: ReadRawTextByBuffer): Promise { const imageBase64 = await image.readAsBase64String(); const uuid = crypto.randomUUID(); diff --git a/packages/web/components/common/Textarea/CodeEditor/Editor.tsx b/packages/web/components/common/Textarea/CodeEditor/Editor.tsx index bb96bdd3c..d6b7b3760 100644 --- a/packages/web/components/common/Textarea/CodeEditor/Editor.tsx +++ b/packages/web/components/common/Textarea/CodeEditor/Editor.tsx @@ -1,9 +1,9 @@ -import React, { useCallback, useRef, useState } from 'react'; +import React, { useCallback, useRef, useState, useEffect } from 'react'; import Editor, { Monaco, loader } from '@monaco-editor/react'; import { Box, BoxProps } from '@chakra-ui/react'; import MyIcon from '../../Icon'; import { getWebReqUrl } from '../../../../common/system/utils'; - +import usePythonCompletion from './usePythonCompletion'; loader.config({ paths: { vs: getWebReqUrl('/js/monaco-editor.0.45.0/vs') } }); @@ -21,6 +21,7 @@ export type Props = Omit & { onOpenModal?: () => void; variables?: EditorVariablePickerType[]; defaultHeight?: number; + language?: string; }; const options = { @@ -53,11 +54,14 @@ const MyEditor = ({ variables = [], defaultHeight = 200, onOpenModal, + language = 'typescript', ...props }: Props) => { const [height, setHeight] = useState(defaultHeight); const initialY = useRef(0); + const registerPythonCompletion = usePythonCompletion(); + const handleMouseDown = useCallback((e: React.MouseEvent) => { initialY.current = e.clientY; @@ -76,35 +80,47 @@ const MyEditor = ({ document.addEventListener('mouseup', handleMouseUp); }, []); - const beforeMount = useCallback((monaco: Monaco) => { - monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ - validate: false, - allowComments: false, - schemas: [ - { - uri: 'http://myserver/foo-schema.json', // 一个假设的 URI - fileMatch: ['*'], // 匹配所有文件 - schema: {} // 空的 Schema - } - ] - }); + const editorRef = useRef(null); + const monacoRef = useRef(null); - monaco.editor.defineTheme('JSONEditorTheme', { - base: 'vs', // 可以基于已有的主题进行定制 - inherit: true, // 继承基础主题的设置 - rules: [{ token: 'variable', foreground: '2B5FD9' }], - colors: { - 'editor.background': '#ffffff00', - 'editorLineNumber.foreground': '#aaa', - 'editorOverviewRuler.border': '#ffffff00', - 'editor.lineHighlightBackground': '#F7F8FA', - 'scrollbarSlider.background': '#E8EAEC', - 'editorIndentGuide.activeBackground': '#ddd', - 'editorIndentGuide.background': '#eee' - } - }); + const handleEditorDidMount = useCallback((editor: any, monaco: Monaco) => { + editorRef.current = editor; + monacoRef.current = monaco; }, []); + const beforeMount = useCallback( + (monaco: Monaco) => { + monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ + validate: false, + allowComments: false, + schemas: [ + { + uri: 'http://myserver/foo-schema.json', // 一个假设的 URI + fileMatch: ['*'], // 匹配所有文件 + schema: {} // 空的 Schema + } + ] + }); + + monaco.editor.defineTheme('JSONEditorTheme', { + base: 'vs', // 可以基于已有的主题进行定制 + inherit: true, // 继承基础主题的设置 + rules: [{ token: 'variable', foreground: '2B5FD9' }], + colors: { + 'editor.background': '#ffffff00', + 'editorLineNumber.foreground': '#aaa', + 'editorOverviewRuler.border': '#ffffff00', + 'editor.lineHighlightBackground': '#F7F8FA', + 'scrollbarSlider.background': '#E8EAEC', + 'editorIndentGuide.activeBackground': '#ddd', + 'editorIndentGuide.background': '#eee' + } + }); + registerPythonCompletion(monaco); + }, + [registerPythonCompletion] + ); + return ( { onChange?.(e || ''); }} + onMount={handleEditorDidMount} /> {resize && ( & {}; +type Props = Omit & { language?: string }; +function getLanguage(language: string | undefined): string { + let fullName: string; + switch (language) { + case 'py': + fullName = 'python'; + break; + case 'js': + fullName = 'typescript'; + break; + default: + fullName = `typescript`; + break; + } + return fullName; +} const CodeEditor = (props: Props) => { const { t } = useTranslation(); const { isOpen, onOpen, onClose } = useDisclosure(); - + const { language, ...otherProps } = props; + const fullName = getLanguage(language); return ( <> - + { isCentered > - +