Python Sandbox (#4380)

* Python3 Sandbox (#3944)

* update python box (#4251)

* update python box

* Adjust the height of the NodeCode border.

* update python sandbox and add test systemcall bash

* update sandbox

* add VERSION_RELEASE (#4376)

* save empty docx

* fix pythonbox log error

* fix: js template

---------

Co-authored-by: dogfar <37035781+dogfar@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
Co-authored-by: gggaaallleee <1293587368@qq.com>

packages/global/core/workflow/template/system/sandbox/constants.ts

@@ -1,7 +1,23 @@
 export const JS_TEMPLATE = `function main({data1, data2}){
     
-    return {
+  return {
-        result: data1,
+      result: data1,
-        data2
+      data2
-    }
+  }
 }`;
+
+export const PY_TEMPLATE = `def main(data1, data2):
+    return {
+        "result": data1,
+        "data2": data2
+    }
+`;
+
+export enum SandboxCodeTypeEnum {
+  js = 'js',
+  py = 'py'
+}
+export const SNADBOX_CODE_TEMPLATE = {
+  [SandboxCodeTypeEnum.js]: JS_TEMPLATE,
+  [SandboxCodeTypeEnum.py]: PY_TEMPLATE
+};

packages/global/core/workflow/template/system/sandbox/index.ts

@@ -68,12 +68,14 @@ export const CodeNode: FlowNodeTemplateType = {
       key: NodeInputKeyEnum.codeType,
       renderTypeList: [FlowNodeInputTypeEnum.hidden],
       label: '',
+      valueType: WorkflowIOValueTypeEnum.string,
       value: 'js'
     },
     {
       key: NodeInputKeyEnum.code,
       renderTypeList: [FlowNodeInputTypeEnum.custom],
       label: '',
+      valueType: WorkflowIOValueTypeEnum.string,
       value: JS_TEMPLATE
     }
   ],

packages/service/core/workflow/dispatch/code/run.ts

@@ -4,9 +4,10 @@ import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/ty
 import axios from 'axios';
 import { formatHttpError } from '../utils';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
+import { SandboxCodeTypeEnum } from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
 
 type RunCodeType = ModuleDispatchProps<{
-  [NodeInputKeyEnum.codeType]: 'js';
+  [NodeInputKeyEnum.codeType]: string;
   [NodeInputKeyEnum.code]: string;
   [NodeInputKeyEnum.addInputParam]: Record<string, any>;
 }>;
@@ -16,6 +17,14 @@ type RunCodeResponse = DispatchNodeResultType<{
   [key: string]: any;
 }>;
 
+function getURL(codeType: string): string {
+  if (codeType == SandboxCodeTypeEnum.py) {
+    return `${process.env.SANDBOX_URL}/sandbox/python`;
+  } else {
+    return `${process.env.SANDBOX_URL}/sandbox/js`;
+  }
+}
+
 export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeResponse> => {
   const {
     params: { codeType, code, [NodeInputKeyEnum.addInputParam]: customVariables }
@@ -27,7 +36,7 @@ export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeRespon
     };
   }
 
-  const sandBoxRequestUrl = `${process.env.SANDBOX_URL}/sandbox/js`;
+  const sandBoxRequestUrl = getURL(codeType);
   try {
     const { data: runResult } = await axios.post<{
       success: boolean;
@@ -40,6 +49,8 @@ export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeRespon
       variables: customVariables
     });
 
+    console.log(runResult);
+
     if (runResult.success) {
       return {
         [NodeOutputKeyEnum.rawResponse]: runResult.data.codeReturn,
@@ -52,7 +63,7 @@ export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeRespon
         ...runResult.data.codeReturn
       };
     } else {
-      throw new Error('Run code failed');
+      return Promise.reject('Run code failed');
     }
   } catch (error) {
     return {

packages/service/core/workflow/dispatch/utils.ts

@@ -106,6 +106,7 @@ export const getHistories = (history?: ChatItemType[] | number, histories: ChatI
 /* value type format */
 export const valueTypeFormat = (value: any, type?: WorkflowIOValueTypeEnum) => {
   if (value === undefined) return;
+  if (!type) return value;
 
   if (type === 'string') {
     if (typeof value !== 'object') return String(value);

packages/service/worker/readFile/extension/docx.ts

@@ -13,6 +13,7 @@ export const readDocsFile = async ({ buffer }: ReadRawTextByBuffer): Promise<Rea
         buffer
       },
       {
+        ignoreEmptyParagraphs: false,
         convertImage: images.imgElement(async (image) => {
           const imageBase64 = await image.readAsBase64String();
           const uuid = crypto.randomUUID();

packages/web/components/common/Textarea/CodeEditor/Editor.tsx

@@ -1,9 +1,9 @@
-import React, { useCallback, useRef, useState } from 'react';
+import React, { useCallback, useRef, useState, useEffect } from 'react';
 import Editor, { Monaco, loader } from '@monaco-editor/react';
 import { Box, BoxProps } from '@chakra-ui/react';
 import MyIcon from '../../Icon';
 import { getWebReqUrl } from '../../../../common/system/utils';
-
+import usePythonCompletion from './usePythonCompletion';
 loader.config({
   paths: { vs: getWebReqUrl('/js/monaco-editor.0.45.0/vs') }
 });
@@ -21,6 +21,7 @@ export type Props = Omit<BoxProps, 'resize' | 'onChange'> & {
   onOpenModal?: () => void;
   variables?: EditorVariablePickerType[];
   defaultHeight?: number;
+  language?: string;
 };
 
 const options = {
@@ -53,11 +54,14 @@ const MyEditor = ({
   variables = [],
   defaultHeight = 200,
   onOpenModal,
+  language = 'typescript',
   ...props
 }: Props) => {
   const [height, setHeight] = useState(defaultHeight);
   const initialY = useRef(0);
 
+  const registerPythonCompletion = usePythonCompletion();
+
   const handleMouseDown = useCallback((e: React.MouseEvent) => {
     initialY.current = e.clientY;
 
@@ -76,35 +80,47 @@ const MyEditor = ({
     document.addEventListener('mouseup', handleMouseUp);
   }, []);
 
-  const beforeMount = useCallback((monaco: Monaco) => {
+  const editorRef = useRef<any>(null);
-    monaco.languages.json.jsonDefaults.setDiagnosticsOptions({
+  const monacoRef = useRef<Monaco | null>(null);
-      validate: false,
-      allowComments: false,
-      schemas: [
-        {
-          uri: 'http://myserver/foo-schema.json', // 一个假设的 URI
-          fileMatch: ['*'], // 匹配所有文件
-          schema: {} // 空的 Schema
-        }
-      ]
-    });
 
-    monaco.editor.defineTheme('JSONEditorTheme', {
+  const handleEditorDidMount = useCallback((editor: any, monaco: Monaco) => {
-      base: 'vs', // 可以基于已有的主题进行定制
+    editorRef.current = editor;
-      inherit: true, // 继承基础主题的设置
+    monacoRef.current = monaco;
-      rules: [{ token: 'variable', foreground: '2B5FD9' }],
-      colors: {
-        'editor.background': '#ffffff00',
-        'editorLineNumber.foreground': '#aaa',
-        'editorOverviewRuler.border': '#ffffff00',
-        'editor.lineHighlightBackground': '#F7F8FA',
-        'scrollbarSlider.background': '#E8EAEC',
-        'editorIndentGuide.activeBackground': '#ddd',
-        'editorIndentGuide.background': '#eee'
-      }
-    });
   }, []);
 
+  const beforeMount = useCallback(
+    (monaco: Monaco) => {
+      monaco.languages.json.jsonDefaults.setDiagnosticsOptions({
+        validate: false,
+        allowComments: false,
+        schemas: [
+          {
+            uri: 'http://myserver/foo-schema.json', // 一个假设的 URI
+            fileMatch: ['*'], // 匹配所有文件
+            schema: {} // 空的 Schema
+          }
+        ]
+      });
+
+      monaco.editor.defineTheme('JSONEditorTheme', {
+        base: 'vs', // 可以基于已有的主题进行定制
+        inherit: true, // 继承基础主题的设置
+        rules: [{ token: 'variable', foreground: '2B5FD9' }],
+        colors: {
+          'editor.background': '#ffffff00',
+          'editorLineNumber.foreground': '#aaa',
+          'editorOverviewRuler.border': '#ffffff00',
+          'editor.lineHighlightBackground': '#F7F8FA',
+          'scrollbarSlider.background': '#E8EAEC',
+          'editorIndentGuide.activeBackground': '#ddd',
+          'editorIndentGuide.background': '#eee'
+        }
+      });
+      registerPythonCompletion(monaco);
+    },
+    [registerPythonCompletion]
+  );
+
   return (
     <Box
       borderWidth={'1px'}
@@ -118,7 +134,7 @@ const MyEditor = ({
     >
       <Editor
         height={'100%'}
-        defaultLanguage="typescript"
+        language={language}
         options={options as any}
         theme="JSONEditorTheme"
         beforeMount={beforeMount}
@@ -127,6 +143,7 @@ const MyEditor = ({
         onChange={(e) => {
           onChange?.(e || '');
         }}
+        onMount={handleEditorDidMount}
       />
       {resize && (
         <Box

packages/web/components/common/Textarea/CodeEditor/index.tsx

@@ -4,15 +4,31 @@ import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react'
 import MyModal from '../../MyModal';
 import { useTranslation } from 'next-i18next';
 
-type Props = Omit<EditorProps, 'resize'> & {};
+type Props = Omit<EditorProps, 'resize'> & { language?: string };
+function getLanguage(language: string | undefined): string {
+  let fullName: string;
+  switch (language) {
+    case 'py':
+      fullName = 'python';
+      break;
+    case 'js':
+      fullName = 'typescript';
+      break;
+    default:
+      fullName = `typescript`;
+      break;
+  }
+  return fullName;
+}
 
 const CodeEditor = (props: Props) => {
   const { t } = useTranslation();
   const { isOpen, onOpen, onClose } = useDisclosure();
-
+  const { language, ...otherProps } = props;
+  const fullName = getLanguage(language);
   return (
     <>
-      <MyEditor {...props} resize onOpenModal={onOpen} />
+      <MyEditor {...props} resize onOpenModal={onOpen} language={fullName} />
       <MyModal
         isOpen={isOpen}
         onClose={onClose}
@@ -23,7 +39,7 @@ const CodeEditor = (props: Props) => {
         isCentered
       >
         <ModalBody flex={'1 0 0'} overflow={'auto'}>
-          <MyEditor {...props} bg={'myGray.50'} height={'100%'} />
+          <MyEditor {...props} bg={'myGray.50'} height={'100%'} language={fullName} />
         </ModalBody>
         <ModalFooter>
           <Button mr={2} onClick={onClose} px={6}>

packages/web/components/common/Textarea/CodeEditor/usePythonCompletion.ts

@@ -0,0 +1,83 @@
+import { Monaco } from '@monaco-editor/react';
+import { useCallback } from 'react';
+let monacoInstance: Monaco | null = null;
+const usePythonCompletion = () => {
+  return useCallback((monaco: Monaco) => {
+    if (monacoInstance === monaco) return;
+    monacoInstance = monaco;
+
+    monaco.languages.registerCompletionItemProvider('python', {
+      provideCompletionItems: (model, position) => {
+        const wordInfo = model.getWordUntilPosition(position);
+        const currentWordPrefix = wordInfo.word;
+
+        const lineContent = model.getLineContent(position.lineNumber);
+
+        const range = {
+          startLineNumber: position.lineNumber,
+          endLineNumber: position.lineNumber,
+          startColumn: wordInfo.startColumn,
+          endColumn: wordInfo.endColumn
+        };
+
+        const baseSuggestions = [
+          {
+            label: 'len',
+            kind: monaco.languages.CompletionItemKind.Function,
+            insertText: 'len()',
+            documentation: 'get length of object',
+            range,
+            sortText: 'a'
+          }
+        ];
+
+        const filtered = baseSuggestions.filter((item) =>
+          item.label.toLowerCase().startsWith(currentWordPrefix.toLowerCase())
+        );
+
+        if (lineContent.startsWith('import')) {
+          const importLength = 'import'.length;
+          const afterImport = lineContent.slice(importLength);
+          const spaceMatch = afterImport.match(/^\s*/);
+          const spaceLength = spaceMatch ? spaceMatch[0].length : 0;
+
+          const startReplaceCol = importLength + spaceLength + 1;
+          const currentCol = position.column;
+
+          const replaceRange = new monaco.Range(
+            position.lineNumber,
+            startReplaceCol,
+            position.lineNumber,
+            currentCol
+          );
+
+          const needsSpace = spaceLength === 0;
+          return {
+            suggestions: [
+              {
+                label: 'numpy',
+                kind: monaco.languages.CompletionItemKind.Module,
+                insertText: `${needsSpace ? ' ' : ''}numpy as np`,
+                documentation: 'numerical computing library',
+                range: replaceRange,
+                sortText: 'a'
+              },
+              {
+                label: 'pandas',
+                kind: monaco.languages.CompletionItemKind.Module,
+                insertText: `${needsSpace ? ' ' : ''}pandas as pd`,
+                documentation: 'data analysis library',
+                range: replaceRange
+              }
+            ]
+          };
+        }
+
+        return { suggestions: filtered };
+      },
+      triggerCharacters: ['.', '_']
+    });
+  }, []);
+};
+
+export default usePythonCompletion;

packages/web/i18n/en/workflow.json

@@ -20,6 +20,7 @@
   "classification_result": "Classification Result",
   "code.Reset template": "Reset Template",
   "code.Reset template confirm": "Confirm reset code template? This will reset all inputs and outputs to template values. Please save your current code.",
+  "code.Switch language confirm": "Switching the language will reset the code, will it continue?",
   "code_execution": "Code Sandbox",
   "collection_metadata_filter": "Collection Metadata Filter",
   "complete_extraction_result": "Complete Extraction Result",
@@ -153,6 +154,7 @@
   "select_another_application_to_call": "You can choose another application to call",
   "special_array_format": "Special array format, returns an empty array when the search result is empty.",
   "start_with": "Starts With",
+  "support_code_language": "Support import list: pandas，numpy",
   "target_fields_description": "A target field consists of 'description' and 'key'. Multiple target fields can be extracted.",
   "template.ai_chat": "AI Chat",
   "template.ai_chat_intro": "AI Large Model Chat",

packages/web/i18n/zh-CN/workflow.json

@@ -20,6 +20,7 @@
   "classification_result": "分类结果",
   "code.Reset template": "还原模板",
   "code.Reset template confirm": "确认还原代码模板？将会重置所有输入和输出至模板值，请注意保存当前代码。",
+  "code.Switch language confirm": "切换语言将重置代码，是否继续？",
   "code_execution": "代码运行",
   "collection_metadata_filter": "集合元数据过滤",
   "complete_extraction_result": "完整提取结果",
@@ -153,6 +154,7 @@
   "select_another_application_to_call": "可以选择一个其他应用进行调用",
   "special_array_format": "特殊数组格式，搜索结果为空时，返回空数组。",
   "start_with": "开始为",
+  "support_code_language": "支持import列表：pandas，numpy",
   "target_fields_description": "由 '描述' 和 'key' 组成一个目标字段，可提取多个目标字段",
   "template.ai_chat": "AI 对话",
   "template.ai_chat_intro": "AI 大模型对话",

packages/web/i18n/zh-Hant/workflow.json

@@ -20,6 +20,7 @@
   "classification_result": "分類結果",
   "code.Reset template": "重設範本",
   "code.Reset template confirm": "確定要重設程式碼範本嗎？這將會把所有輸入和輸出重設為範本值。請儲存您目前的程式碼。",
+  "code.Switch language confirm": "切換語言將重置代碼，是否繼續？",
   "code_execution": "程式碼執行",
   "collection_metadata_filter": "資料集詮釋資料篩選器",
   "complete_extraction_result": "完整擷取結果",
@@ -153,6 +154,7 @@
   "select_another_application_to_call": "可以選擇另一個應用程式來呼叫",
   "special_array_format": "特殊陣列格式，搜尋結果為空時，回傳空陣列。",
   "start_with": "開頭為",
+  "support_code_language": "支持import列表：pandas，numpy",
   "target_fields_description": "由「描述」和「鍵值」組成一個目標欄位，可以擷取多個目標欄位",
   "template.ai_chat": "AI 對話",
   "template.ai_chat_intro": "AI 大型語言模型對話",

plugins/webcrawler/deploy/docker-compose.yaml

@@ -0,0 +1,80 @@
+name: spider
+version: "2.2"
+
+services:
+  searxng:
+    container_name: searxng
+    image: docker.io/searxng/searxng:latest
+    platform: linux/amd64
+    restart: unless-stopped
+    networks:
+      - spider_net
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./searxng:/etc/searxng:rw
+    environment:
+      - SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
+      - UWSGI_WORKERS=4 # UWSGI 工作进程数
+      - UWSGI_THREADS=4 # UWSGI 线程数
+    cap_drop:
+      - ALL
+
+  mongodb:
+    container_name: mongodb
+    image: mongo:4.4
+    restart: unless-stopped
+    networks:
+      - spider_net
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongo-data:/data/db
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root # MongoDB 根用户名
+      MONGO_INITDB_ROOT_PASSWORD: example # MongoDB 根用户密码
+
+  nodeapp:
+    container_name: main
+    platform: linux/amd64
+    #build:
+    #  context: . 
+    image: gggaaallleee/webcrawler-test-new:latest
+    ports:
+      - "3000:3000"
+    networks:
+      - spider_net
+    depends_on:
+      - mongodb
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1m"
+        max-file: "1"
+    volumes:
+      - /dev/shm:/dev/shm
+    environment:
+      - ACCESS_TOKEN=webcrawler # 访问令牌
+      - DETECT_WEBSITE=zhuanlan.zhihu.com # 无法处理跳过的网站
+      - STRATEGIES=[{"waitUntil":"networkidle0","timeout":5000},{"waitUntil":"networkidle2","timeout":10000},{"waitUntil":"load","timeout":15000}] # 页面加载策略
+      - PORT=3000
+      - MAX_CONCURRENCY=10 # 最大并发数
+      - NODE_ENV=development
+      - ENGINE_BAIDUURL=https://www.baidu.com/s  # 百度搜索引擎 URL
+      - ENGINE_SEARCHXNGURL=http://searxng:8080/search # Searxng 搜索引擎 URL
+      - MONGODB_URI=mongodb://root:example@mongodb:27017 # MongoDB 连接 URI
+      - BLACKLIST=[".gov.cn",".edu.cn"] # 受保护域名
+      - STD_TTL=3600 # 标准 TTL（秒）
+      - EXPIRE_AFTER_SECONDS=9000 # 过期时间（秒）
+     #- VALIDATE_PROXY=[{"ip":"","port":},{"ip":"","port":}] #代理池
+    deploy:
+        resources:
+          limits:
+            memory: 4G
+            cpus: '2.0'
+
+networks:
+  spider_net:
+
+volumes:
+  mongo-data:

plugins/webcrawler/deploy/searxng/limiter.toml

@@ -0,0 +1,6 @@
+# This configuration file updates the default configuration file
+# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
+
+[botdetection.ip_limit]
+# activate link_token method in the ip_limit method
+link_token = true

plugins/webcrawler/deploy/searxng/settings.yml

@@ -0,0 +1,122 @@
+general:
+  debug: false
+  instance_name: "searxng"
+  privacypolicy_url: false
+  donation_url: false
+  contact_url: false
+  enable_metrics: true
+  open_metrics: ''
+
+brand:
+  new_issue_url: https://github.com/searxng/searxng/issues/new
+  docs_url: https://docs.searxng.org/
+  public_instances: https://searx.space
+  wiki_url: https://github.com/searxng/searxng/wiki
+  issue_url: https://github.com/searxng/searxng/issues
+
+search:
+  safe_search: 0
+  autocomplete: ""
+  autocomplete_min: 4
+  default_lang: "auto"
+  ban_time_on_fail: 5
+  max_ban_time_on_fail: 120
+  formats:
+    - html
+
+server:
+  port: 8080
+  bind_address: "0.0.0.0"
+  base_url: false
+  limiter: false
+  public_instance: false
+  secret_key: "example"
+  image_proxy: false
+  http_protocol_version: "1.0"
+  method: "POST"
+  default_http_headers:
+    X-Content-Type-Options: nosniff
+    X-Download-Options: noopen
+    X-Robots-Tag: noindex, nofollow
+    Referrer-Policy: no-referrer
+
+redis:
+  url: false
+
+ui:
+  static_path: ""
+  static_use_hash: false
+  templates_path: ""
+  default_theme: simple
+  default_locale: ""
+  query_in_title: false
+  infinite_scroll: false
+  center_alignment: false
+  theme_args:
+    simple_style: auto
+      # 启用 cn 分类
+  enabled_categories: [cn,en, general, images,en]
+  # 或者定义分类显示顺序
+  categories_order: [cn, en,general, images]
+
+outgoing:
+  request_timeout: 30.0
+  max_request_timeout: 40.0
+  pool_connections: 200
+  pool_maxsize: 50
+  enable_http2: false
+  retries: 5
+
+engines:
+    - name: bing
+      engine: bing
+      disabled: false
+      categories: cn
+    #- name: bilibili
+     # engine: bilibili
+     # shortcut: bil
+     # disabled: false
+     # categories: cn
+    - name : baidu
+      engine : json_engine
+      paging : True
+      first_page_num : 0
+      search_url : https://www.baidu.com/s?tn=json&wd={query}&pn={pageno}&rn=50
+      url_query : url
+      title_query : title
+      content_query : abs
+      categories : cn
+    - name : 360search
+      engine: 360search
+      disabled: false
+      categories: cn
+    - name : sogou
+      disabled: false
+      categories: cn
+
+    - name: google
+      disabled: false
+      categories: en
+    - name: yahoo
+      disabled: false
+      categories: en
+    - name: duckduckgo
+      disabled: false
+      categories: en
+
+
+
+search:
+    formats:
+        - html
+        - json
+doi_resolvers:
+  oadoi.org: 'https://oadoi.org/'
+  doi.org: 'https://doi.org/'
+  doai.io: 'https://dissem.in/'
+  sci-hub.se: 'https://sci-hub.se/'
+  sci-hub.st: 'https://sci-hub.st/'
+  sci-hub.ru: 'https://sci-hub.ru/'
+
+default_doi_resolver: 'oadoi.org'
+

projects/README.md

@@ -2,5 +2,15 @@
 
 该目录为 FastGPT 主项目。
 
-- app 前端项目，用于展示和使用 FastGPT。
+- app fastgpt 核心应用。
-- sandbox 沙盒项目，用于测试和开发。
+- sandbox 沙盒项目，用于运行工作流里的代码执行 （需求python环境为python:3.11，额外安装的包请于requirements.txt填写，同时注意个别包可能额外安装库（如pandas需要安装libffi））。
+  - 新加入python包遇见超时或者权限拦截的问题(确定不是自己的语法问题)，请进入docker容器内部执行以下指令：
+
+  ```shell
+    docker exec -it 《替换成容器名》 /bin/bash
+    chmod -x testSystemCall.sh
+    bash ./testSystemCall.sh
+  ```
+  
+    然后将新的数组替换src下sandbox的constants.py中的SYSTEM_CALLS数组即可
+

projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodeCode.tsx

@@ -15,38 +15,89 @@ import RenderOutput from './render/RenderOutput';
 import CodeEditor from '@fastgpt/web/components/common/Textarea/CodeEditor';
 import { Box, Flex } from '@chakra-ui/react';
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
-import { JS_TEMPLATE } from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
+import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
+import {
+  JS_TEMPLATE,
+  PY_TEMPLATE,
+  SandboxCodeTypeEnum,
+  SNADBOX_CODE_TEMPLATE
+} from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
+import MySelect from '@fastgpt/web/components/common/MySelect';
 
 const NodeCode = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
   const { t } = useTranslation();
   const { nodeId, inputs, outputs } = data;
 
+  const codeType = inputs.find(
+    (item) => item.key === NodeInputKeyEnum.codeType
+  ) as FlowNodeInputItemType;
+
   const splitToolInputs = useContextSelector(WorkflowContext, (ctx) => ctx.splitToolInputs);
   const onChangeNode = useContextSelector(WorkflowContext, (ctx) => ctx.onChangeNode);
 
-  const { ConfirmModal, openConfirm } = useConfirm({
+  // 重置模板确认
+  const { ConfirmModal: ResetTemplateConfirm, openConfirm: openResetTemplateConfirm } = useConfirm({
     content: t('workflow:code.Reset template confirm')
   });
 
+  // 切换语言确认
+  const { ConfirmModal: SwitchLangConfirm, openConfirm: openSwitchLangConfirm } = useConfirm({
+    content: t('workflow:code.Switch language confirm')
+  });
+
   const CustomComponent = useMemo(() => {
     return {
       [NodeInputKeyEnum.code]: (item: FlowNodeInputItemType) => {
         return (
-          <Box mt={-3}>
+          <Box mt={-4}>
-            <Flex mb={2} alignItems={'flex-end'}>
+            <Flex mb={2} alignItems={'center'} className="nodrag">
-              <Box flex={'1'}>{'Javascript ' + t('workflow:Code')}</Box>
+              <MySelect<SandboxCodeTypeEnum>
+                fontSize="xs"
+                size="sm"
+                list={[
+                  { label: 'JavaScript', value: SandboxCodeTypeEnum.js },
+                  { label: 'Python 3', value: SandboxCodeTypeEnum.py }
+                ]}
+                value={codeType?.value}
+                onChange={(newLang) => {
+                  console.log(newLang);
+                  openSwitchLangConfirm(() => {
+                    onChangeNode({
+                      nodeId,
+                      type: 'updateInput',
+                      key: NodeInputKeyEnum.codeType,
+                      value: { ...codeType, value: newLang }
+                    });
+
+                    onChangeNode({
+                      nodeId,
+                      type: 'updateInput',
+                      key: item.key,
+                      value: {
+                        ...item,
+                        value: SNADBOX_CODE_TEMPLATE[newLang]
+                      }
+                    });
+                  })();
+                }}
+              />
+              {codeType.value === 'py' && (
+                <QuestionTip ml={2} label={t('workflow:support_code_language')} />
+              )}
               <Box
                 cursor={'pointer'}
                 color={'primary.500'}
                 fontSize={'xs'}
-                onClick={openConfirm(() => {
+                ml="auto"
+                mr={2}
+                onClick={openResetTemplateConfirm(() => {
                   onChangeNode({
                     nodeId,
                     type: 'updateInput',
                     key: item.key,
                     value: {
                       ...item,
-                      value: JS_TEMPLATE
+                      value: codeType.value === 'js' ? JS_TEMPLATE : PY_TEMPLATE
                     }
                   });
                 })}
@@ -63,29 +114,25 @@ const NodeCode = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
                   nodeId,
                   type: 'updateInput',
                   key: item.key,
-                  value: {
+                  value: { ...item, value: e }
-                    ...item,
-                    value: e
-                  }
                 });
               }}
+              language={codeType.value}
             />
           </Box>
         );
       }
     };
-  }, [nodeId, onChangeNode, openConfirm, t]);
+  }, [codeType, nodeId, onChangeNode, openResetTemplateConfirm, openSwitchLangConfirm, t]);
 
   const { isTool, commonInputs } = splitToolInputs(inputs, nodeId);
 
   return (
     <NodeCard minW={'400px'} selected={selected} {...data}>
       {isTool && (
-        <>
+        <Container>
-          <Container>
+          <RenderToolInput nodeId={nodeId} inputs={inputs} />
-            <RenderToolInput nodeId={nodeId} inputs={inputs} />
+        </Container>
-          </Container>
-        </>
       )}
       <Container>
         <IOTitle text={t('common:common.Input')} mb={-1} />
@@ -99,7 +146,8 @@ const NodeCode = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
         <IOTitle text={t('common:common.Output')} />
         <RenderOutput nodeId={nodeId} flowOutputList={outputs} />
       </Container>
-      <ConfirmModal />
+      <ResetTemplateConfirm />
+      <SwitchLangConfirm />
     </NodeCard>
   );
 };

projects/sandbox/Dockerfile

@@ -1,8 +1,22 @@
 # --------- install dependence -----------
 FROM python:3.11-alpine AS python_base
-
+ENV VERSION_RELEASE = Alpine3.11
 # 安装make和g++
-RUN apk add --no-cache make g++
+RUN apk add --no-cache make g++ tar wget gperf automake libtool linux-headers
+
+WORKDIR /app
+COPY projects/sandbox/requirements.txt /app/requirements.txt
+RUN wget https://github.com/seccomp/libseccomp/releases/download/v2.5.5/libseccomp-2.5.5.tar.gz && \
+    tar -zxvf libseccomp-2.5.5.tar.gz && \
+    cd libseccomp-2.5.5 && \
+    ./configure --prefix=/usr && \
+    make && \
+    make install && \
+    pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple Cython && \
+    pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple -r /app/requirements.txt && \
+    cd src/python && \
+    python setup.py install  
+
 
 FROM node:20.14.0-alpine AS install
 
@@ -10,7 +24,7 @@ WORKDIR /app
 
 ARG proxy
 RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
-RUN apk add --no-cache make g++
+RUN apk add --no-cache make g++ python3
 
 # copy py3.11
 COPY --from=python_base /usr/local /usr/local
@@ -42,9 +56,12 @@ RUN pnpm --filter=sandbox build
 FROM node:20.14.0-alpine AS runner
 WORKDIR /app
 
+RUN apk add --no-cache libffi libffi-dev strace bash
+COPY --from=python_base /usr/local /usr/local
 COPY --from=builder /app/node_modules /app/node_modules
 COPY --from=builder /app/projects/sandbox /app/projects/sandbox
 
 ENV NODE_ENV=production
+ENV PATH="/usr/local/bin:${PATH}"
 
 CMD ["node", "--no-node-snapshot", "projects/sandbox/dist/main.js"]

projects/sandbox/requirements.txt

@@ -0,0 +1,2 @@
+numpy
+pandas

projects/sandbox/src/sandbox/constants.ts

@@ -0,0 +1,130 @@
+export const pythonScript = `
+import subprocess
+import json
+import ast
+import base64
+
+def extract_imports(code):
+    tree = ast.parse(code)
+    imports = []
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.Import, ast.ImportFrom)):
+            if isinstance(node, ast.Import):
+                for alias in node.names:
+                    imports.append(f"import {alias.name}")
+            elif isinstance(node, ast.ImportFrom):
+                module = node.module
+                for alias in node.names:
+                    imports.append(f"from {module} import {alias.name}")
+    return imports
+seccomp_prefix = """
+from seccomp import *
+import sys
+allowed_syscalls = [
+    "syscall.SYS_ARCH_PRCTL", "syscall.SYS_BRK", "syscall.SYS_CLONE",
+    "syscall.SYS_CLOSE", "syscall.SYS_EPOLL_CREATE1", "syscall.SYS_EXECVE",
+    "syscall.SYS_EXIT", "syscall.SYS_EXIT_GROUP", "syscall.SYS_FCNTL",
+    "syscall.SYS_FSTAT", "syscall.SYS_FUTEX", "syscall.SYS_GETDENTS64",
+    "syscall.SYS_GETEGID", "syscall.SYS_GETEUID", "syscall.SYS_GETGID",
+    "syscall.SYS_GETRANDOM", "syscall.SYS_GETTID", "syscall.SYS_GETUID",
+    "syscall.SYS_IOCTL", "syscall.SYS_LSEEK", "syscall.SYS_LSTAT",
+    "syscall.SYS_MBIND", "syscall.SYS_MEMBARRIER", "syscall.SYS_MMAP",
+    "syscall.SYS_MPROTECT", "syscall.SYS_MUNMAP", "syscall.SYS_OPEN",
+    "syscall.SYS_PREAD64", "syscall.SYS_READ", "syscall.SYS_READLINK",
+    "syscall.SYS_READV", "syscall.SYS_RT_SIGACTION", "syscall.SYS_RT_SIGPROCMASK",
+    "syscall.SYS_SCHED_GETAFFINITY", "syscall.SYS_SET_TID_ADDRESS",
+    "syscall.SYS_STAT", "syscall.SYS_UNAME",
+    "syscall.SYS_MREMAP", "syscall.SYS_RT_SIGRETURN", "syscall.SYS_SETUID",
+    "syscall.SYS_SETGID", "syscall.SYS_GETPID", "syscall.SYS_GETPPID",
+    "syscall.SYS_TGKILL", "syscall.SYS_SCHED_YIELD", "syscall.SYS_SET_ROBUST_LIST",
+    "syscall.SYS_GET_ROBUST_LIST", "syscall.SYS_RSEQ", "syscall.SYS_CLOCK_GETTIME",
+    "syscall.SYS_GETTIMEOFDAY", "syscall.SYS_NANOSLEEP", "syscall.SYS_EPOLL_CTL",
+    "syscall.SYS_CLOCK_NANOSLEEP", "syscall.SYS_PSELECT6", "syscall.SYS_TIME",
+    "syscall.SYS_SIGALTSTACK", "syscall.SYS_MKDIRAT", "syscall.SYS_MKDIR"
+]
+allowed_syscalls_tmp = allowed_syscalls
+L = []
+for item in allowed_syscalls_tmp:
+    item = item.strip()
+    parts = item.split(".")[1][4:].lower()
+    L.append(parts)
+f = SyscallFilter(defaction=KILL)
+for item in L:
+    f.add_rule(ALLOW, item)
+f.add_rule(ALLOW, "write", Arg(0, EQ, sys.stdout.fileno()))
+f.add_rule(ALLOW, "write", Arg(0, EQ, sys.stderr.fileno()))
+f.add_rule(ALLOW, 307)
+f.add_rule(ALLOW, 318)
+f.add_rule(ALLOW, 334)
+f.load()
+"""
+
+def remove_print_statements(code):
+    class PrintRemover(ast.NodeTransformer):
+        def visit_Expr(self, node):
+            if (
+                isinstance(node.value, ast.Call)
+                and isinstance(node.value.func, ast.Name)
+                and node.value.func.id == "print"
+            ):
+                return None
+            return node
+
+    tree = ast.parse(code)
+    modified_tree = PrintRemover().visit(tree)
+    ast.fix_missing_locations(modified_tree)
+    return ast.unparse(modified_tree)
+
+def detect_dangerous_imports(code):
+    dangerous_modules = ["os", "sys", "subprocess", "shutil", "socket", "ctypes", "multiprocessing", "threading", "pickle"]
+    tree = ast.parse(code)
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                if alias.name in dangerous_modules:
+                    return alias.name
+        elif isinstance(node, ast.ImportFrom):
+            if node.module in dangerous_modules:
+                return node.module
+    return None
+
+def run_pythonCode(data:dict):
+    if not data or "code" not in data or "variables" not in data:
+        return {"error": "Invalid request format"}
+    code = data["code"]
+    code = remove_print_statements(code)
+    dangerous_import = detect_dangerous_imports(code)
+    if dangerous_import:
+        return {"error": f"Importing {dangerous_import} is not allowed."}
+    variables = data["variables"]
+    imports = "\\n".join(extract_imports(code))
+    var_def = ""
+    output_code = "res = main("
+    for k, v in variables.items():
+        if isinstance(v, str):
+            one_var = k + " = \\"" + v + "\\"\\n"
+        else:
+            one_var = k + " = " + str(v) + "\\n"
+        var_def = var_def + one_var
+        output_code = output_code + k + ", "
+    if output_code[-1] == "(":
+        output_code = output_code + ")\\n"
+    else:
+        output_code = output_code[:-2] + ")\\n"
+    output_code = output_code + "print(res)"
+    code = imports + "\\n" + seccomp_prefix + "\\n" + var_def + "\\n" + code + "\\n" + output_code
+    try:
+        result = subprocess.run(["python3", "-c", code], capture_output=True, text=True, timeout=10)
+        if result.returncode == -31:
+            return {"error": "Dangerous behavior detected."}
+        if result.stderr != "":
+            return {"error": result.stderr}
+
+        out = ast.literal_eval(result.stdout.strip())
+        return out
+    except subprocess.TimeoutExpired:
+        return {"error": "Timeout error"}
+    except Exception as e:
+        return {"error": str(e)}
+
+`;

projects/sandbox/src/sandbox/sandbox.controller.ts

@@ -1,6 +1,6 @@
 import { Controller, Post, Body, HttpCode } from '@nestjs/common';
 import { RunCodeDto } from './dto/create-sandbox.dto';
-import { runSandbox } from './utils';
+import { runJsSandbox, runPythonSandbox } from './utils';
 
 @Controller('sandbox')
 export class SandboxController {
@@ -9,6 +9,12 @@ export class SandboxController {
   @Post('/js')
   @HttpCode(200)
   runJs(@Body() codeProps: RunCodeDto) {
-    return runSandbox(codeProps);
+    return runJsSandbox(codeProps);
+  }
+
+  @Post('/python')
+  @HttpCode(200)
+  runPython(@Body() codeProps: RunCodeDto) {
+    return runPythonSandbox(codeProps);
   }
 }

projects/sandbox/src/sandbox/utils.ts

@@ -6,24 +6,30 @@ import { timeDelay } from './jsFn/delay';
 import { strToBase64 } from './jsFn/str2Base64';
 import { createHmac } from './jsFn/crypto';
 
+import { spawn } from 'child_process';
+import { pythonScript } from './constants';
 const CustomLogStr = 'CUSTOM_LOG';
 
-/* 
+export const runJsSandbox = async ({
-  Rewrite code to add custom functions: Promise function; Log.
+  code,
-*/
+  variables = {}
-function getFnCode(code: string) {
+}: RunCodeDto): Promise<RunCodeResponse> => {
-  // rewrite log
+  /* 
-  code = code.replace(/console\.log/g, `${CustomLogStr}`);
+    Rewrite code to add custom functions: Promise function; Log.
+  */
+  function getFnCode(code: string) {
+    // rewrite log
+    code = code.replace(/console\.log/g, `${CustomLogStr}`);
 
-  // Promise function rewrite
+    // Promise function rewrite
-  const rewriteSystemFn = `
+    const rewriteSystemFn = `
     const thisDelay = (...args) => global_delay.applySyncPromise(undefined,args)
 `;
 
-  // rewrite delay
+    // rewrite delay
-  code = code.replace(/delay\((.*)\)/g, `thisDelay($1)`);
+    code = code.replace(/delay\((.*)\)/g, `thisDelay($1)`);
 
-  const runCode = `
+    const runCode = `
     (async() => { 
         try {
             ${rewriteSystemFn}
@@ -36,23 +42,18 @@ function getFnCode(code: string) {
         }
     })
 `;
-  return runCode;
+    return runCode;
-}
+  }
+  // Register global function
+  function registerSystemFn(jail: IsolatedVM.Reference<Record<string | number | symbol, any>>) {
+    return Promise.all([
+      jail.set('global_delay', new Reference(timeDelay)),
+      jail.set('countToken', countToken),
+      jail.set('strToBase64', strToBase64),
+      jail.set('createHmac', createHmac)
+    ]);
+  }
 
-// Register global function
-function registerSystemFn(jail: IsolatedVM.Reference<Record<string | number | symbol, any>>) {
-  return Promise.all([
-    jail.set('global_delay', new Reference(timeDelay)),
-    jail.set('countToken', countToken),
-    jail.set('strToBase64', strToBase64),
-    jail.set('createHmac', createHmac)
-  ]);
-}
-
-export const runSandbox = async ({
-  code,
-  variables = {}
-}: RunCodeDto): Promise<RunCodeResponse> => {
   const logData = [];
 
   const isolate = new Isolate({ memoryLimit: 32 });
@@ -106,3 +107,50 @@ export const runSandbox = async ({
     return Promise.reject(err);
   }
 };
+
+export const runPythonSandbox = async ({
+  code,
+  variables = {}
+}: RunCodeDto): Promise<RunCodeResponse> => {
+  const mainCallCode = `
+data = ${JSON.stringify({ code, variables })}
+res = run_pythonCode(data)
+print(json.dumps(res))
+`;
+
+  const fullCode = [pythonScript, mainCallCode].filter(Boolean).join('\n');
+
+  const pythonProcess = spawn('python3', ['-u', '-c', fullCode]);
+
+  const stdoutChunks: string[] = [];
+  const stderrChunks: string[] = [];
+
+  pythonProcess.stdout.on('data', (data) => stdoutChunks.push(data.toString()));
+  pythonProcess.stderr.on('data', (data) => stderrChunks.push(data.toString()));
+
+  const stdoutPromise = new Promise<string>((resolve) => {
+    pythonProcess.on('close', (code) => {
+      if (code !== 0) {
+        resolve(JSON.stringify({ error: stderrChunks.join('') }));
+      } else {
+        resolve(stdoutChunks.join(''));
+      }
+    });
+  });
+  const stdout = await stdoutPromise;
+
+  try {
+    const parsedOutput = JSON.parse(stdout);
+    if (parsedOutput.error) {
+      return Promise.reject(parsedOutput.error || 'Unknown error');
+    }
+    return { codeReturn: parsedOutput, log: '' };
+  } catch (err) {
+    if (stdout.includes('malformed node or string on line 1')) {
+      return Promise.reject(`The result should be a parsable variable, such as a list.  ${stdout}`);
+    } else if (stdout.includes('Unexpected end of JSON input')) {
+      return Promise.reject(`Not allowed print or ${stdout}`);
+    }
+    return Promise.reject(`Run failed: ${err}`);
+  }
+};

projects/sandbox/testSystemCall.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+temp_dir=$(mktemp -d)
+trap 'rm -rf "$temp_dir"' EXIT
+
+syscall_table_file="$temp_dir/syscall_table.txt"
+code_file="$temp_dir/test_code.py"
+strace_log="$temp_dir/strace.log"
+syscalls_file="$temp_dir/syscalls.txt"
+
+code='
+import pandas as pd
+def main():
+    data = {"Name": ["Alice", "Bob"], "Age": [25, 30]}
+    df = pd.DataFrame(data)
+    return {
+        "head": df.head().to_dict()
+    }
+'
+
+if ! ausyscall --dump > "$syscall_table_file" 2>/dev/null; then
+    grep -E '^#define __NR_' /usr/include/asm/unistd_64.h | \
+        sed 's/#define __NR_//;s/[ \t]\+/ /g' | \
+        awk '{print $1, $2}' > "$syscall_table_file"
+fi
+
+echo "$code" > "$code_file"
+
+strace -ff -e trace=all -o "$strace_log" python3 "$code_file" >/dev/null 2>&1
+
+cat "$strace_log"* 2>/dev/null | grep -oE '^[[:alnum:]_]+' | sort -u > "$syscalls_file"
+
+allowed_syscalls=()
+while read raw_name; do
+    go_name=$(echo "$raw_name" | tr 'a-z' 'A-Z' | sed 's/-/_/g')
+    allowed_syscalls+=("\"syscall.SYS_${go_name}\"")
+done < "$syscalls_file"
+
+echo "allowed_syscalls = ["
+printf '    %s,\n' "${allowed_syscalls[@]}" | paste -sd '    \n'
+echo "]"

test/cases/function/packages/global/common/string/textSplitter.test.ts

@@ -436,6 +436,28 @@ FastGPT是一款基于大语言模型（LLM）的智能问答系统，专为提
   expect(chunks).toEqual(mock.result);
 });
 
+// 自定义分隔符测试：换行符号
+it(`Test splitText2Chunks 1`, () => {
+  const mock = {
+    text: `111
+222
+
+333`,
+    result: [
+      `111
+222`,
+      '333'
+    ]
+  };
+
+  const { chunks } = splitText2Chunks({ customReg: ['\\n\\n'], text: mock.text, chunkSize: 2000 });
+  fs.writeFileSync(
+    '/Users/yjl/fastgpt-pro/FastGPT/test/cases/function/packages/global/common/string/test.md',
+    chunks.join('------')
+  );
+  expect(chunks).toEqual(mock.result);
+});
+
 // 长代码块分割
 it(`Test splitText2Chunks 7`, () => {
   const mock = {