mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-22 12:20:34 +00:00
Python Sandbox (#4380)
* Python3 Sandbox (#3944) * update python box (#4251) * update python box * Adjust the height of the NodeCode border. * update python sandbox and add test systemcall bash * update sandbox * add VERSION_RELEASE (#4376) * save empty docx * fix pythonbox log error * fix: js template --------- Co-authored-by: dogfar <37035781+dogfar@users.noreply.github.com> Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> Co-authored-by: gggaaallleee <1293587368@qq.com>
This commit is contained in:
@@ -5,3 +5,19 @@ export const JS_TEMPLATE = `function main({data1, data2}){
|
||||
data2
|
||||
}
|
||||
}`;
|
||||
|
||||
export const PY_TEMPLATE = `def main(data1, data2):
|
||||
return {
|
||||
"result": data1,
|
||||
"data2": data2
|
||||
}
|
||||
`;
|
||||
|
||||
export enum SandboxCodeTypeEnum {
|
||||
js = 'js',
|
||||
py = 'py'
|
||||
}
|
||||
export const SNADBOX_CODE_TEMPLATE = {
|
||||
[SandboxCodeTypeEnum.js]: JS_TEMPLATE,
|
||||
[SandboxCodeTypeEnum.py]: PY_TEMPLATE
|
||||
};
|
||||
|
@@ -68,12 +68,14 @@ export const CodeNode: FlowNodeTemplateType = {
|
||||
key: NodeInputKeyEnum.codeType,
|
||||
renderTypeList: [FlowNodeInputTypeEnum.hidden],
|
||||
label: '',
|
||||
valueType: WorkflowIOValueTypeEnum.string,
|
||||
value: 'js'
|
||||
},
|
||||
{
|
||||
key: NodeInputKeyEnum.code,
|
||||
renderTypeList: [FlowNodeInputTypeEnum.custom],
|
||||
label: '',
|
||||
valueType: WorkflowIOValueTypeEnum.string,
|
||||
value: JS_TEMPLATE
|
||||
}
|
||||
],
|
||||
|
@@ -4,9 +4,10 @@ import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/ty
|
||||
import axios from 'axios';
|
||||
import { formatHttpError } from '../utils';
|
||||
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
|
||||
import { SandboxCodeTypeEnum } from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
|
||||
|
||||
type RunCodeType = ModuleDispatchProps<{
|
||||
[NodeInputKeyEnum.codeType]: 'js';
|
||||
[NodeInputKeyEnum.codeType]: string;
|
||||
[NodeInputKeyEnum.code]: string;
|
||||
[NodeInputKeyEnum.addInputParam]: Record<string, any>;
|
||||
}>;
|
||||
@@ -16,6 +17,14 @@ type RunCodeResponse = DispatchNodeResultType<{
|
||||
[key: string]: any;
|
||||
}>;
|
||||
|
||||
function getURL(codeType: string): string {
|
||||
if (codeType == SandboxCodeTypeEnum.py) {
|
||||
return `${process.env.SANDBOX_URL}/sandbox/python`;
|
||||
} else {
|
||||
return `${process.env.SANDBOX_URL}/sandbox/js`;
|
||||
}
|
||||
}
|
||||
|
||||
export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeResponse> => {
|
||||
const {
|
||||
params: { codeType, code, [NodeInputKeyEnum.addInputParam]: customVariables }
|
||||
@@ -27,7 +36,7 @@ export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeRespon
|
||||
};
|
||||
}
|
||||
|
||||
const sandBoxRequestUrl = `${process.env.SANDBOX_URL}/sandbox/js`;
|
||||
const sandBoxRequestUrl = getURL(codeType);
|
||||
try {
|
||||
const { data: runResult } = await axios.post<{
|
||||
success: boolean;
|
||||
@@ -40,6 +49,8 @@ export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeRespon
|
||||
variables: customVariables
|
||||
});
|
||||
|
||||
console.log(runResult);
|
||||
|
||||
if (runResult.success) {
|
||||
return {
|
||||
[NodeOutputKeyEnum.rawResponse]: runResult.data.codeReturn,
|
||||
@@ -52,7 +63,7 @@ export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeRespon
|
||||
...runResult.data.codeReturn
|
||||
};
|
||||
} else {
|
||||
throw new Error('Run code failed');
|
||||
return Promise.reject('Run code failed');
|
||||
}
|
||||
} catch (error) {
|
||||
return {
|
||||
|
@@ -106,6 +106,7 @@ export const getHistories = (history?: ChatItemType[] | number, histories: ChatI
|
||||
/* value type format */
|
||||
export const valueTypeFormat = (value: any, type?: WorkflowIOValueTypeEnum) => {
|
||||
if (value === undefined) return;
|
||||
if (!type) return value;
|
||||
|
||||
if (type === 'string') {
|
||||
if (typeof value !== 'object') return String(value);
|
||||
|
@@ -13,6 +13,7 @@ export const readDocsFile = async ({ buffer }: ReadRawTextByBuffer): Promise<Rea
|
||||
buffer
|
||||
},
|
||||
{
|
||||
ignoreEmptyParagraphs: false,
|
||||
convertImage: images.imgElement(async (image) => {
|
||||
const imageBase64 = await image.readAsBase64String();
|
||||
const uuid = crypto.randomUUID();
|
||||
|
@@ -1,9 +1,9 @@
|
||||
import React, { useCallback, useRef, useState } from 'react';
|
||||
import React, { useCallback, useRef, useState, useEffect } from 'react';
|
||||
import Editor, { Monaco, loader } from '@monaco-editor/react';
|
||||
import { Box, BoxProps } from '@chakra-ui/react';
|
||||
import MyIcon from '../../Icon';
|
||||
import { getWebReqUrl } from '../../../../common/system/utils';
|
||||
|
||||
import usePythonCompletion from './usePythonCompletion';
|
||||
loader.config({
|
||||
paths: { vs: getWebReqUrl('/js/monaco-editor.0.45.0/vs') }
|
||||
});
|
||||
@@ -21,6 +21,7 @@ export type Props = Omit<BoxProps, 'resize' | 'onChange'> & {
|
||||
onOpenModal?: () => void;
|
||||
variables?: EditorVariablePickerType[];
|
||||
defaultHeight?: number;
|
||||
language?: string;
|
||||
};
|
||||
|
||||
const options = {
|
||||
@@ -53,11 +54,14 @@ const MyEditor = ({
|
||||
variables = [],
|
||||
defaultHeight = 200,
|
||||
onOpenModal,
|
||||
language = 'typescript',
|
||||
...props
|
||||
}: Props) => {
|
||||
const [height, setHeight] = useState(defaultHeight);
|
||||
const initialY = useRef(0);
|
||||
|
||||
const registerPythonCompletion = usePythonCompletion();
|
||||
|
||||
const handleMouseDown = useCallback((e: React.MouseEvent) => {
|
||||
initialY.current = e.clientY;
|
||||
|
||||
@@ -76,7 +80,16 @@ const MyEditor = ({
|
||||
document.addEventListener('mouseup', handleMouseUp);
|
||||
}, []);
|
||||
|
||||
const beforeMount = useCallback((monaco: Monaco) => {
|
||||
const editorRef = useRef<any>(null);
|
||||
const monacoRef = useRef<Monaco | null>(null);
|
||||
|
||||
const handleEditorDidMount = useCallback((editor: any, monaco: Monaco) => {
|
||||
editorRef.current = editor;
|
||||
monacoRef.current = monaco;
|
||||
}, []);
|
||||
|
||||
const beforeMount = useCallback(
|
||||
(monaco: Monaco) => {
|
||||
monaco.languages.json.jsonDefaults.setDiagnosticsOptions({
|
||||
validate: false,
|
||||
allowComments: false,
|
||||
@@ -103,7 +116,10 @@ const MyEditor = ({
|
||||
'editorIndentGuide.background': '#eee'
|
||||
}
|
||||
});
|
||||
}, []);
|
||||
registerPythonCompletion(monaco);
|
||||
},
|
||||
[registerPythonCompletion]
|
||||
);
|
||||
|
||||
return (
|
||||
<Box
|
||||
@@ -118,7 +134,7 @@ const MyEditor = ({
|
||||
>
|
||||
<Editor
|
||||
height={'100%'}
|
||||
defaultLanguage="typescript"
|
||||
language={language}
|
||||
options={options as any}
|
||||
theme="JSONEditorTheme"
|
||||
beforeMount={beforeMount}
|
||||
@@ -127,6 +143,7 @@ const MyEditor = ({
|
||||
onChange={(e) => {
|
||||
onChange?.(e || '');
|
||||
}}
|
||||
onMount={handleEditorDidMount}
|
||||
/>
|
||||
{resize && (
|
||||
<Box
|
||||
|
@@ -4,15 +4,31 @@ import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react'
|
||||
import MyModal from '../../MyModal';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
|
||||
type Props = Omit<EditorProps, 'resize'> & {};
|
||||
type Props = Omit<EditorProps, 'resize'> & { language?: string };
|
||||
function getLanguage(language: string | undefined): string {
|
||||
let fullName: string;
|
||||
switch (language) {
|
||||
case 'py':
|
||||
fullName = 'python';
|
||||
break;
|
||||
case 'js':
|
||||
fullName = 'typescript';
|
||||
break;
|
||||
default:
|
||||
fullName = `typescript`;
|
||||
break;
|
||||
}
|
||||
return fullName;
|
||||
}
|
||||
|
||||
const CodeEditor = (props: Props) => {
|
||||
const { t } = useTranslation();
|
||||
const { isOpen, onOpen, onClose } = useDisclosure();
|
||||
|
||||
const { language, ...otherProps } = props;
|
||||
const fullName = getLanguage(language);
|
||||
return (
|
||||
<>
|
||||
<MyEditor {...props} resize onOpenModal={onOpen} />
|
||||
<MyEditor {...props} resize onOpenModal={onOpen} language={fullName} />
|
||||
<MyModal
|
||||
isOpen={isOpen}
|
||||
onClose={onClose}
|
||||
@@ -23,7 +39,7 @@ const CodeEditor = (props: Props) => {
|
||||
isCentered
|
||||
>
|
||||
<ModalBody flex={'1 0 0'} overflow={'auto'}>
|
||||
<MyEditor {...props} bg={'myGray.50'} height={'100%'} />
|
||||
<MyEditor {...props} bg={'myGray.50'} height={'100%'} language={fullName} />
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button mr={2} onClick={onClose} px={6}>
|
||||
|
@@ -0,0 +1,83 @@
|
||||
import { Monaco } from '@monaco-editor/react';
|
||||
import { useCallback } from 'react';
|
||||
let monacoInstance: Monaco | null = null;
|
||||
const usePythonCompletion = () => {
|
||||
return useCallback((monaco: Monaco) => {
|
||||
if (monacoInstance === monaco) return;
|
||||
monacoInstance = monaco;
|
||||
|
||||
monaco.languages.registerCompletionItemProvider('python', {
|
||||
provideCompletionItems: (model, position) => {
|
||||
const wordInfo = model.getWordUntilPosition(position);
|
||||
const currentWordPrefix = wordInfo.word;
|
||||
|
||||
const lineContent = model.getLineContent(position.lineNumber);
|
||||
|
||||
const range = {
|
||||
startLineNumber: position.lineNumber,
|
||||
endLineNumber: position.lineNumber,
|
||||
startColumn: wordInfo.startColumn,
|
||||
endColumn: wordInfo.endColumn
|
||||
};
|
||||
|
||||
const baseSuggestions = [
|
||||
{
|
||||
label: 'len',
|
||||
kind: monaco.languages.CompletionItemKind.Function,
|
||||
insertText: 'len()',
|
||||
documentation: 'get length of object',
|
||||
range,
|
||||
sortText: 'a'
|
||||
}
|
||||
];
|
||||
|
||||
const filtered = baseSuggestions.filter((item) =>
|
||||
item.label.toLowerCase().startsWith(currentWordPrefix.toLowerCase())
|
||||
);
|
||||
|
||||
if (lineContent.startsWith('import')) {
|
||||
const importLength = 'import'.length;
|
||||
const afterImport = lineContent.slice(importLength);
|
||||
const spaceMatch = afterImport.match(/^\s*/);
|
||||
const spaceLength = spaceMatch ? spaceMatch[0].length : 0;
|
||||
|
||||
const startReplaceCol = importLength + spaceLength + 1;
|
||||
const currentCol = position.column;
|
||||
|
||||
const replaceRange = new monaco.Range(
|
||||
position.lineNumber,
|
||||
startReplaceCol,
|
||||
position.lineNumber,
|
||||
currentCol
|
||||
);
|
||||
|
||||
const needsSpace = spaceLength === 0;
|
||||
return {
|
||||
suggestions: [
|
||||
{
|
||||
label: 'numpy',
|
||||
kind: monaco.languages.CompletionItemKind.Module,
|
||||
insertText: `${needsSpace ? ' ' : ''}numpy as np`,
|
||||
documentation: 'numerical computing library',
|
||||
range: replaceRange,
|
||||
sortText: 'a'
|
||||
},
|
||||
{
|
||||
label: 'pandas',
|
||||
kind: monaco.languages.CompletionItemKind.Module,
|
||||
insertText: `${needsSpace ? ' ' : ''}pandas as pd`,
|
||||
documentation: 'data analysis library',
|
||||
range: replaceRange
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
return { suggestions: filtered };
|
||||
},
|
||||
triggerCharacters: ['.', '_']
|
||||
});
|
||||
}, []);
|
||||
};
|
||||
|
||||
export default usePythonCompletion;
|
@@ -20,6 +20,7 @@
|
||||
"classification_result": "Classification Result",
|
||||
"code.Reset template": "Reset Template",
|
||||
"code.Reset template confirm": "Confirm reset code template? This will reset all inputs and outputs to template values. Please save your current code.",
|
||||
"code.Switch language confirm": "Switching the language will reset the code, will it continue?",
|
||||
"code_execution": "Code Sandbox",
|
||||
"collection_metadata_filter": "Collection Metadata Filter",
|
||||
"complete_extraction_result": "Complete Extraction Result",
|
||||
@@ -153,6 +154,7 @@
|
||||
"select_another_application_to_call": "You can choose another application to call",
|
||||
"special_array_format": "Special array format, returns an empty array when the search result is empty.",
|
||||
"start_with": "Starts With",
|
||||
"support_code_language": "Support import list: pandas,numpy",
|
||||
"target_fields_description": "A target field consists of 'description' and 'key'. Multiple target fields can be extracted.",
|
||||
"template.ai_chat": "AI Chat",
|
||||
"template.ai_chat_intro": "AI Large Model Chat",
|
||||
|
@@ -20,6 +20,7 @@
|
||||
"classification_result": "分类结果",
|
||||
"code.Reset template": "还原模板",
|
||||
"code.Reset template confirm": "确认还原代码模板?将会重置所有输入和输出至模板值,请注意保存当前代码。",
|
||||
"code.Switch language confirm": "切换语言将重置代码,是否继续?",
|
||||
"code_execution": "代码运行",
|
||||
"collection_metadata_filter": "集合元数据过滤",
|
||||
"complete_extraction_result": "完整提取结果",
|
||||
@@ -153,6 +154,7 @@
|
||||
"select_another_application_to_call": "可以选择一个其他应用进行调用",
|
||||
"special_array_format": "特殊数组格式,搜索结果为空时,返回空数组。",
|
||||
"start_with": "开始为",
|
||||
"support_code_language": "支持import列表:pandas,numpy",
|
||||
"target_fields_description": "由 '描述' 和 'key' 组成一个目标字段,可提取多个目标字段",
|
||||
"template.ai_chat": "AI 对话",
|
||||
"template.ai_chat_intro": "AI 大模型对话",
|
||||
|
@@ -20,6 +20,7 @@
|
||||
"classification_result": "分類結果",
|
||||
"code.Reset template": "重設範本",
|
||||
"code.Reset template confirm": "確定要重設程式碼範本嗎?這將會把所有輸入和輸出重設為範本值。請儲存您目前的程式碼。",
|
||||
"code.Switch language confirm": "切換語言將重置代碼,是否繼續?",
|
||||
"code_execution": "程式碼執行",
|
||||
"collection_metadata_filter": "資料集詮釋資料篩選器",
|
||||
"complete_extraction_result": "完整擷取結果",
|
||||
@@ -153,6 +154,7 @@
|
||||
"select_another_application_to_call": "可以選擇另一個應用程式來呼叫",
|
||||
"special_array_format": "特殊陣列格式,搜尋結果為空時,回傳空陣列。",
|
||||
"start_with": "開頭為",
|
||||
"support_code_language": "支持import列表:pandas,numpy",
|
||||
"target_fields_description": "由「描述」和「鍵值」組成一個目標欄位,可以擷取多個目標欄位",
|
||||
"template.ai_chat": "AI 對話",
|
||||
"template.ai_chat_intro": "AI 大型語言模型對話",
|
||||
|
80
plugins/webcrawler/deploy/docker-compose.yaml
Normal file
80
plugins/webcrawler/deploy/docker-compose.yaml
Normal file
@@ -0,0 +1,80 @@
|
||||
name: spider
|
||||
version: "2.2"
|
||||
|
||||
services:
|
||||
searxng:
|
||||
container_name: searxng
|
||||
image: docker.io/searxng/searxng:latest
|
||||
platform: linux/amd64
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- spider_net
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./searxng:/etc/searxng:rw
|
||||
environment:
|
||||
- SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
|
||||
- UWSGI_WORKERS=4 # UWSGI 工作进程数
|
||||
- UWSGI_THREADS=4 # UWSGI 线程数
|
||||
cap_drop:
|
||||
- ALL
|
||||
|
||||
mongodb:
|
||||
container_name: mongodb
|
||||
image: mongo:4.4
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- spider_net
|
||||
ports:
|
||||
- "27017:27017"
|
||||
volumes:
|
||||
- mongo-data:/data/db
|
||||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: root # MongoDB 根用户名
|
||||
MONGO_INITDB_ROOT_PASSWORD: example # MongoDB 根用户密码
|
||||
|
||||
nodeapp:
|
||||
container_name: main
|
||||
platform: linux/amd64
|
||||
#build:
|
||||
# context: .
|
||||
image: gggaaallleee/webcrawler-test-new:latest
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks:
|
||||
- spider_net
|
||||
depends_on:
|
||||
- mongodb
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "1m"
|
||||
max-file: "1"
|
||||
volumes:
|
||||
- /dev/shm:/dev/shm
|
||||
environment:
|
||||
- ACCESS_TOKEN=webcrawler # 访问令牌
|
||||
- DETECT_WEBSITE=zhuanlan.zhihu.com # 无法处理跳过的网站
|
||||
- STRATEGIES=[{"waitUntil":"networkidle0","timeout":5000},{"waitUntil":"networkidle2","timeout":10000},{"waitUntil":"load","timeout":15000}] # 页面加载策略
|
||||
- PORT=3000
|
||||
- MAX_CONCURRENCY=10 # 最大并发数
|
||||
- NODE_ENV=development
|
||||
- ENGINE_BAIDUURL=https://www.baidu.com/s # 百度搜索引擎 URL
|
||||
- ENGINE_SEARCHXNGURL=http://searxng:8080/search # Searxng 搜索引擎 URL
|
||||
- MONGODB_URI=mongodb://root:example@mongodb:27017 # MongoDB 连接 URI
|
||||
- BLACKLIST=[".gov.cn",".edu.cn"] # 受保护域名
|
||||
- STD_TTL=3600 # 标准 TTL(秒)
|
||||
- EXPIRE_AFTER_SECONDS=9000 # 过期时间(秒)
|
||||
#- VALIDATE_PROXY=[{"ip":"","port":},{"ip":"","port":}] #代理池
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
cpus: '2.0'
|
||||
|
||||
networks:
|
||||
spider_net:
|
||||
|
||||
volumes:
|
||||
mongo-data:
|
6
plugins/webcrawler/deploy/searxng/limiter.toml
Normal file
6
plugins/webcrawler/deploy/searxng/limiter.toml
Normal file
@@ -0,0 +1,6 @@
|
||||
# This configuration file updates the default configuration file
|
||||
# See https://github.com/searxng/searxng/blob/master/searx/limiter.toml
|
||||
|
||||
[botdetection.ip_limit]
|
||||
# activate link_token method in the ip_limit method
|
||||
link_token = true
|
122
plugins/webcrawler/deploy/searxng/settings.yml
Normal file
122
plugins/webcrawler/deploy/searxng/settings.yml
Normal file
@@ -0,0 +1,122 @@
|
||||
general:
|
||||
debug: false
|
||||
instance_name: "searxng"
|
||||
privacypolicy_url: false
|
||||
donation_url: false
|
||||
contact_url: false
|
||||
enable_metrics: true
|
||||
open_metrics: ''
|
||||
|
||||
brand:
|
||||
new_issue_url: https://github.com/searxng/searxng/issues/new
|
||||
docs_url: https://docs.searxng.org/
|
||||
public_instances: https://searx.space
|
||||
wiki_url: https://github.com/searxng/searxng/wiki
|
||||
issue_url: https://github.com/searxng/searxng/issues
|
||||
|
||||
search:
|
||||
safe_search: 0
|
||||
autocomplete: ""
|
||||
autocomplete_min: 4
|
||||
default_lang: "auto"
|
||||
ban_time_on_fail: 5
|
||||
max_ban_time_on_fail: 120
|
||||
formats:
|
||||
- html
|
||||
|
||||
server:
|
||||
port: 8080
|
||||
bind_address: "0.0.0.0"
|
||||
base_url: false
|
||||
limiter: false
|
||||
public_instance: false
|
||||
secret_key: "example"
|
||||
image_proxy: false
|
||||
http_protocol_version: "1.0"
|
||||
method: "POST"
|
||||
default_http_headers:
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Download-Options: noopen
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
Referrer-Policy: no-referrer
|
||||
|
||||
redis:
|
||||
url: false
|
||||
|
||||
ui:
|
||||
static_path: ""
|
||||
static_use_hash: false
|
||||
templates_path: ""
|
||||
default_theme: simple
|
||||
default_locale: ""
|
||||
query_in_title: false
|
||||
infinite_scroll: false
|
||||
center_alignment: false
|
||||
theme_args:
|
||||
simple_style: auto
|
||||
# 启用 cn 分类
|
||||
enabled_categories: [cn,en, general, images,en]
|
||||
# 或者定义分类显示顺序
|
||||
categories_order: [cn, en,general, images]
|
||||
|
||||
outgoing:
|
||||
request_timeout: 30.0
|
||||
max_request_timeout: 40.0
|
||||
pool_connections: 200
|
||||
pool_maxsize: 50
|
||||
enable_http2: false
|
||||
retries: 5
|
||||
|
||||
engines:
|
||||
- name: bing
|
||||
engine: bing
|
||||
disabled: false
|
||||
categories: cn
|
||||
#- name: bilibili
|
||||
# engine: bilibili
|
||||
# shortcut: bil
|
||||
# disabled: false
|
||||
# categories: cn
|
||||
- name : baidu
|
||||
engine : json_engine
|
||||
paging : True
|
||||
first_page_num : 0
|
||||
search_url : https://www.baidu.com/s?tn=json&wd={query}&pn={pageno}&rn=50
|
||||
url_query : url
|
||||
title_query : title
|
||||
content_query : abs
|
||||
categories : cn
|
||||
- name : 360search
|
||||
engine: 360search
|
||||
disabled: false
|
||||
categories: cn
|
||||
- name : sogou
|
||||
disabled: false
|
||||
categories: cn
|
||||
|
||||
- name: google
|
||||
disabled: false
|
||||
categories: en
|
||||
- name: yahoo
|
||||
disabled: false
|
||||
categories: en
|
||||
- name: duckduckgo
|
||||
disabled: false
|
||||
categories: en
|
||||
|
||||
|
||||
|
||||
search:
|
||||
formats:
|
||||
- html
|
||||
- json
|
||||
doi_resolvers:
|
||||
oadoi.org: 'https://oadoi.org/'
|
||||
doi.org: 'https://doi.org/'
|
||||
doai.io: 'https://dissem.in/'
|
||||
sci-hub.se: 'https://sci-hub.se/'
|
||||
sci-hub.st: 'https://sci-hub.st/'
|
||||
sci-hub.ru: 'https://sci-hub.ru/'
|
||||
|
||||
default_doi_resolver: 'oadoi.org'
|
||||
|
@@ -2,5 +2,15 @@
|
||||
|
||||
该目录为 FastGPT 主项目。
|
||||
|
||||
- app 前端项目,用于展示和使用 FastGPT。
|
||||
- sandbox 沙盒项目,用于测试和开发。
|
||||
- app fastgpt 核心应用。
|
||||
- sandbox 沙盒项目,用于运行工作流里的代码执行 (需求python环境为python:3.11,额外安装的包请于requirements.txt填写,同时注意个别包可能额外安装库(如pandas需要安装libffi))。
|
||||
- 新加入python包遇见超时或者权限拦截的问题(确定不是自己的语法问题),请进入docker容器内部执行以下指令:
|
||||
|
||||
```shell
|
||||
docker exec -it 《替换成容器名》 /bin/bash
|
||||
chmod -x testSystemCall.sh
|
||||
bash ./testSystemCall.sh
|
||||
```
|
||||
|
||||
然后将新的数组替换src下sandbox的constants.py中的SYSTEM_CALLS数组即可
|
||||
|
||||
|
@@ -15,38 +15,89 @@ import RenderOutput from './render/RenderOutput';
|
||||
import CodeEditor from '@fastgpt/web/components/common/Textarea/CodeEditor';
|
||||
import { Box, Flex } from '@chakra-ui/react';
|
||||
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
|
||||
import { JS_TEMPLATE } from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
|
||||
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
|
||||
import {
|
||||
JS_TEMPLATE,
|
||||
PY_TEMPLATE,
|
||||
SandboxCodeTypeEnum,
|
||||
SNADBOX_CODE_TEMPLATE
|
||||
} from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
|
||||
import MySelect from '@fastgpt/web/components/common/MySelect';
|
||||
|
||||
const NodeCode = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
|
||||
const { t } = useTranslation();
|
||||
const { nodeId, inputs, outputs } = data;
|
||||
|
||||
const codeType = inputs.find(
|
||||
(item) => item.key === NodeInputKeyEnum.codeType
|
||||
) as FlowNodeInputItemType;
|
||||
|
||||
const splitToolInputs = useContextSelector(WorkflowContext, (ctx) => ctx.splitToolInputs);
|
||||
const onChangeNode = useContextSelector(WorkflowContext, (ctx) => ctx.onChangeNode);
|
||||
|
||||
const { ConfirmModal, openConfirm } = useConfirm({
|
||||
// 重置模板确认
|
||||
const { ConfirmModal: ResetTemplateConfirm, openConfirm: openResetTemplateConfirm } = useConfirm({
|
||||
content: t('workflow:code.Reset template confirm')
|
||||
});
|
||||
|
||||
// 切换语言确认
|
||||
const { ConfirmModal: SwitchLangConfirm, openConfirm: openSwitchLangConfirm } = useConfirm({
|
||||
content: t('workflow:code.Switch language confirm')
|
||||
});
|
||||
|
||||
const CustomComponent = useMemo(() => {
|
||||
return {
|
||||
[NodeInputKeyEnum.code]: (item: FlowNodeInputItemType) => {
|
||||
return (
|
||||
<Box mt={-3}>
|
||||
<Flex mb={2} alignItems={'flex-end'}>
|
||||
<Box flex={'1'}>{'Javascript ' + t('workflow:Code')}</Box>
|
||||
<Box
|
||||
cursor={'pointer'}
|
||||
color={'primary.500'}
|
||||
fontSize={'xs'}
|
||||
onClick={openConfirm(() => {
|
||||
<Box mt={-4}>
|
||||
<Flex mb={2} alignItems={'center'} className="nodrag">
|
||||
<MySelect<SandboxCodeTypeEnum>
|
||||
fontSize="xs"
|
||||
size="sm"
|
||||
list={[
|
||||
{ label: 'JavaScript', value: SandboxCodeTypeEnum.js },
|
||||
{ label: 'Python 3', value: SandboxCodeTypeEnum.py }
|
||||
]}
|
||||
value={codeType?.value}
|
||||
onChange={(newLang) => {
|
||||
console.log(newLang);
|
||||
openSwitchLangConfirm(() => {
|
||||
onChangeNode({
|
||||
nodeId,
|
||||
type: 'updateInput',
|
||||
key: NodeInputKeyEnum.codeType,
|
||||
value: { ...codeType, value: newLang }
|
||||
});
|
||||
|
||||
onChangeNode({
|
||||
nodeId,
|
||||
type: 'updateInput',
|
||||
key: item.key,
|
||||
value: {
|
||||
...item,
|
||||
value: JS_TEMPLATE
|
||||
value: SNADBOX_CODE_TEMPLATE[newLang]
|
||||
}
|
||||
});
|
||||
})();
|
||||
}}
|
||||
/>
|
||||
{codeType.value === 'py' && (
|
||||
<QuestionTip ml={2} label={t('workflow:support_code_language')} />
|
||||
)}
|
||||
<Box
|
||||
cursor={'pointer'}
|
||||
color={'primary.500'}
|
||||
fontSize={'xs'}
|
||||
ml="auto"
|
||||
mr={2}
|
||||
onClick={openResetTemplateConfirm(() => {
|
||||
onChangeNode({
|
||||
nodeId,
|
||||
type: 'updateInput',
|
||||
key: item.key,
|
||||
value: {
|
||||
...item,
|
||||
value: codeType.value === 'js' ? JS_TEMPLATE : PY_TEMPLATE
|
||||
}
|
||||
});
|
||||
})}
|
||||
@@ -63,29 +114,25 @@ const NodeCode = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
|
||||
nodeId,
|
||||
type: 'updateInput',
|
||||
key: item.key,
|
||||
value: {
|
||||
...item,
|
||||
value: e
|
||||
}
|
||||
value: { ...item, value: e }
|
||||
});
|
||||
}}
|
||||
language={codeType.value}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
};
|
||||
}, [nodeId, onChangeNode, openConfirm, t]);
|
||||
}, [codeType, nodeId, onChangeNode, openResetTemplateConfirm, openSwitchLangConfirm, t]);
|
||||
|
||||
const { isTool, commonInputs } = splitToolInputs(inputs, nodeId);
|
||||
|
||||
return (
|
||||
<NodeCard minW={'400px'} selected={selected} {...data}>
|
||||
{isTool && (
|
||||
<>
|
||||
<Container>
|
||||
<RenderToolInput nodeId={nodeId} inputs={inputs} />
|
||||
</Container>
|
||||
</>
|
||||
)}
|
||||
<Container>
|
||||
<IOTitle text={t('common:common.Input')} mb={-1} />
|
||||
@@ -99,7 +146,8 @@ const NodeCode = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
|
||||
<IOTitle text={t('common:common.Output')} />
|
||||
<RenderOutput nodeId={nodeId} flowOutputList={outputs} />
|
||||
</Container>
|
||||
<ConfirmModal />
|
||||
<ResetTemplateConfirm />
|
||||
<SwitchLangConfirm />
|
||||
</NodeCard>
|
||||
);
|
||||
};
|
||||
|
@@ -1,8 +1,22 @@
|
||||
# --------- install dependence -----------
|
||||
FROM python:3.11-alpine AS python_base
|
||||
|
||||
ENV VERSION_RELEASE = Alpine3.11
|
||||
# 安装make和g++
|
||||
RUN apk add --no-cache make g++
|
||||
RUN apk add --no-cache make g++ tar wget gperf automake libtool linux-headers
|
||||
|
||||
WORKDIR /app
|
||||
COPY projects/sandbox/requirements.txt /app/requirements.txt
|
||||
RUN wget https://github.com/seccomp/libseccomp/releases/download/v2.5.5/libseccomp-2.5.5.tar.gz && \
|
||||
tar -zxvf libseccomp-2.5.5.tar.gz && \
|
||||
cd libseccomp-2.5.5 && \
|
||||
./configure --prefix=/usr && \
|
||||
make && \
|
||||
make install && \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple Cython && \
|
||||
pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple -r /app/requirements.txt && \
|
||||
cd src/python && \
|
||||
python setup.py install
|
||||
|
||||
|
||||
FROM node:20.14.0-alpine AS install
|
||||
|
||||
@@ -10,7 +24,7 @@ WORKDIR /app
|
||||
|
||||
ARG proxy
|
||||
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
|
||||
RUN apk add --no-cache make g++
|
||||
RUN apk add --no-cache make g++ python3
|
||||
|
||||
# copy py3.11
|
||||
COPY --from=python_base /usr/local /usr/local
|
||||
@@ -42,9 +56,12 @@ RUN pnpm --filter=sandbox build
|
||||
FROM node:20.14.0-alpine AS runner
|
||||
WORKDIR /app
|
||||
|
||||
RUN apk add --no-cache libffi libffi-dev strace bash
|
||||
COPY --from=python_base /usr/local /usr/local
|
||||
COPY --from=builder /app/node_modules /app/node_modules
|
||||
COPY --from=builder /app/projects/sandbox /app/projects/sandbox
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PATH="/usr/local/bin:${PATH}"
|
||||
|
||||
CMD ["node", "--no-node-snapshot", "projects/sandbox/dist/main.js"]
|
||||
|
2
projects/sandbox/requirements.txt
Normal file
2
projects/sandbox/requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
numpy
|
||||
pandas
|
130
projects/sandbox/src/sandbox/constants.ts
Normal file
130
projects/sandbox/src/sandbox/constants.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
export const pythonScript = `
|
||||
import subprocess
|
||||
import json
|
||||
import ast
|
||||
import base64
|
||||
|
||||
def extract_imports(code):
|
||||
tree = ast.parse(code)
|
||||
imports = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
imports.append(f"import {alias.name}")
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
module = node.module
|
||||
for alias in node.names:
|
||||
imports.append(f"from {module} import {alias.name}")
|
||||
return imports
|
||||
seccomp_prefix = """
|
||||
from seccomp import *
|
||||
import sys
|
||||
allowed_syscalls = [
|
||||
"syscall.SYS_ARCH_PRCTL", "syscall.SYS_BRK", "syscall.SYS_CLONE",
|
||||
"syscall.SYS_CLOSE", "syscall.SYS_EPOLL_CREATE1", "syscall.SYS_EXECVE",
|
||||
"syscall.SYS_EXIT", "syscall.SYS_EXIT_GROUP", "syscall.SYS_FCNTL",
|
||||
"syscall.SYS_FSTAT", "syscall.SYS_FUTEX", "syscall.SYS_GETDENTS64",
|
||||
"syscall.SYS_GETEGID", "syscall.SYS_GETEUID", "syscall.SYS_GETGID",
|
||||
"syscall.SYS_GETRANDOM", "syscall.SYS_GETTID", "syscall.SYS_GETUID",
|
||||
"syscall.SYS_IOCTL", "syscall.SYS_LSEEK", "syscall.SYS_LSTAT",
|
||||
"syscall.SYS_MBIND", "syscall.SYS_MEMBARRIER", "syscall.SYS_MMAP",
|
||||
"syscall.SYS_MPROTECT", "syscall.SYS_MUNMAP", "syscall.SYS_OPEN",
|
||||
"syscall.SYS_PREAD64", "syscall.SYS_READ", "syscall.SYS_READLINK",
|
||||
"syscall.SYS_READV", "syscall.SYS_RT_SIGACTION", "syscall.SYS_RT_SIGPROCMASK",
|
||||
"syscall.SYS_SCHED_GETAFFINITY", "syscall.SYS_SET_TID_ADDRESS",
|
||||
"syscall.SYS_STAT", "syscall.SYS_UNAME",
|
||||
"syscall.SYS_MREMAP", "syscall.SYS_RT_SIGRETURN", "syscall.SYS_SETUID",
|
||||
"syscall.SYS_SETGID", "syscall.SYS_GETPID", "syscall.SYS_GETPPID",
|
||||
"syscall.SYS_TGKILL", "syscall.SYS_SCHED_YIELD", "syscall.SYS_SET_ROBUST_LIST",
|
||||
"syscall.SYS_GET_ROBUST_LIST", "syscall.SYS_RSEQ", "syscall.SYS_CLOCK_GETTIME",
|
||||
"syscall.SYS_GETTIMEOFDAY", "syscall.SYS_NANOSLEEP", "syscall.SYS_EPOLL_CTL",
|
||||
"syscall.SYS_CLOCK_NANOSLEEP", "syscall.SYS_PSELECT6", "syscall.SYS_TIME",
|
||||
"syscall.SYS_SIGALTSTACK", "syscall.SYS_MKDIRAT", "syscall.SYS_MKDIR"
|
||||
]
|
||||
allowed_syscalls_tmp = allowed_syscalls
|
||||
L = []
|
||||
for item in allowed_syscalls_tmp:
|
||||
item = item.strip()
|
||||
parts = item.split(".")[1][4:].lower()
|
||||
L.append(parts)
|
||||
f = SyscallFilter(defaction=KILL)
|
||||
for item in L:
|
||||
f.add_rule(ALLOW, item)
|
||||
f.add_rule(ALLOW, "write", Arg(0, EQ, sys.stdout.fileno()))
|
||||
f.add_rule(ALLOW, "write", Arg(0, EQ, sys.stderr.fileno()))
|
||||
f.add_rule(ALLOW, 307)
|
||||
f.add_rule(ALLOW, 318)
|
||||
f.add_rule(ALLOW, 334)
|
||||
f.load()
|
||||
"""
|
||||
|
||||
def remove_print_statements(code):
|
||||
class PrintRemover(ast.NodeTransformer):
|
||||
def visit_Expr(self, node):
|
||||
if (
|
||||
isinstance(node.value, ast.Call)
|
||||
and isinstance(node.value.func, ast.Name)
|
||||
and node.value.func.id == "print"
|
||||
):
|
||||
return None
|
||||
return node
|
||||
|
||||
tree = ast.parse(code)
|
||||
modified_tree = PrintRemover().visit(tree)
|
||||
ast.fix_missing_locations(modified_tree)
|
||||
return ast.unparse(modified_tree)
|
||||
|
||||
def detect_dangerous_imports(code):
|
||||
dangerous_modules = ["os", "sys", "subprocess", "shutil", "socket", "ctypes", "multiprocessing", "threading", "pickle"]
|
||||
tree = ast.parse(code)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name in dangerous_modules:
|
||||
return alias.name
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module in dangerous_modules:
|
||||
return node.module
|
||||
return None
|
||||
|
||||
def run_pythonCode(data:dict):
|
||||
if not data or "code" not in data or "variables" not in data:
|
||||
return {"error": "Invalid request format"}
|
||||
code = data["code"]
|
||||
code = remove_print_statements(code)
|
||||
dangerous_import = detect_dangerous_imports(code)
|
||||
if dangerous_import:
|
||||
return {"error": f"Importing {dangerous_import} is not allowed."}
|
||||
variables = data["variables"]
|
||||
imports = "\\n".join(extract_imports(code))
|
||||
var_def = ""
|
||||
output_code = "res = main("
|
||||
for k, v in variables.items():
|
||||
if isinstance(v, str):
|
||||
one_var = k + " = \\"" + v + "\\"\\n"
|
||||
else:
|
||||
one_var = k + " = " + str(v) + "\\n"
|
||||
var_def = var_def + one_var
|
||||
output_code = output_code + k + ", "
|
||||
if output_code[-1] == "(":
|
||||
output_code = output_code + ")\\n"
|
||||
else:
|
||||
output_code = output_code[:-2] + ")\\n"
|
||||
output_code = output_code + "print(res)"
|
||||
code = imports + "\\n" + seccomp_prefix + "\\n" + var_def + "\\n" + code + "\\n" + output_code
|
||||
try:
|
||||
result = subprocess.run(["python3", "-c", code], capture_output=True, text=True, timeout=10)
|
||||
if result.returncode == -31:
|
||||
return {"error": "Dangerous behavior detected."}
|
||||
if result.stderr != "":
|
||||
return {"error": result.stderr}
|
||||
|
||||
out = ast.literal_eval(result.stdout.strip())
|
||||
return out
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"error": "Timeout error"}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
`;
|
@@ -1,6 +1,6 @@
|
||||
import { Controller, Post, Body, HttpCode } from '@nestjs/common';
|
||||
import { RunCodeDto } from './dto/create-sandbox.dto';
|
||||
import { runSandbox } from './utils';
|
||||
import { runJsSandbox, runPythonSandbox } from './utils';
|
||||
|
||||
@Controller('sandbox')
|
||||
export class SandboxController {
|
||||
@@ -9,6 +9,12 @@ export class SandboxController {
|
||||
@Post('/js')
|
||||
@HttpCode(200)
|
||||
runJs(@Body() codeProps: RunCodeDto) {
|
||||
return runSandbox(codeProps);
|
||||
return runJsSandbox(codeProps);
|
||||
}
|
||||
|
||||
@Post('/python')
|
||||
@HttpCode(200)
|
||||
runPython(@Body() codeProps: RunCodeDto) {
|
||||
return runPythonSandbox(codeProps);
|
||||
}
|
||||
}
|
||||
|
@@ -6,8 +6,14 @@ import { timeDelay } from './jsFn/delay';
|
||||
import { strToBase64 } from './jsFn/str2Base64';
|
||||
import { createHmac } from './jsFn/crypto';
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { pythonScript } from './constants';
|
||||
const CustomLogStr = 'CUSTOM_LOG';
|
||||
|
||||
export const runJsSandbox = async ({
|
||||
code,
|
||||
variables = {}
|
||||
}: RunCodeDto): Promise<RunCodeResponse> => {
|
||||
/*
|
||||
Rewrite code to add custom functions: Promise function; Log.
|
||||
*/
|
||||
@@ -38,7 +44,6 @@ function getFnCode(code: string) {
|
||||
`;
|
||||
return runCode;
|
||||
}
|
||||
|
||||
// Register global function
|
||||
function registerSystemFn(jail: IsolatedVM.Reference<Record<string | number | symbol, any>>) {
|
||||
return Promise.all([
|
||||
@@ -49,10 +54,6 @@ function registerSystemFn(jail: IsolatedVM.Reference<Record<string | number | sy
|
||||
]);
|
||||
}
|
||||
|
||||
export const runSandbox = async ({
|
||||
code,
|
||||
variables = {}
|
||||
}: RunCodeDto): Promise<RunCodeResponse> => {
|
||||
const logData = [];
|
||||
|
||||
const isolate = new Isolate({ memoryLimit: 32 });
|
||||
@@ -106,3 +107,50 @@ export const runSandbox = async ({
|
||||
return Promise.reject(err);
|
||||
}
|
||||
};
|
||||
|
||||
export const runPythonSandbox = async ({
|
||||
code,
|
||||
variables = {}
|
||||
}: RunCodeDto): Promise<RunCodeResponse> => {
|
||||
const mainCallCode = `
|
||||
data = ${JSON.stringify({ code, variables })}
|
||||
res = run_pythonCode(data)
|
||||
print(json.dumps(res))
|
||||
`;
|
||||
|
||||
const fullCode = [pythonScript, mainCallCode].filter(Boolean).join('\n');
|
||||
|
||||
const pythonProcess = spawn('python3', ['-u', '-c', fullCode]);
|
||||
|
||||
const stdoutChunks: string[] = [];
|
||||
const stderrChunks: string[] = [];
|
||||
|
||||
pythonProcess.stdout.on('data', (data) => stdoutChunks.push(data.toString()));
|
||||
pythonProcess.stderr.on('data', (data) => stderrChunks.push(data.toString()));
|
||||
|
||||
const stdoutPromise = new Promise<string>((resolve) => {
|
||||
pythonProcess.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
resolve(JSON.stringify({ error: stderrChunks.join('') }));
|
||||
} else {
|
||||
resolve(stdoutChunks.join(''));
|
||||
}
|
||||
});
|
||||
});
|
||||
const stdout = await stdoutPromise;
|
||||
|
||||
try {
|
||||
const parsedOutput = JSON.parse(stdout);
|
||||
if (parsedOutput.error) {
|
||||
return Promise.reject(parsedOutput.error || 'Unknown error');
|
||||
}
|
||||
return { codeReturn: parsedOutput, log: '' };
|
||||
} catch (err) {
|
||||
if (stdout.includes('malformed node or string on line 1')) {
|
||||
return Promise.reject(`The result should be a parsable variable, such as a list. ${stdout}`);
|
||||
} else if (stdout.includes('Unexpected end of JSON input')) {
|
||||
return Promise.reject(`Not allowed print or ${stdout}`);
|
||||
}
|
||||
return Promise.reject(`Run failed: ${err}`);
|
||||
}
|
||||
};
|
||||
|
41
projects/sandbox/testSystemCall.sh
Normal file
41
projects/sandbox/testSystemCall.sh
Normal file
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
|
||||
temp_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$temp_dir"' EXIT
|
||||
|
||||
syscall_table_file="$temp_dir/syscall_table.txt"
|
||||
code_file="$temp_dir/test_code.py"
|
||||
strace_log="$temp_dir/strace.log"
|
||||
syscalls_file="$temp_dir/syscalls.txt"
|
||||
|
||||
code='
|
||||
import pandas as pd
|
||||
def main():
|
||||
data = {"Name": ["Alice", "Bob"], "Age": [25, 30]}
|
||||
df = pd.DataFrame(data)
|
||||
return {
|
||||
"head": df.head().to_dict()
|
||||
}
|
||||
'
|
||||
|
||||
if ! ausyscall --dump > "$syscall_table_file" 2>/dev/null; then
|
||||
grep -E '^#define __NR_' /usr/include/asm/unistd_64.h | \
|
||||
sed 's/#define __NR_//;s/[ \t]\+/ /g' | \
|
||||
awk '{print $1, $2}' > "$syscall_table_file"
|
||||
fi
|
||||
|
||||
echo "$code" > "$code_file"
|
||||
|
||||
strace -ff -e trace=all -o "$strace_log" python3 "$code_file" >/dev/null 2>&1
|
||||
|
||||
cat "$strace_log"* 2>/dev/null | grep -oE '^[[:alnum:]_]+' | sort -u > "$syscalls_file"
|
||||
|
||||
allowed_syscalls=()
|
||||
while read raw_name; do
|
||||
go_name=$(echo "$raw_name" | tr 'a-z' 'A-Z' | sed 's/-/_/g')
|
||||
allowed_syscalls+=("\"syscall.SYS_${go_name}\"")
|
||||
done < "$syscalls_file"
|
||||
|
||||
echo "allowed_syscalls = ["
|
||||
printf ' %s,\n' "${allowed_syscalls[@]}" | paste -sd ' \n'
|
||||
echo "]"
|
@@ -436,6 +436,28 @@ FastGPT是一款基于大语言模型(LLM)的智能问答系统,专为提
|
||||
expect(chunks).toEqual(mock.result);
|
||||
});
|
||||
|
||||
// 自定义分隔符测试:换行符号
|
||||
it(`Test splitText2Chunks 1`, () => {
|
||||
const mock = {
|
||||
text: `111
|
||||
222
|
||||
|
||||
333`,
|
||||
result: [
|
||||
`111
|
||||
222`,
|
||||
'333'
|
||||
]
|
||||
};
|
||||
|
||||
const { chunks } = splitText2Chunks({ customReg: ['\\n\\n'], text: mock.text, chunkSize: 2000 });
|
||||
fs.writeFileSync(
|
||||
'/Users/yjl/fastgpt-pro/FastGPT/test/cases/function/packages/global/common/string/test.md',
|
||||
chunks.join('------')
|
||||
);
|
||||
expect(chunks).toEqual(mock.result);
|
||||
});
|
||||
|
||||
// 长代码块分割
|
||||
it(`Test splitText2Chunks 7`, () => {
|
||||
const mock = {
|
||||
|
Reference in New Issue
Block a user