This commit is contained in:
Archer
2024-05-28 14:47:10 +08:00
committed by GitHub
parent d9f5f4ede0
commit 9639139b52
58 changed files with 4715 additions and 283 deletions

View File

@@ -0,0 +1,84 @@
name: Build fastgpt-sandbox images and copy image to docker hub
on:
workflow_dispatch:
push:
paths:
- 'projects/sandbox/**'
tags:
- 'v*'
jobs:
build-fastgpt-sandbox-images:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install Dependencies
run: |
sudo apt update && sudo apt install -y nodejs npm
- name: Set up QEMU (optional)
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
driver-opts: network=host
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GH_PAT }}
- name: Set DOCKER_REPO_TAGGED based on branch or tag
run: |
if [[ "${{ github.ref_name }}" == "main" ]]; then
echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:latest" >> $GITHUB_ENV
else
echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:${{ github.ref_name }}" >> $GITHUB_ENV
fi
- name: Build and publish image for main branch or tag push event
env:
DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
run: |
docker buildx build \
-f projects/sandbox/Dockerfile \
--platform linux/amd64,linux/arm64 \
--label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/fastgpt-sandbox" \
--label "org.opencontainers.image.description=fastgpt-sandbox image" \
--push \
--cache-from=type=local,src=/tmp/.buildx-cache \
--cache-to=type=local,dest=/tmp/.buildx-cache \
-t ${DOCKER_REPO_TAGGED} \
.
push-to-ali-hub:
needs: build-fastgpt-sandbox-images
runs-on: ubuntu-20.04
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Login to Ali Hub
uses: docker/login-action@v2
with:
registry: registry.cn-hangzhou.aliyuncs.com
username: ${{ secrets.ALI_HUB_USERNAME }}
password: ${{ secrets.ALI_HUB_PASSWORD }}
- name: Set DOCKER_REPO_TAGGED based on branch or tag
run: |
if [[ "${{ github.ref_name }}" == "main" ]]; then
echo "IMAGE_TAG=latest" >> $GITHUB_ENV
else
echo "IMAGE_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
fi
- name: Pull image from GitHub Container Registry
run: docker pull ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:${{env.IMAGE_TAG}}
- name: Tag image with Docker Hub repository name and version tag
run: docker tag ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:${{env.IMAGE_TAG}} ${{ secrets.ALI_IMAGE_NAME }}/fastgpt-sandbox:${{env.IMAGE_TAG}}
- name: Push image to Docker Hub
run: docker push ${{ secrets.ALI_IMAGE_NAME }}/fastgpt-sandbox:${{env.IMAGE_TAG}}

View File

@@ -49,7 +49,7 @@ jobs:
DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
run: |
docker buildx build \
--build-arg name=app \
-f projects/app/Dockerfile \
--platform linux/amd64,linux/arm64 \
--label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \
--label "org.opencontainers.image.description=fastgpt image" \
@@ -57,7 +57,6 @@ jobs:
--cache-from=type=local,src=/tmp/.buildx-cache \
--cache-to=type=local,dest=/tmp/.buildx-cache \
-t ${DOCKER_REPO_TAGGED} \
-f Dockerfile \
.
push-to-docker-hub:
needs: build-fastgpt-images

View File

@@ -44,15 +44,14 @@ jobs:
DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
run: |
docker buildx build \
--build-arg name=app \
-f projects/app/Dockerfile \
--label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \
--label "org.opencontainers.image.description=fastgpt-pr image" \
--label "org.opencontainers.image.licenses=Apache" \
--label "org.opencontainers.image.description=fastgpt-pr imae" \
--label "org.opencontainers.image.licenses=Apache" \g
--push \
--cache-from=type=local,src=/tmp/.buildx-cache \
--cache-to=type=local,dest=/tmp/.buildx-cache \
-t ${DOCKER_REPO_TAGGED} \
-f Dockerfile \
.
helm-check:

25
Makefile Normal file
View File

@@ -0,0 +1,25 @@
# 定义默认变量
proxy=null
image=null
# 定义目标
.PHONY: build
# 检查 target 是否定义
ifndef name
$(error name is not defined)
endif
filePath=./projects/$(name)/Dockerfile
dev:
pnpm --prefix ./projects/$(name) dev
build:
ifeq ($(proxy), taobao)
docker build -f $(filePath) -t $(image) . --build-arg proxy=taobao
else ifeq ($(proxy), clash)
docker build -f $(filePath) -t $(image) . --network host --build-arg HTTP_PROXY=http://127.0.0.1:7890 --build-arg HTTPS_PROXY=http://127.0.0.1:7890
else
docker build -f $(filePath) -t $(image) .
endif

43
dev.md
View File

@@ -1,17 +1,40 @@
# 打包命令
## Premise
Since FastGPT is managed in the same way as monorepo, it is recommended to install 'make' first during development.
monorepo Project Name:
- app: main project
-......
## Dev
```sh
# Build image, not proxy
docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app .
# Give automatic script code execution permission (on non-Linux systems, you can manually execute the postinstall.sh file content)
chmod -R +x ./scripts/
# Executing under the code root directory installs all dependencies within the root package, projects, and packages
pnpm i
# build image with proxy
docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app --build-arg proxy=taobao .
# Not make cmd
cd projects/app
pnpm dev
# Make cmd
make dev name=app
```
# Pg 常用索引
```sql
CREATE INDEX IF NOT EXISTS modelData_dataset_id_index ON modeldata (dataset_id);
CREATE INDEX IF NOT EXISTS modelData_collection_id_index ON modeldata (collection_id);
CREATE INDEX IF NOT EXISTS modelData_teamId_index ON modeldata (team_id);
## Build
```sh
# Docker cmd: Build image, not proxy
docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app
# Make cmd: Build image, not proxy
make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1
# Docker cmd: Build image with proxy
docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app --build-arg proxy=taobao
# Make cmd: Build image with proxy
make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 proxy=taobao
```

View File

@@ -16,8 +16,9 @@ weight: 705
- [Git](http://git-scm.com/)
- [Docker](https://www.docker.com/)(构建镜像)
- [Node.js v18.x (不推荐最新的,可能有兼容问题)](http://nodejs.org)
- [pnpm](https://pnpm.io/) 版本 8.x.x
- [Node.js v18.17 / v20.x](http://nodejs.org)
- [pnpm](https://pnpm.io/) 版本 8.6.0 (目前官方的开发环境)
- make命令: 根据不同平台,百度安装 (官方是GNU Make 4.3)
## 开始本地开发
@@ -72,24 +73,34 @@ Mongo 数据库需要注意,需要注意在连接地址中增加 `directConnec
### 5. 运行
可参考项目根目录下的 `dev.md`
```bash
# 给自动化脚本代码执行权限(非 linux 系统, 可以手动执行里面的 postinstall.sh 文件内容)
chmod -R +x ./scripts/
# 代码根目录下执行,会安装根 package、projects 和 packages 内所有依赖
pnpm i
# 切换到应用目录
# 非 Make 运行
cd projects/app
# 开发模式运行
pnpm dev
# Make 运行
make dev name=app
```
### 6. 部署打包
```bash
# 根目录下执行
docker build -t dockername/fastgpt:tag --build-arg name=app .
# 使用代理
docker build -t dockername/fastgpt:tag --build-arg name=app --build-arg proxy=taobao .
# Docker cmd: Build image, not proxy
docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app
# Make cmd: Build image, not proxy
make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1
# Docker cmd: Build image with proxy
docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app --build-arg proxy=taobao
# Make cmd: Build image with proxy
make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 proxy=taobao
```
## 提交代码至开源仓库
@@ -101,21 +112,21 @@ docker build -t dockername/fastgpt:tag --build-arg name=app --build-arg proxy=ta
如果遇到问题,比如合并冲突或不知道如何打开拉取请求,请查看 GitHub 的[拉取请求教程](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests),了解如何解决合并冲突和其他问题。一旦您的 PR 被合并,您将自豪地被列为[贡献者表](https://github.com/labring/FastGPT/graphs/contributors)中的一员。
## QA
### 本地数据库无法连接
1. 如果你是连接远程的数据库,先检查对应的端口是否开放。
2. 如果是本地运行的数据库,可尝试`host`改成`localhost``127.0.0.1`
3. 本地连接远程的 Mongo需要增加 `directConnection=true` 参数,才能连接上副本集的数据库。
4. mongo使用`mongocompass`客户端进行连接测试和可视化管理。
5. pg使用`navicat`进行连接和管理。
### sh ./scripts/postinstall.sh 没权限
FastGPT 在`pnpm i`后会执行`postinstall`脚本,用于自动生成`ChakraUI``Type`。如果没有权限,可以先执行`chmod -R +x ./scripts/`,再执行`pnpm i`
### 长时间运行后崩溃
似乎是由于 tiktoken 库的开发环境问题,生产环境中未遇到,暂时可忽略。
仍不可行的话,可以手动执行`./scripts/postinstall.sh`里的内容。
### TypeError: Cannot read properties of null (reading 'useMemo' )

View File

@@ -108,7 +108,11 @@ export enum NodeInputKeyEnum {
ifElseList = 'ifElseList',
// variable update
updateList = 'updateList'
updateList = 'updateList',
// code
code = 'code',
codeType = 'codeType' // js|py
}
export enum NodeOutputKeyEnum {
@@ -121,6 +125,7 @@ export enum NodeOutputKeyEnum {
error = 'error',
text = 'system_text',
addOutputParam = 'system_addOutputParam',
rawResponse = 'system_rawResponse',
// dataset
datasetQuoteQA = 'quoteQA',

View File

@@ -113,7 +113,8 @@ export enum FlowNodeTypeEnum {
stopTool = 'stopTool',
lafModule = 'lafModule',
ifElseNode = 'ifElseNode',
variableUpdate = 'variableUpdate'
variableUpdate = 'variableUpdate',
code = 'code'
}
export const EDGE_TYPE = 'default';

View File

@@ -31,6 +31,8 @@ export type DispatchNodeResponseType = {
runningTime?: number;
query?: string;
textOutput?: string;
customInputs?: Record<string, any>;
customOutputs?: Record<string, any>;
// bill
tokens?: number;

View File

@@ -22,6 +22,7 @@ import type { FlowNodeTemplateType } from '../type';
import { LafModule } from './system/laf';
import { IfElseNode } from './system/ifElse/index';
import { VariableUpdateNode } from './system/variableUpdate';
import { CodeNode } from './system/sandbox';
/* app flow module templates */
export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -40,7 +41,8 @@ export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
AiQueryExtension,
LafModule,
IfElseNode,
VariableUpdateNode
VariableUpdateNode,
CodeNode
];
/* plugin flow module templates */
export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -59,7 +61,8 @@ export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
AiQueryExtension,
LafModule,
IfElseNode,
VariableUpdateNode
VariableUpdateNode,
CodeNode
];
/* all module */
@@ -84,5 +87,6 @@ export const moduleTemplatesFlat: FlowNodeTemplateType[] = [
AiQueryExtension,
LafModule,
IfElseNode,
VariableUpdateNode
VariableUpdateNode,
CodeNode
];

View File

@@ -0,0 +1,7 @@
export const JS_TEMPLATE = `function main({data1, data2}){
return {
result: data1,
data2
}
}`;

View File

@@ -0,0 +1,72 @@
import {
FlowNodeTemplateTypeEnum,
NodeInputKeyEnum,
NodeOutputKeyEnum,
WorkflowIOValueTypeEnum
} from '../../../constants';
import {
FlowNodeInputTypeEnum,
FlowNodeOutputTypeEnum,
FlowNodeTypeEnum
} from '../../../node/constant';
import { FlowNodeTemplateType } from '../../../type';
import { getHandleConfig } from '../../utils';
import { Input_Template_DynamicInput } from '../../input';
import { Output_Template_AddOutput } from '../../output';
import { JS_TEMPLATE } from './constants';
export const CodeNode: FlowNodeTemplateType = {
id: FlowNodeTypeEnum.code,
templateType: FlowNodeTemplateTypeEnum.tools,
flowNodeType: FlowNodeTypeEnum.code,
sourceHandle: getHandleConfig(true, true, true, true),
targetHandle: getHandleConfig(true, true, true, true),
avatar: '/imgs/workflow/code.svg',
name: '代码运行',
intro: '执行一段简单的脚本代码,通常用于进行复杂的数据处理。',
showStatus: true,
version: '482',
inputs: [
{
...Input_Template_DynamicInput,
description: '这些变量会作为代码的运行的输入参数',
editField: {
key: true,
valueType: true
}
},
{
key: NodeInputKeyEnum.codeType,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
value: 'js'
},
{
key: NodeInputKeyEnum.code,
renderTypeList: [FlowNodeInputTypeEnum.custom],
label: '',
value: JS_TEMPLATE
}
],
outputs: [
{
...Output_Template_AddOutput,
description: '将代码中 return 的对象作为输出,传递给后续的节点'
},
{
id: NodeOutputKeyEnum.rawResponse,
key: NodeOutputKeyEnum.rawResponse,
label: '完整响应数据',
valueType: WorkflowIOValueTypeEnum.object,
type: FlowNodeOutputTypeEnum.static
},
{
id: NodeOutputKeyEnum.error,
key: NodeOutputKeyEnum.error,
label: '运行错误',
description: '代码运行错误信息,成功时返回空',
valueType: WorkflowIOValueTypeEnum.object,
type: FlowNodeOutputTypeEnum.static
}
]
};

View File

@@ -1,38 +0,0 @@
// import { addLog } from '../../../../common/system/log';
// const ivm = require('isolated-vm');
// export const runJsCode = ({
// code,
// variables
// }: {
// code: string;
// variables: Record<string, any>;
// }) => {
// const isolate = new ivm.Isolate({ memoryLimit: 16 });
// const context = isolate.createContextSync();
// const jail = context.global;
// return new Promise((resolve, reject) => {
// // custom log function
// jail.setSync('responseData', function (args: any): any {
// if (typeof args === 'object') {
// resolve(args);
// } else {
// reject('Not an invalid response');
// }
// });
// // Add global variables
// jail.setSync('variables', new ivm.ExternalCopy(variables).copyInto());
// try {
// const scriptCode = `
// ${code}
// responseData(main(variables))`;
// context.evalSync(scriptCode, { timeout: 2000 });
// } catch (err) {
// addLog.error('Error during script execution:', err);
// reject(err);
// }
// });
// };

View File

@@ -0,0 +1,51 @@
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type/index.d';
import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import axios from 'axios';
import { formatHttpError } from '../utils';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
type RunCodeType = ModuleDispatchProps<{
[NodeInputKeyEnum.codeType]: 'js';
[NodeInputKeyEnum.code]: string;
[key: string]: any;
}>;
type RunCodeResponse = DispatchNodeResultType<{
[NodeOutputKeyEnum.error]?: any;
[NodeOutputKeyEnum.rawResponse]?: Record<string, any>;
[key: string]: any;
}>;
export const dispatchRunCode = async (props: RunCodeType): Promise<RunCodeResponse> => {
const {
params: { codeType, code, ...customVariables }
} = props;
const sandBoxRequestUrl = `${process.env.SANDBOX_URL}/sandbox/js`;
try {
const { data: runResult } = await axios.post<{
success: boolean;
data: Record<string, any>;
}>(sandBoxRequestUrl, {
code,
variables: customVariables
});
if (runResult.success) {
return {
[NodeOutputKeyEnum.rawResponse]: runResult.data,
[DispatchNodeResponseKeyEnum.nodeResponse]: {
customInputs: customVariables,
customOutputs: runResult.data
},
...runResult.data
};
} else {
throw new Error('Run code failed');
}
} catch (error) {
return {
[NodeOutputKeyEnum.error]: formatHttpError(error)
};
}
};

View File

@@ -46,6 +46,7 @@ import { dispatchSystemConfig } from './init/systemConfig';
import { dispatchUpdateVariable } from './tools/runUpdateVar';
import { addLog } from '../../../common/system/log';
import { surrenderProcess } from '../../../common/system/tools';
import { dispatchRunCode } from './code/run';
const callbackMap: Record<FlowNodeTypeEnum, Function> = {
[FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart,
@@ -66,6 +67,7 @@ const callbackMap: Record<FlowNodeTypeEnum, Function> = {
[FlowNodeTypeEnum.lafModule]: dispatchLafRequest,
[FlowNodeTypeEnum.ifElseNode]: dispatchIfElse,
[FlowNodeTypeEnum.variableUpdate]: dispatchUpdateVariable,
[FlowNodeTypeEnum.code]: dispatchRunCode,
// none
[FlowNodeTypeEnum.systemConfig]: dispatchSystemConfig,

View File

@@ -9,7 +9,7 @@ import {
SseResponseEventEnum
} from '@fastgpt/global/core/workflow/runtime/constants';
import axios from 'axios';
import { valueTypeFormat } from '../utils';
import { formatHttpError, valueTypeFormat } from '../utils';
import { SERVICE_LOCAL_HOST } from '../../../../common/system/tools';
import { addLog } from '../../../../common/system/log';
import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
@@ -310,14 +310,3 @@ function removeUndefinedSign(obj: Record<string, any>) {
}
return obj;
}
function formatHttpError(error: any) {
return {
message: error?.message,
name: error?.name,
method: error?.config?.method,
baseURL: error?.config?.baseURL,
url: error?.config?.url,
code: error?.code,
status: error?.status
};
}

View File

@@ -3,12 +3,7 @@ import {
WorkflowIOValueTypeEnum,
NodeOutputKeyEnum
} from '@fastgpt/global/core/workflow/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import {
RuntimeEdgeItemType,
RuntimeNodeItemType
} from '@fastgpt/global/core/workflow/runtime/type';
import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/index.d';
import { RuntimeEdgeItemType } from '@fastgpt/global/core/workflow/runtime/type';
export const filterToolNodeIdByEdges = ({
nodeId,
@@ -91,3 +86,15 @@ export const removeSystemVariable = (variables: Record<string, any>) => {
return copyVariables;
};
export const formatHttpError = (error: any) => {
return {
message: error?.message,
name: error?.name,
method: error?.config?.method,
baseURL: error?.config?.baseURL,
url: error?.config?.url,
code: error?.code,
status: error?.status
};
};

View File

@@ -0,0 +1,160 @@
import React, { useCallback, useRef, useState } from 'react';
import Editor, { Monaco, loader } from '@monaco-editor/react';
import { Box, BoxProps } from '@chakra-ui/react';
import MyIcon from '../../Icon';
loader.config({
paths: { vs: '/js/monaco-editor.0.45.0/vs' }
});
type EditorVariablePickerType = {
key: string;
label: string;
};
export type Props = Omit<BoxProps, 'resize' | 'onChange'> & {
height?: number;
resize?: boolean;
defaultValue?: string;
value?: string;
onChange?: (e: string) => void;
onOpenModal?: () => void;
variables?: EditorVariablePickerType[];
defaultHeight?: number;
};
const options = {
lineNumbers: 'on',
guides: {
indentation: false
},
automaticLayout: true,
minimap: {
enabled: false
},
scrollbar: {
verticalScrollbarSize: 4,
horizontalScrollbarSize: 8,
alwaysConsumeMouseWheel: false
},
lineNumbersMinChars: 0,
fontSize: 14,
scrollBeyondLastLine: false,
folding: true,
overviewRulerBorder: false,
tabSize: 2
};
const MyEditor = ({
defaultValue,
value,
onChange,
resize,
variables = [],
defaultHeight = 200,
onOpenModal,
...props
}: Props) => {
const [height, setHeight] = useState(defaultHeight);
const initialY = useRef(0);
const handleMouseDown = useCallback((e: React.MouseEvent) => {
initialY.current = e.clientY;
const handleMouseMove = (e: MouseEvent) => {
const deltaY = e.clientY - initialY.current;
initialY.current = e.clientY;
setHeight((prevHeight) => (prevHeight + deltaY < 100 ? 100 : prevHeight + deltaY));
};
const handleMouseUp = () => {
document.removeEventListener('mousemove', handleMouseMove);
document.removeEventListener('mouseup', handleMouseUp);
};
document.addEventListener('mousemove', handleMouseMove);
document.addEventListener('mouseup', handleMouseUp);
}, []);
const beforeMount = useCallback((monaco: Monaco) => {
monaco.languages.json.jsonDefaults.setDiagnosticsOptions({
validate: false,
allowComments: false,
schemas: [
{
uri: 'http://myserver/foo-schema.json', // 一个假设的 URI
fileMatch: ['*'], // 匹配所有文件
schema: {} // 空的 Schema
}
]
});
monaco.editor.defineTheme('JSONEditorTheme', {
base: 'vs', // 可以基于已有的主题进行定制
inherit: true, // 继承基础主题的设置
rules: [{ token: 'variable', foreground: '2B5FD9' }],
colors: {
'editor.background': '#ffffff00',
'editorLineNumber.foreground': '#aaa',
'editorOverviewRuler.border': '#ffffff00',
'editor.lineHighlightBackground': '#F7F8FA',
'scrollbarSlider.background': '#E8EAEC',
'editorIndentGuide.activeBackground': '#ddd',
'editorIndentGuide.background': '#eee'
}
});
}, []);
return (
<Box
borderWidth={'1px'}
borderRadius={'md'}
borderColor={'myGray.200'}
py={2}
height={height}
position={'relative'}
pl={2}
{...props}
>
<Editor
height={'100%'}
defaultLanguage="typescript"
options={options as any}
theme="JSONEditorTheme"
beforeMount={beforeMount}
defaultValue={defaultValue}
value={value}
onChange={(e) => {
onChange?.(e || '');
}}
/>
{resize && (
<Box
position={'absolute'}
right={'-1'}
bottom={'-1'}
zIndex={10}
cursor={'ns-resize'}
px={'4px'}
onMouseDown={handleMouseDown}
>
<MyIcon name={'common/editor/resizer'} width={'16px'} height={'16px'} />
</Box>
)}
{!!onOpenModal && (
<Box
zIndex={10}
position={'absolute'}
bottom={0}
right={2}
cursor={'pointer'}
onClick={onOpenModal}
>
<MyIcon name={'common/fullScreenLight'} w={'14px'} color={'myGray.600'} />
</Box>
)}
</Box>
);
};
export default React.memo(MyEditor);

View File

@@ -0,0 +1,36 @@
import React from 'react';
import MyEditor, { type Props as EditorProps } from './Editor';
import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react';
import MyModal from '../../MyModal';
import { useTranslation } from 'next-i18next';
type Props = Omit<EditorProps, 'resize'> & {};
const CodeEditor = (props: Props) => {
const { t } = useTranslation();
const { isOpen, onOpen, onClose } = useDisclosure();
return (
<>
<MyEditor {...props} resize onOpenModal={onOpen} />
<MyModal
isOpen={isOpen}
onClose={onClose}
iconSrc="modal/edit"
title={t('Code editor')}
w={'full'}
>
<ModalBody>
<MyEditor {...props} bg={'myGray.50'} defaultHeight={600} />
</ModalBody>
<ModalFooter>
<Button mr={2} onClick={onClose}>
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
</>
);
};
export default React.memo(CodeEditor);

3469
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -20,6 +20,8 @@ CHAT_API_KEY=sk-xxxx
MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt?authSource=admin
# PG 数据库连接参数
PG_URL=postgresql://username:password@host:port/postgres
# code sandbox url
SANDBOX_URL=http://localhost:3001
# 商业版地址
PRO_URL=
# 首页路径

View File

@@ -2,7 +2,6 @@
FROM node:18.17-alpine AS mainDeps
WORKDIR /app
ARG name
ARG proxy
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
@@ -13,7 +12,7 @@ RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npmmirror.com
# copy packages and one project
COPY pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./
COPY ./packages ./packages
COPY ./projects/$name/package.json ./projects/$name/package.json
COPY ./projects/app/package.json ./projects/app/package.json
RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
@@ -23,28 +22,26 @@ RUN pnpm i
FROM node:18.17-alpine AS builder
WORKDIR /app
ARG name
ARG proxy
# copy common node_modules and one project node_modules
COPY package.json pnpm-workspace.yaml .npmrc ./
COPY --from=mainDeps /app/node_modules ./node_modules
COPY --from=mainDeps /app/packages ./packages
COPY ./projects/$name ./projects/$name
COPY --from=mainDeps /app/projects/$name/node_modules ./projects/$name/node_modules
COPY ./projects/app ./projects/app
COPY --from=mainDeps /app/projects/app/node_modules ./projects/app/node_modules
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
ENV NODE_OPTIONS="--max-old-space-size=4096"
RUN pnpm --filter=$name build
RUN pnpm --filter=app build
# --------- runner -----------
FROM node:18.17-alpine AS runner
WORKDIR /app
ARG name
ARG proxy
# create user and use it
@@ -56,23 +53,23 @@ RUN apk add --no-cache curl ca-certificates \
&& update-ca-certificates
# copy running files
COPY --from=builder /app/projects/$name/public /app/projects/$name/public
COPY --from=builder /app/projects/$name/next.config.js /app/projects/$name/next.config.js
COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/standalone /app/
COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/static /app/projects/$name/.next/static
COPY --from=builder /app/projects/app/public /app/projects/app/public
COPY --from=builder /app/projects/app/next.config.js /app/projects/app/next.config.js
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/.next/standalone /app/
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/.next/static /app/projects/app/.next/static
# copy server chunks
COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/server/chunks /app/projects/$name/.next/server/chunks
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/.next/server/chunks /app/projects/app/.next/server/chunks
# copy worker
COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/server/worker /app/projects/$name/.next/server/worker
COPY --from=builder --chown=nextjs:nodejs /app/projects/app/.next/server/worker /app/projects/app/.next/server/worker
# copy tiktoken but not copy ./node_modules/tiktoken/encoders
COPY --from=mainDeps /app/node_modules/tiktoken ./node_modules/tiktoken
RUN rm -rf ./node_modules/tiktoken/encoders
# copy package.json to version file
COPY --from=builder /app/projects/$name/package.json ./package.json
COPY --from=builder /app/projects/app/package.json ./package.json
# copy config
COPY ./projects/$name/data /app/data
COPY ./projects/app/data /app/data
RUN chown -R nextjs:nodejs /app/data
@@ -84,6 +81,6 @@ EXPOSE 3000
USER nextjs
ENV serverPath=./projects/$name/server.js
ENV serverPath=./projects/app/server.js
ENTRYPOINT ["sh","-c","node --max-old-space-size=4096 ${serverPath}"]

View File

@@ -1,6 +1,7 @@
{
"Add new": "Add new",
"App": "App",
"Code editor": "Code edit",
"Export": "Export",
"Folder": "Folder",
"Is open": "Opened",

View File

@@ -1,3 +1,12 @@
{
"Field required": "Required"
"Code": "Code",
"Field required": "Required",
"code": {
"Reset template": "Reset template",
"Reset template confirm": "Are you sure to restore the code template? Be careful to save the current code."
},
"response": {
"Custom inputs": "Custom inputs",
"Custom outputs": "Custom outputs"
}
}

View File

@@ -1,6 +1,7 @@
{
"Add new": "新增",
"App": "应用",
"Code editor": "代码编辑",
"Export": "导出",
"Folder": "文件夹",
"Is open": "是否开启",

View File

@@ -1,3 +1,12 @@
{
"Field required": "必填"
"Code": "代码",
"Field required": "必填",
"code": {
"Reset template": "还原模板",
"Reset template confirm": "确认还原代码模板?请注意保存当前代码。"
},
"response": {
"Custom inputs": "自定义输入",
"Custom outputs": "自定义输出"
}
}

View File

@@ -89,7 +89,7 @@ const nextConfig = {
transpilePackages: ['@fastgpt/*', 'ahooks'],
experimental: {
// 优化 Server Components 的构建和运行,避免不必要的客户端打包。
serverComponentsExternalPackages: ['mongoose', 'pg'],
serverComponentsExternalPackages: ['mongoose', 'pg', '@node-rs/jieba'],
outputFileTracingRoot: path.join(__dirname, '../../')
}
};

View File

@@ -0,0 +1,24 @@
<svg t="1716862994060" class="icon" viewBox="0 0 1218 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4475"
width="256" height="256">
<path
d="M1169.592417 975.469194V145.592417c0-53.383886-43.677725-97.061611-97.061611-97.061611H145.592417c-53.383886 0-97.061611 43.677725-97.061611 97.061611v829.876777h1121.061611z"
fill="#7CDEDC" p-id="4476"></path>
<path
d="M1193.85782 975.469194V145.592417c0-66.899716-54.427299-121.327014-121.327014-121.327014H145.592417C78.692701 24.265403 24.265403 78.692701 24.265403 145.592417v854.14218h1169.592417v-24.265403z m-1121.061611 0V145.592417c0-40.139829 32.656379-72.796209 72.796208-72.796208h926.938389c40.139829 0 72.796209 32.656379 72.796208 72.796208v829.876777l24.265403-24.265403H48.530806l24.265403 24.265403z"
fill="#6E6E96" p-id="4477"></path>
<path d="M48.530806 237.800948h1145.327014v48.530806H48.530806z" fill="#6E6E96" p-id="4478"></path>
<path
d="M1072.530806 48.530806H145.592417c-53.383886 0-97.061611 43.677725-97.061611 97.061611v111.620853h1121.061611V145.592417c0-53.383886-43.677725-97.061611-97.061611-97.061611z"
fill="#FFF491" p-id="4479"></path>
<path
d="M1072.530806 24.265403H145.592417C78.692701 24.265403 24.265403 78.692701 24.265403 145.592417v135.886256h1169.592417V145.592417c0-66.899716-54.427299-121.327014-121.327014-121.327014z m72.796208 121.327014v111.620853l24.265403-24.265403H48.530806l24.265403 24.265403V145.592417c0-40.139829 32.656379-72.796209 72.796208-72.796208h926.938389c40.139829 0 72.796209 32.656379 72.796208 72.796208zM374.725763 431.74946l-167.43128 167.436132L190.138844 616.341232l17.155639 17.15564 167.43128 167.43128 34.320986-34.31128-167.43128-167.43128v34.31128l167.43128-167.426427zM813.934408 466.070445l167.431279 167.426427v-34.31128l-167.431279 167.43128 34.311279 34.31128 167.43128-167.43128L1032.832607 616.341232l-17.15564-17.15564-167.43128-167.436132z"
fill="#6E6E96" p-id="4480"></path>
<path
d="M531.412322 618.767773m-41.251185 0a41.251185 41.251185 0 1 0 82.50237 0 41.251185 41.251185 0 1 0-82.50237 0Z"
fill="#6E6E96" p-id="4481"></path>
<path
d="M706.123223 618.767773m-41.251185 0a41.251185 41.251185 0 1 0 82.50237 0 41.251185 41.251185 0 1 0-82.50237 0Z"
fill="#6E6E96" p-id="4482"></path>
<path d="M72.796209 281.478673h1072.530805v53.383886H72.796209z" fill="#6E6E96" opacity=".15" p-id="4483"></path>
<path d="M72.796209 189.270142h1072.530805v43.677725H72.796209z" fill="#6E6E96" opacity=".15" p-id="4484"></path>
</svg>

After

Width:  |  Height:  |  Size: 2.5 KiB

View File

@@ -1,5 +1,5 @@
import React, { useMemo, useState } from 'react';
import { Box, useTheme, Flex, Image } from '@chakra-ui/react';
import React, { useCallback, useMemo, useState } from 'react';
import { Box, useTheme, Flex, Image, BoxProps } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d';
import { useTranslation } from 'next-i18next';
import { moduleTemplatesFlat } from '@fastgpt/global/core/workflow/template/constants';
@@ -12,42 +12,68 @@ import Markdown from '../Markdown';
import { QuoteList } from './QuoteModal';
import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constants';
import { formatNumber } from '@fastgpt/global/common/math/tools';
import { useI18n } from '@/web/context/I18n';
function RowRender({
children,
mb,
label,
...props
}: { children: React.ReactNode; label: string } & BoxProps) {
return (
<Box mb={3}>
<Box fontSize={['sm', 'md']} mb={mb} flex={'0 0 90px'}>
{label}:
</Box>
<Box borderRadius={'sm'} fontSize={'sm'} bg={'myGray.50'} {...props}>
{children}
</Box>
</Box>
);
}
function Row({
label,
value,
rawDom
}: {
label: string;
value?: string | number | boolean;
value?: string | number | boolean | object;
rawDom?: React.ReactNode;
}) {
const { t } = useTranslation();
const theme = useTheme();
const val = value || rawDom;
const strValue = `${value}`;
const isCodeBlock = strValue.startsWith('~~~json');
const isObject = typeof value === 'object';
return val !== undefined && val !== '' && val !== 'undefined' ? (
<Box mb={3}>
<Box fontSize={['sm', 'md']} mb={isCodeBlock ? 0 : 1} flex={'0 0 90px'}>
{t(label)}:
</Box>
<Box
borderRadius={'sm'}
fontSize={'sm'}
bg={'myGray.50'}
{...(isCodeBlock
const formatValue = useMemo(() => {
if (isObject) {
return `~~~json\n${JSON.stringify(value, null, 2)}`;
}
return `${value}`;
}, [isObject, value]);
if (rawDom) {
return (
<RowRender label={label} mb={1}>
{rawDom}
</RowRender>
);
}
if (val === undefined || val === '' || val === 'undefined') return null;
return (
<RowRender
label={label}
mb={isObject ? 0 : 1}
{...(isObject
? { transform: 'translateY(-3px)' }
: value
? { px: 3, py: 2, border: theme.borders.base }
: {})}
>
{value && <Markdown source={strValue} />}
{rawDom}
</Box>
</Box>
) : null;
<Markdown source={formatValue} />
</RowRender>
);
}
const WholeResponseModal = ({
@@ -98,6 +124,7 @@ export const ResponseBox = React.memo(function ResponseBox({
}) {
const theme = useTheme();
const { t } = useTranslation();
const { workflowT } = useI18n();
const list = useMemo(
() =>
@@ -251,47 +278,26 @@ export const ResponseBox = React.memo(function ResponseBox({
label={t('core.chat.response.module extract description')}
value={activeModule?.extractDescription}
/>
{activeModule?.extractResult && (
<Row
label={t('core.chat.response.module extract result')}
value={`~~~json\n${JSON.stringify(activeModule?.extractResult, null, 2)}`}
value={activeModule?.extractResult}
/>
)}
</>
{/* http */}
<>
{activeModule?.headers && (
<Row
label={'Headers'}
value={`~~~json\n${JSON.stringify(activeModule?.headers, null, 2)}`}
/>
)}
{activeModule?.params && (
<Row
label={'Params'}
value={`~~~json\n${JSON.stringify(activeModule?.params, null, 2)}`}
/>
)}
{activeModule?.body && (
<Row label={'Body'} value={`~~~json\n${JSON.stringify(activeModule?.body, null, 2)}`} />
)}
{activeModule?.httpResult && (
<Row label={'Headers'} value={activeModule?.headers} />
<Row label={'Params'} value={activeModule?.params} />
<Row label={'Body'} value={activeModule?.body} />
<Row
label={t('core.chat.response.module http result')}
value={`~~~json\n${JSON.stringify(activeModule?.httpResult, null, 2)}`}
value={activeModule?.httpResult}
/>
)}
</>
{/* plugin */}
<>
{activeModule?.pluginOutput && (
<Row
label={t('core.chat.response.plugin output')}
value={`~~~json\n${JSON.stringify(activeModule?.pluginOutput, null, 2)}`}
/>
)}
<Row label={t('core.chat.response.plugin output')} value={activeModule?.pluginOutput} />
{activeModule?.pluginDetail && activeModule?.pluginDetail.length > 0 && (
<Row
label={t('core.chat.response.Plugin response detail')}
@@ -310,6 +316,10 @@ export const ResponseBox = React.memo(function ResponseBox({
rawDom={<ResponseBox response={activeModule.toolDetail} showDetail={showDetail} />}
/>
)}
{/* code */}
<Row label={workflowT('response.Custom inputs')} value={activeModule?.customInputs} />
<Row label={workflowT('response.Custom outputs')} value={activeModule?.customOutputs} />
</Box>
</>
);

View File

@@ -58,7 +58,8 @@ const nodeTypes: Record<FlowNodeTypeEnum, any> = {
),
[FlowNodeTypeEnum.lafModule]: dynamic(() => import('./nodes/NodeLaf')),
[FlowNodeTypeEnum.ifElseNode]: dynamic(() => import('./nodes/NodeIfElse')),
[FlowNodeTypeEnum.variableUpdate]: dynamic(() => import('./nodes/NodeVariableUpdate'))
[FlowNodeTypeEnum.variableUpdate]: dynamic(() => import('./nodes/NodeVariableUpdate')),
[FlowNodeTypeEnum.code]: dynamic(() => import('./nodes/NodeCode'))
};
const edgeTypes = {
[EDGE_TYPE]: ButtonEdge

View File

@@ -0,0 +1,107 @@
import React, { useCallback, useMemo } from 'react';
import { NodeProps } from 'reactflow';
import NodeCard from './render/NodeCard';
import { FlowNodeItemType } from '@fastgpt/global/core/workflow/type/index.d';
import Container from '../components/Container';
import RenderInput from './render/RenderInput';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { useTranslation } from 'next-i18next';
import { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io.d';
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '../../context';
import IOTitle from '../components/IOTitle';
import RenderToolInput from './render/RenderToolInput';
import RenderOutput from './render/RenderOutput';
import CodeEditor from '@fastgpt/web/components/common/Textarea/CodeEditor';
import { Box, Flex } from '@chakra-ui/react';
import { useI18n } from '@/web/context/I18n';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { JS_TEMPLATE } from '@fastgpt/global/core/workflow/template/system/sandbox/constants';
const NodeCode = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
const { t } = useTranslation();
const { workflowT } = useI18n();
const { nodeId, inputs, outputs } = data;
const splitToolInputs = useContextSelector(WorkflowContext, (ctx) => ctx.splitToolInputs);
const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode);
const { toolInputs, commonInputs } = splitToolInputs(inputs, nodeId);
const { ConfirmModal, openConfirm } = useConfirm({
content: workflowT('code.Reset template confirm')
});
const CustomComponent = useMemo(
() => ({
[NodeInputKeyEnum.code]: (item: FlowNodeInputItemType) => {
return (
<Box>
<Flex mb={1} alignItems={'flex-end'}>
<Box flex={'1'}>{workflowT('Code')}</Box>
<Box
cursor={'pointer'}
color={'primary.500'}
fontSize={'xs'}
onClick={openConfirm(() => {
onChangeNode({
nodeId,
type: 'updateInput',
key: item.key,
value: {
...item,
value: JS_TEMPLATE
}
});
})}
>
{workflowT('code.Reset template')}
</Box>
</Flex>
<CodeEditor
bg={'white'}
borderRadius={'sm'}
value={item.value}
onChange={(e) => {
onChangeNode({
nodeId,
type: 'updateInput',
key: item.key,
value: {
...item,
value: e
}
});
}}
/>
</Box>
);
}
}),
[nodeId, onChangeNode, openConfirm, workflowT]
);
return (
<NodeCard minW={'400px'} selected={selected} {...data}>
{toolInputs.length > 0 && (
<>
<Container>
<IOTitle text={t('core.module.tool.Tool input')} />
<RenderToolInput nodeId={nodeId} inputs={toolInputs} />
</Container>
</>
)}
<Container>
<IOTitle text={t('common.Input')} />
<RenderInput
nodeId={nodeId}
flowInputList={commonInputs}
CustomComponent={CustomComponent}
/>
</Container>
<Container>
<IOTitle text={t('common.Output')} />
<RenderOutput nodeId={nodeId} flowOutputList={outputs} />
</Container>
<ConfirmModal />
</NodeCard>
);
};
export default React.memo(NodeCode);

View File

@@ -72,7 +72,6 @@ const AddInputParam = (props: RenderInputProps) => {
leftIcon={<SmallAddIcon />}
iconSpacing={1}
size={'sm'}
mr={'-5px'}
onClick={() => setEditField(item.dynamicParamDefaultValue ?? {})}
>
{t('common.Add New')}

View File

@@ -13,6 +13,7 @@ import { FlowValueTypeMap } from '@/web/core/workflow/constants/dataType';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '@/components/core/workflow/context';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
const RenderList: {
types: `${FlowNodeOutputTypeEnum}`[];
@@ -61,6 +62,7 @@ const RenderOutput = ({
<Box position={'relative'} fontWeight={'medium'}>
{t('core.workflow.Custom outputs')}
</Box>
<QuestionTip ml={1} label={addOutput.description} />
<Box flex={'1 0 0'} />
<Button
variant={'whitePrimary'}

View File

@@ -148,7 +148,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
if ((await targetCol.countDocuments()) > 1) {
// 除了root
console.log('team_members 中有数据,无法自动将 buffer.tts 迁移到 team_members请手动操作');
console.log('team_members 中有数据,无法自动将 team.tts 迁移到 team_members请手动操作');
} else {
await sourceCol.rename('team_members', { dropTarget: true });
console.log('success rename team.members -> team_members');

View File

@@ -0,0 +1,11 @@
Dockerfile
.dockerignore
node_modules
npm-debug.log
README.md
.next
.git
.yalc/
yalc.lock
testApi/

View File

@@ -0,0 +1,25 @@
module.exports = {
parser: '@typescript-eslint/parser',
parserOptions: {
project: 'tsconfig.json',
tsconfigRootDir: __dirname,
sourceType: 'module'
},
plugins: ['@typescript-eslint/eslint-plugin'],
extends: ['plugin:@typescript-eslint/recommended'],
root: true,
env: {
node: true,
jest: true
},
ignorePatterns: ['.eslintrc.js'],
rules: {
'@typescript-eslint/interface-name-prefix': 'off',
'@typescript-eslint/explicit-function-return-type': 'off',
'@typescript-eslint/explicit-module-boundary-types': 'off',
'@typescript-eslint/no-explicit-any': 'off',
'@typescript-eslint/no-unused-vars': 'warn',
'@typescript-eslint/ban-ts-comment': 'off',
'@typescript-eslint/no-var-requires': 'off'
}
};

34
projects/sandbox/.gitignore vendored Normal file
View File

@@ -0,0 +1,34 @@
# dependencies
node_modules/
# next.js
.next/
out/
# production
build/
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# local env files
.env*.local
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts
platform.json
testApi/
local/
.husky/
data/*.local.*
storage/

View File

@@ -0,0 +1,50 @@
# --------- install dependence -----------
FROM python:3.11-alpine AS python_base
# 安装make和g++
RUN apk add --no-cache make g++
FROM node:20.13-alpine AS install
WORKDIR /app
ARG proxy
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache make g++
# copy py3.11
COPY --from=python_base /usr/local /usr/local
RUN npm install -g pnpm@8.6.2
RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npmmirror.com
COPY pnpm-lock.yaml pnpm-workspace.yaml ./
COPY ./projects/sandbox/package.json ./projects/sandbox/package.json
RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
RUN pnpm i
# --------- builder -----------
FROM node:20.13-alpine AS builder
WORKDIR /app
COPY package.json pnpm-workspace.yaml /app
COPY --from=install /app/node_modules /app/node_modules
COPY ./projects/sandbox /app/projects/sandbox
COPY --from=install /app/projects/sandbox /app/projects/sandbox
RUN npm install -g pnpm@8.6.2
RUN pnpm --filter=sandbox build
# --------- runner -----------
FROM node:20.13-alpine AS runner
WORKDIR /app
COPY --from=builder /app/node_modules /app/node_modules
COPY --from=builder /app/projects/sandbox /app/projects/sandbox
ENV NODE_ENV=production
CMD ["node", "projects/sandbox/dist/main.js"]

View File

@@ -0,0 +1,73 @@
<p align="center">
<a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo-small.svg" width="200" alt="Nest Logo" /></a>
</p>
[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
[circleci-url]: https://circleci.com/gh/nestjs/nest
<p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
<p align="center">
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
<a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
<a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
</p>
<!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
[![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
## Description
[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
## Installation
```bash
$ pnpm install
```
## Running the app
```bash
# development
$ pnpm run start
# watch mode
$ pnpm run start:dev
# production mode
$ pnpm run start:prod
```
## Test
```bash
# unit tests
$ pnpm run test
# e2e tests
$ pnpm run test:e2e
# test coverage
$ pnpm run test:cov
```
## Support
Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
## Stay in touch
- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
- Website - [https://nestjs.com](https://nestjs.com/)
- Twitter - [@nestframework](https://twitter.com/nestframework)
## License
Nest is [MIT licensed](LICENSE).

View File

@@ -0,0 +1,9 @@
{
"$schema": "https://json.schemastore.org/nest-cli",
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"deleteOutDir": true,
"plugins": ["@nestjs/swagger"]
}
}

View File

@@ -0,0 +1,69 @@
{
"name": "sandbox",
"version": "0.0.1",
"description": "",
"author": "",
"private": true,
"license": "UNLICENSED",
"scripts": {
"build": "nest build",
"start": "nest start",
"dev": "nest start --watch",
"start:debug": "nest start --debug --watch",
"start:prod": "node dist/main",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@fastify/static": "^7.0.4",
"@nestjs/common": "^10.0.0",
"@nestjs/core": "^10.0.0",
"@nestjs/platform-fastify": "^10.3.8",
"@nestjs/swagger": "^7.3.1",
"fastify": "^4.27.0",
"isolated-vm": "^4.7.2",
"node-gyp": "^10.1.0",
"reflect-metadata": "^0.2.0",
"rxjs": "^7.8.1"
},
"devDependencies": {
"@nestjs/cli": "^10.0.0",
"@nestjs/schematics": "^10.0.0",
"@nestjs/testing": "^10.0.0",
"@types/jest": "^29.5.2",
"@types/node": "^20.3.1",
"@types/supertest": "^6.0.0",
"@typescript-eslint/eslint-plugin": "^6.0.0",
"@typescript-eslint/parser": "^6.0.0",
"eslint": "^8.42.0",
"jest": "^29.5.0",
"source-map-support": "^0.5.21",
"supertest": "^6.3.3",
"ts-jest": "^29.1.0",
"ts-loader": "^9.4.3",
"ts-node": "^10.9.1",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.1.3"
},
"jest": {
"moduleFileExtensions": [
"js",
"json",
"ts"
],
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},
"collectCoverageFrom": [
"**/*.(t|j)s"
],
"coverageDirectory": "../coverage",
"testEnvironment": "node"
}
}

View File

@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { SandboxController } from './sandbox/sandbox.controller';
import { SandboxService } from './sandbox/sandbox.service';
@Module({
imports: [],
controllers: [SandboxController],
providers: [SandboxService]
})
export class AppModule {}

View File

@@ -0,0 +1,18 @@
import { ExceptionFilter, Catch, ArgumentsHost } from '@nestjs/common';
import { FastifyRequest, FastifyReply } from 'fastify';
import { getErrText } from './utils';
@Catch()
export class HttpExceptionFilter implements ExceptionFilter {
catch(error: any, host: ArgumentsHost) {
const ctx = host.switchToHttp();
const response = ctx.getResponse<FastifyReply>();
const request = ctx.getRequest<FastifyRequest>();
response.status(500).send({
success: false,
time: new Date(),
msg: getErrText(error)
});
}
}

View File

@@ -0,0 +1,38 @@
import { NestFactory } from '@nestjs/core';
import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify';
import { AppModule } from './app.module';
import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger';
import { HttpExceptionFilter } from './http-exception.filter';
import { ResponseInterceptor } from './response';
async function bootstrap(port: number) {
const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter());
// 使用全局异常过滤器
app.useGlobalFilters(new HttpExceptionFilter());
app.useGlobalInterceptors(new ResponseInterceptor());
const config = new DocumentBuilder()
.setTitle('Cats example')
.setDescription('The cats API description')
.setVersion('1.0')
.addTag('cats')
.build();
const document = SwaggerModule.createDocument(app, config);
SwaggerModule.setup('api', app, document);
try {
await app.listen(port, '0.0.0.0');
console.log(`Application is running on: ${await app.getUrl()}`);
} catch (error) {
if (error.code === 'EADDRINUSE') {
console.warn(`Port ${port} is already in use, trying next port...`);
await bootstrap(port + 1);
} else {
console.error(`Failed to start application: ${error.message}`);
process.exit(1);
}
}
}
bootstrap(3000);

View File

@@ -0,0 +1,15 @@
import { Injectable, NestInterceptor, ExecutionContext, CallHandler } from '@nestjs/common';
import { Observable } from 'rxjs';
import { map } from 'rxjs/operators';
@Injectable()
export class ResponseInterceptor implements NestInterceptor {
intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
return next.handle().pipe(
map((data) => ({
success: true,
data
}))
);
}
}

View File

@@ -0,0 +1,4 @@
export class RunCodeDto {
code: string;
variables: object;
}

View File

@@ -0,0 +1,20 @@
import { Test, TestingModule } from '@nestjs/testing';
import { SandboxController } from './sandbox.controller';
import { SandboxService } from './sandbox.service';
describe('SandboxController', () => {
let controller: SandboxController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [SandboxController],
providers: [SandboxService]
}).compile();
controller = module.get<SandboxController>(SandboxController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

View File

@@ -0,0 +1,15 @@
import { Controller, Post, Body, HttpCode } from '@nestjs/common';
import { SandboxService } from './sandbox.service';
import { RunCodeDto } from './dto/create-sandbox.dto';
import { WorkerNameEnum, runWorker } from 'src/worker/utils';
@Controller('sandbox')
export class SandboxController {
constructor(private readonly sandboxService: SandboxService) {}
@Post('/js')
@HttpCode(200)
runJs(@Body() codeProps: RunCodeDto) {
return runWorker(WorkerNameEnum.runJs, codeProps);
}
}

View File

@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { SandboxService } from './sandbox.service';
import { SandboxController } from './sandbox.controller';
@Module({
controllers: [SandboxController],
providers: [SandboxService]
})
export class SandboxModule {}

View File

@@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { SandboxService } from './sandbox.service';
describe('SandboxService', () => {
let service: SandboxService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [SandboxService]
}).compile();
service = module.get<SandboxService>(SandboxService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@@ -0,0 +1,10 @@
import { Injectable } from '@nestjs/common';
import { RunCodeDto } from './dto/create-sandbox.dto';
import { WorkerNameEnum, runWorker } from 'src/worker/utils';
@Injectable()
export class SandboxService {
runJs(params: RunCodeDto) {
return runWorker(WorkerNameEnum.runJs, params);
}
}

View File

@@ -0,0 +1,14 @@
export const replaceSensitiveText = (text: string) => {
// 1. http link
text = text.replace(/(?<=https?:\/\/)[^\s]+/g, 'xxx');
// 2. nx-xxx 全部替换成xxx
text = text.replace(/ns-[\w-]+/g, 'xxx');
return text;
};
export const getErrText = (err: any, def = '') => {
const msg: string = typeof err === 'string' ? err : err?.message ?? def;
msg && console.log('error =>', msg);
return replaceSensitiveText(msg);
};

View File

@@ -0,0 +1,38 @@
import { RunCodeDto } from 'src/sandbox/dto/create-sandbox.dto';
import { parentPort } from 'worker_threads';
import { workerResponse } from './utils';
// @ts-ignore
const ivm = require('isolated-vm');
parentPort?.on('message', ({ code, variables = {} }: RunCodeDto) => {
const resolve = (data: any) => workerResponse({ parentPort, type: 'success', data });
const reject = (error: any) => workerResponse({ parentPort, type: 'error', data: error });
const isolate = new ivm.Isolate({ memoryLimit: 32 });
const context = isolate.createContextSync();
const jail = context.global;
// custom log function
jail.setSync('responseData', function (args: any): any {
if (typeof args === 'object') {
resolve(args);
} else {
reject('Not an invalid response');
}
});
// Add global variables
jail.setSync('variables', new ivm.ExternalCopy(variables).copyInto());
try {
const scriptCode = `
${code}
responseData(main(variables))`;
context.evalSync(scriptCode, { timeout: 6000 });
} catch (err) {
reject(err);
}
process.exit();
});

View File

@@ -0,0 +1,47 @@
import { type MessagePort, Worker } from 'worker_threads';
import * as path from 'path';
export enum WorkerNameEnum {
runJs = 'runJs',
runPy = 'runPy'
}
type WorkerResponseType = { type: 'success' | 'error'; data: any };
export const getWorker = (name: WorkerNameEnum) => {
const baseUrl =
process.env.NODE_ENV === 'production' ? 'projects/sandbox/dist/worker' : 'dist/worker';
const workerPath = path.join(process.cwd(), baseUrl, `${name}.js`);
return new Worker(workerPath);
};
export const runWorker = <T = any>(name: WorkerNameEnum, params?: Record<string, any>) => {
return new Promise<T>((resolve, reject) => {
const worker = getWorker(name);
worker.postMessage(params);
worker.on('message', (msg: WorkerResponseType) => {
if (msg.type === 'error') return reject(msg.data);
resolve(msg.data);
worker.terminate();
});
worker.on('error', (err) => {
reject(err);
worker.terminate();
});
worker.on('messageerror', (err) => {
reject(err);
worker.terminate();
});
});
};
export const workerResponse = ({
parentPort,
...data
}: WorkerResponseType & { parentPort?: MessagePort }) => {
parentPort?.postMessage(data);
};

View File

@@ -0,0 +1,21 @@
import { Test, TestingModule } from '@nestjs/testing';
import { INestApplication } from '@nestjs/common';
import * as request from 'supertest';
import { AppModule } from './../src/app.module';
describe('AppController (e2e)', () => {
let app: INestApplication;
beforeEach(async () => {
const moduleFixture: TestingModule = await Test.createTestingModule({
imports: [AppModule]
}).compile();
app = moduleFixture.createNestApplication();
await app.init();
});
it('/ (GET)', () => {
return request(app.getHttpServer()).get('/').expect(200).expect('Hello World!');
});
});

View File

@@ -0,0 +1,9 @@
{
"moduleFileExtensions": ["js", "json", "ts"],
"rootDir": ".",
"testEnvironment": "node",
"testRegex": ".e2e-spec.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
}
}

View File

@@ -0,0 +1,4 @@
{
"extends": "./tsconfig.json",
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
}

View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"module": "commonjs",
"declaration": true,
"removeComments": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"allowSyntheticDefaultImports": true,
"target": "ES2021",
"sourceMap": true,
"outDir": "./dist",
"baseUrl": "./",
"incremental": true,
"skipLibCheck": true,
"strictNullChecks": false,
"noImplicitAny": false,
"strictBindCallApply": false,
"forceConsistentCasingInFileNames": false,
"noFallthroughCasesInSwitch": false
}
}