ChatGPT/FastGPT
1
0
Fork 0
mirror of https://github.com/labring/FastGPT.git synced 2025-08-03 13:38:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.6.7-alpha commit (#743)

Browse Source 
Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-01-19 11:17:28 +08:00
committed by GitHub
parent 8ee7407c4c
commit c031e6dcc9
324 changed files with 8509 additions and 4757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
projects/app/src/web/core/app/templates.ts
View File

@@ -10,13 +10,13 @@ export const appTemplates: (AppItemType & {
{
id: 'simpleChat',
avatar: '/imgs/module/AI.png',
name: '简单的对话',
intro: '一个极其简单的 AI 对话应用',
name: 'core.app.template.Simple chat',
intro: 'core.app.template.Simple chat desc',
type: AppTypeEnum.simple,
modules: [
{
moduleId: 'userGuide',
name: '用户引导',
name: 'core.module.template.User guide',
avatar: '/imgs/module/userGuide.png',
flowType: 'userGuide',
position: {
@@ -28,7 +28,7 @@ export const appTemplates: (AppItemType & {
key: 'welcomeText',
type: 'hidden',
valueType: 'string',
label: '开场白',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -37,7 +37,7 @@ export const appTemplates: (AppItemType & {
key: 'variables',
type: 'hidden',
valueType: 'any',
label: '对话框变量',
label: '',
value: [],
showTargetInApp: false,
showTargetInPlugin: false,
@@ -47,7 +47,7 @@ export const appTemplates: (AppItemType & {
key: 'questionGuide',
valueType: 'boolean',
type: 'switch',
label: '问题引导',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -56,7 +56,7 @@ export const appTemplates: (AppItemType & {
key: 'tts',
type: 'hidden',
valueType: 'any',
label: '语音播报',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -66,7 +66,7 @@ export const appTemplates: (AppItemType & {
},
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -78,7 +78,7 @@ export const appTemplates: (AppItemType & {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -87,7 +87,7 @@ export const appTemplates: (AppItemType & {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -122,7 +122,7 @@ export const appTemplates: (AppItemType & {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -216,7 +216,7 @@ export const appTemplates: (AppItemType & {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -293,13 +293,13 @@ export const appTemplates: (AppItemType & {
{
id: 'simpleDatasetChat',
avatar: '/imgs/module/db.png',
name: '知识库 + 对话引导',
intro: '每次提问时进行一次知识库搜索，将搜索结果注入 LLM 模型进行参考回答',
name: 'core.app.template.Dataset and guide',
intro: 'core.app.template.Dataset and guide desc',
type: AppTypeEnum.simple,
modules: [
{
moduleId: 'userGuide',
name: '用户引导',
name: 'core.module.template.User guide',
avatar: '/imgs/module/userGuide.png',
flowType: 'userGuide',
position: {
@@ -311,7 +311,7 @@ export const appTemplates: (AppItemType & {
key: 'welcomeText',
type: 'hidden',
valueType: 'string',
label: '开场白',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
value: '你好，我是知识库助手，请不要忘记选择知识库噢~\n[你是谁]\n[如何使用]',
@@ -321,7 +321,7 @@ export const appTemplates: (AppItemType & {
key: 'variables',
type: 'hidden',
valueType: 'any',
label: '对话框变量',
label: '',
value: [],
showTargetInApp: false,
showTargetInPlugin: false,
@@ -331,7 +331,7 @@ export const appTemplates: (AppItemType & {
key: 'questionGuide',
valueType: 'boolean',
type: 'switch',
label: '问题引导',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
value: false,
@@ -341,7 +341,7 @@ export const appTemplates: (AppItemType & {
key: 'tts',
type: 'hidden',
valueType: 'any',
label: '语音播报',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
value: {
@@ -354,7 +354,7 @@ export const appTemplates: (AppItemType & {
},
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -366,7 +366,7 @@ export const appTemplates: (AppItemType & {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -375,7 +375,7 @@ export const appTemplates: (AppItemType & {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -393,7 +393,7 @@ export const appTemplates: (AppItemType & {
},
{
moduleId: 'datasetSearch',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -549,7 +549,7 @@ export const appTemplates: (AppItemType & {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -645,7 +645,7 @@ export const appTemplates: (AppItemType & {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -723,13 +723,13 @@ export const appTemplates: (AppItemType & {
{
id: 'chatGuide',
avatar: '/imgs/module/userGuide.png',
name: '对话引导 + 变量',
intro: '可以在对话开始发送一段提示，或者让用户填写一些内容，作为本次对话的变量',
name: 'core.app.template.Guide and variables',
intro: 'core.app.template.Guide and variables desc',
type: AppTypeEnum.simple,
modules: [
{
moduleId: 'userGuide',
name: '用户引导',
name: 'core.module.template.User guide',
avatar: '/imgs/module/userGuide.png',
flowType: 'userGuide',
position: {
@@ -741,7 +741,7 @@ export const appTemplates: (AppItemType & {
key: 'welcomeText',
type: 'hidden',
valueType: 'string',
label: '开场白',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
value: '你好，我可以为你翻译各种语言，请告诉我你需要翻译成什么语言？',
@@ -751,7 +751,7 @@ export const appTemplates: (AppItemType & {
key: 'variables',
type: 'hidden',
valueType: 'any',
label: '对话框变量',
label: '',
value: [
{
id: '35c640eb-cf22-431f-bb57-3fc21643880e',
@@ -791,7 +791,7 @@ export const appTemplates: (AppItemType & {
key: 'questionGuide',
valueType: 'boolean',
type: 'switch',
label: '问题引导',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
value: false,
@@ -801,7 +801,7 @@ export const appTemplates: (AppItemType & {
key: 'tts',
type: 'hidden',
valueType: 'any',
label: '语音播报',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -811,7 +811,7 @@ export const appTemplates: (AppItemType & {
},
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -823,7 +823,7 @@ export const appTemplates: (AppItemType & {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -832,7 +832,7 @@ export const appTemplates: (AppItemType & {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -867,7 +867,7 @@ export const appTemplates: (AppItemType & {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -961,7 +961,7 @@ export const appTemplates: (AppItemType & {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -1039,13 +1039,13 @@ export const appTemplates: (AppItemType & {
{
id: 'CQ',
avatar: '/imgs/module/cq.png',
name: '问题分类 + 知识库',
intro: '先对用户的问题进行分类，再根据不同类型问题，执行不同的操作',
name: 'core.app.template.Classify and dataset',
intro: 'core.app.template.Classify and dataset desc',
type: AppTypeEnum.advanced,
modules: [
{
moduleId: '7z5g5h',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -1057,7 +1057,7 @@ export const appTemplates: (AppItemType & {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -1066,7 +1066,7 @@ export const appTemplates: (AppItemType & {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -1325,7 +1325,7 @@ export const appTemplates: (AppItemType & {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -1419,7 +1419,7 @@ export const appTemplates: (AppItemType & {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -1494,7 +1494,7 @@ export const appTemplates: (AppItemType & {
},
{
moduleId: 'fljhzy',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -1639,7 +1639,7 @@ export const appTemplates: (AppItemType & {
},
{
moduleId: 'q9equb',
name: '用户引导',
name: 'core.module.template.User guide',
avatar: '/imgs/module/userGuide.png',
flowType: 'userGuide',
position: {
@@ -1651,7 +1651,7 @@ export const appTemplates: (AppItemType & {
key: 'welcomeText',
type: 'hidden',
valueType: 'string',
label: '开场白',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
value:
@@ -1662,7 +1662,7 @@ export const appTemplates: (AppItemType & {
key: 'variables',
type: 'hidden',
valueType: 'any',
label: '对话框变量',
label: '',
value: [],
showTargetInApp: false,
showTargetInPlugin: false,
@@ -1672,7 +1672,7 @@ export const appTemplates: (AppItemType & {
key: 'questionGuide',
valueType: 'boolean',
type: 'switch',
label: '问题引导',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -1681,7 +1681,7 @@ export const appTemplates: (AppItemType & {
key: 'tts',
type: 'hidden',
valueType: 'any',
label: '语音播报',
label: '',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -1736,7 +1736,7 @@ export const appTemplates: (AppItemType & {
},
{
moduleId: '9act94',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -1748,7 +1748,7 @@ export const appTemplates: (AppItemType & {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -1757,7 +1757,7 @@ export const appTemplates: (AppItemType & {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [

2
projects/app/src/web/core/app/utils.ts
View File

@@ -13,7 +13,7 @@ export async function postForm2Modules(
function userGuideTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
return [
{
name: '用户引导',
name: 'core.module.template.User guide',
flowType: FlowNodeTypeEnum.userGuide,
inputs: [
{

6
projects/app/src/web/core/dataset/api.ts
View File

@@ -9,6 +9,7 @@ import type {
import type {
CreateDatasetCollectionParams,
DatasetUpdateBody,
LinkCreateDatasetCollectionParams,
PostWebsiteSyncParams
} from '@fastgpt/global/core/dataset/api.d';
import type {
@@ -28,7 +29,7 @@ import type { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/typ
import {
DatasetCollectionSyncResultEnum,
DatasetTypeEnum
} from '@fastgpt/global/core/dataset/constant';
} from '@fastgpt/global/core/dataset/constants';
import type { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import { PagingData } from '@/types';
@@ -72,6 +73,9 @@ export const getDatasetCollectionById = (id: string) =>
GET<DatasetCollectionItemType>(`/core/dataset/collection/detail`, { id });
export const postDatasetCollection = (data: CreateDatasetCollectionParams) =>
POST<string>(`/core/dataset/collection/create`, data);
export const postCreateDatasetLinkCollection = (data: LinkCreateDatasetCollectionParams) =>
POST<{ collectionId: string }>(`/core/dataset/collection/create/link`, data);
export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
POST(`/core/dataset/collection/update`, data);
export const delDatasetCollectionById = (params: { id: string }) =>

200
projects/app/src/web/core/dataset/components/FileSelector.tsx Normal file
View File

@@ -0,0 +1,200 @@
import MyBox from '@/components/common/MyBox';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useToast } from '@/web/common/hooks/useToast';
import { Box, FlexProps } from '@chakra-ui/react';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import React, { DragEvent, useCallback, useState } from 'react';
export type SelectFileItemType = {
folderPath: string;
file: File;
};
const FileSelector = ({
fileType,
multiple,
maxCount,
maxSize,
isLoading,
onSelectFile,
...props
}: {
fileType: string;
multiple?: boolean;
maxCount?: number;
maxSize?: number;
isLoading?: boolean;
onSelectFile: (e: SelectFileItemType[]) => any;
} & FlexProps) => {
const { t } = useTranslation();
const { toast } = useToast();
const { File, onOpen } = useSelectFile({
fileType,
multiple,
maxCount
});
const [isDragging, setIsDragging] = useState(false);
const filterTypeReg = new RegExp(
`(${fileType
.split(',')
.map((item) => item.trim())
.join('|')})$`,
'i'
);
const selectFileCallback = useCallback(
(files: SelectFileItemType[]) => {
// size check
if (!maxSize) {
return onSelectFile(files);
}
const filterFiles = files.filter((item) => item.file.size <= maxSize);
if (filterFiles.length < files.length) {
toast({
status: 'warning',
title: t('common.file.Some file size exceeds limit', { maxSize: formatFileSize(maxSize) })
});
}
return onSelectFile(filterFiles);
},
[maxSize, onSelectFile, t, toast]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(true);
};
const handleDragLeave = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
};
const handleDrop = async (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
const items = e.dataTransfer.items;
const fileList: SelectFileItemType[] = [];
if (e.dataTransfer.items.length <= 1) {
const traverseFileTree = async (item: any) => {
return new Promise<void>((resolve, reject) => {
if (item.isFile) {
item.file((file: File) => {
const folderPath = (item.fullPath || '').split('/').slice(2, -1).join('/');
if (filterTypeReg.test(file.name)) {
fileList.push({
folderPath,
file
});
}
resolve();
});
} else if (item.isDirectory) {
const dirReader = item.createReader();
dirReader.readEntries(async (entries: any[]) => {
for (let i = 0; i < entries.length; i++) {
await traverseFileTree(entries[i]);
}
resolve();
});
}
});
};
for await (const item of items) {
await traverseFileTree(item.webkitGetAsEntry());
}
} else {
const files = Array.from(e.dataTransfer.files);
let isErr = files.some((item) => item.type === '');
if (isErr) {
return toast({
title: t('file.upload error description'),
status: 'error'
});
}
fileList.push(
...files
.filter((item) => filterTypeReg.test(item.name))
.map((file) => ({
folderPath: '',
file
}))
);
}
selectFileCallback(fileList.slice(0, maxCount));
};
return (
<MyBox
isLoading={isLoading}
display={'flex'}
flexDirection={'column'}
alignItems={'center'}
justifyContent={'center'}
px={3}
py={[4, 7]}
borderWidth={'1.5px'}
borderStyle={'dashed'}
borderRadius={'md'}
cursor={'pointer'}
_hover={{
bg: 'primary.50',
borderColor: 'primary.600'
}}
{...(isDragging
? {
borderColor: 'primary.600'
}
: {
borderColor: 'borderColor.high'
})}
{...props}
onDragEnter={handleDragEnter}
onDragOver={(e) => e.preventDefault()}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
onClick={onOpen}
>
<MyIcon name={'common/uploadFileFill'} w={'32px'} />
<Box fontWeight={'bold'}>
{isDragging
? t('file.Release the mouse to upload the file')
: t('common.file.Select and drag file tip')}
</Box>
{/* file type */}
<Box color={'myGray.500'} fontSize={'xs'}>
{t('common.file.Support file type', { fileType })}
</Box>
<Box color={'myGray.500'} fontSize={'xs'}>
{/* max count */}
{maxCount && t('common.file.Support max count', { maxCount })}
{/* max size */}
{maxSize && t('common.file.Support max size', { maxSize: formatFileSize(maxSize) })}
</Box>
<File
onSelect={(files) =>
selectFileCallback(
files.map((file) => ({
folderPath: '',
file
}))
)
}
/>
</MyBox>
);
};
export default React.memo(FileSelector);

12
projects/app/src/web/core/dataset/components/SelectCollections.tsx
View File

@@ -16,7 +16,7 @@ import {
Image,
ModalBody
} from '@chakra-ui/react';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import { useQuery } from '@tanstack/react-query';
import React, { useMemo, useState } from 'react';
@@ -128,7 +128,7 @@ const SelectCollections = ({
{title
? title
: type === 'folder'
? t('common.Select One Folder')
? t('common.Root folder')
: t('dataset.collections.Select Collection')}
</Box>
{!!tip && (
@@ -150,7 +150,6 @@ const SelectCollections = ({
gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)']}
gridGap={3}
userSelect={'none'}
overflowY={'auto'}
mt={2}
>
{collections.map((item) =>
@@ -164,7 +163,8 @@ const SelectCollections = ({
boxShadow={'sm'}
cursor={'pointer'}
_hover={{
boxShadow: 'md'
bg: 'primary.50',
borderColor: 'primary.300'
}}
{...(selected
? {
@@ -189,7 +189,7 @@ const SelectCollections = ({
}}
>
<Flex alignItems={'center'} h={'38px'}>
<Image src={item.icon} w={'18px'} alt={''} />
<MyIcon name={item.icon as any} w={'18px'} />
<Box ml={3} fontSize={'sm'} className="textEllipsis">
{item.name}
</Box>
@@ -218,7 +218,7 @@ const SelectCollections = ({
isDisabled={type === 'collection' && selectedDatasetCollectionIds.length === 0}
onClick={mutate}
>
{type === 'folder' ? t('common.Confirm Move') : t('Confirm')}
{type === 'folder' ? t('common.Confirm Move') : t('common.Confirm')}
</Button>
</ModalFooter>
)}

4
projects/app/src/web/core/dataset/constants.ts Normal file
View File

@@ -0,0 +1,4 @@
export enum ImportProcessWayEnum {
auto = 'auto',
custom = 'custom'
}

5
projects/app/src/web/core/dataset/store/dataset.ts
View File

@@ -11,7 +11,7 @@ import {
} from '@/web/core/dataset/api';
import { defaultDatasetDetail } from '@/constants/dataset';
import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';
@@ -96,8 +96,7 @@ export const useDatasetStore = create<State>()(
}),
postCreateTrainingBill({
name: 'core.dataset.training.Website Sync',
vectorModel: get().datasetDetail.vectorModel.model,
agentModel: get().datasetDetail.agentModel.model
datasetId: get().datasetDetail._id
})
]);
try {

2
projects/app/src/web/core/dataset/store/searchTest.ts
View File

@@ -2,7 +2,7 @@ import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
export type SearchTestStoreItemType = {
id: string;

38
projects/app/src/web/core/dataset/type.d.ts vendored Normal file
View File

@@ -0,0 +1,38 @@
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { ImportProcessWayEnum } from './constants';
import { UseFormReturn } from 'react-hook-form';
export type ImportDataComponentProps = {
activeStep: number;
goToNext: () => void;
};
export type ImportSourceItemType = {
id: string;
rawText: string;
chunks: PushDatasetDataChunkProps[];
chunkChars: number;
sourceFolderPath?: string;
sourceName: string;
sourceSize?: string;
icon: string;
metadata?: Record<string, any>;
errorMsg?: string;
// source
file?: File;
link?: string;
};
export type ImportSourceParamsType = UseFormReturn<
{
chunkSize: number;
chunkOverlapRatio: number;
customSplitChar: string;
prompt: string;
mode: TrainingModeEnum;
way: ImportProcessWayEnum;
},
any
>;

52
projects/app/src/web/core/dataset/utils.ts
View File

@@ -1,8 +1,45 @@
import { getFileViewUrl, postChunks2Dataset } from '@/web/core/dataset/api';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { delay } from '@fastgpt/global/common/system/utils';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
import type {
FileCreateDatasetCollectionParams,
PushDatasetDataChunkProps
} from '@fastgpt/global/core/dataset/api.d';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { POST } from '@/web/common/api/request';
/* upload a file to create collection */
export const fileCollectionCreate = ({
file,
metadata = {},
data,
percentListen
}: {
file: File;
metadata?: Record<string, any>;
data: FileCreateDatasetCollectionParams;
percentListen: (percent: number) => void;
}) => {
const form = new FormData();
form.append('data', JSON.stringify(data));
form.append('metadata', JSON.stringify(metadata));
form.append('bucketName', BucketNameEnum.dataset);
form.append('file', file, encodeURIComponent(file.name));
return POST<string>(`/core/dataset/collection/create/file?datasetId=${data.datasetId}`, form, {
timeout: 480000,
onUploadProgress: (e) => {
if (!e.total) return;
const percent = Math.round((e.loaded / e.total) * 100);
percentListen && percentListen(percent);
},
headers: {
'Content-Type': 'multipart/form-data; charset=utf-8'
}
});
};
export async function chunksUpload({
billId,
@@ -10,7 +47,7 @@ export async function chunksUpload({
trainingMode,
chunks,
prompt,
rate = 150,
rate = 50,
onUploading
}: {
billId: string;
@@ -19,7 +56,7 @@ export async function chunksUpload({
chunks: PushDatasetDataChunkProps[];
prompt?: string;
rate?: number;
onUploading?: (insertLen: number, total: number) => void;
onUploading?: (rate: number) => void;
}) {
async function upload(data: PushDatasetDataChunkProps[]) {
return postChunks2Dataset({
@@ -41,8 +78,11 @@ export async function chunksUpload({
let retryTimes = 10;
for (let i = 0; i < chunks.length; i += rate) {
try {
const { insertLen } = await upload(chunks.slice(i, i + rate));
onUploading && onUploading(insertLen, chunks.length);
const uploadChunks = chunks.slice(i, i + rate);
const { insertLen } = await upload(uploadChunks);
if (onUploading) {
onUploading(Math.round(((i + uploadChunks.length) / chunks.length) * 100));
}
successInsert += insertLen;
} catch (error) {
if (retryTimes === 0) {

14
projects/app/src/web/core/modules/template/system.ts
View File

@@ -63,37 +63,37 @@ export const moduleTemplatesFlat: FlowModuleTemplateType[] = [
export const moduleTemplatesList: moduleTemplateListType = [
{
type: ModuleTemplateTypeEnum.userGuide,
label: '引导模块',
label: 'core.module.template.Guide module',
list: []
},
{
type: ModuleTemplateTypeEnum.systemInput,
label: '系统输入',
label: 'core.module.template.System input module',
list: []
},
{
type: ModuleTemplateTypeEnum.tools,
label: '工具',
label: 'core.module.template.Tool module',
list: []
},
{
type: ModuleTemplateTypeEnum.textAnswer,
label: '文本输出',
label: 'core.module.template.Response module',
list: []
},
{
type: ModuleTemplateTypeEnum.functionCall,
label: '功能调用',
label: 'core.module.template.Function module',
list: []
},
{
type: ModuleTemplateTypeEnum.externalCall,
label: '外部调用',
label: 'core.module.template.External module',
list: []
},
{
type: ModuleTemplateTypeEnum.personalPlugin,
label: '个人插件',
label: 'core.module.template.My plugin module',
list: []
},
{