Yuque dataset baseurl (#4742)

* Yuque dataset baseurl (#4512)

* feat: 增加API数据集功能和国际化支持

* 在apiDataset.d.ts中添加uuid、slug、parent_uuid和children字段
* 更新英文、简体中文和繁体中文的dataset.json文件,增加多条提示信息
* 在ApiDatasetForm组件中实现目录选择功能,支持获取Yuque路径
* 新增BaseUrlSelector组件,用于选择根目录
* 实现getpath API,支持根据Yuque服务器获取路径
* 更新相关API调用,确保兼容性和功能完整性

* feat: 更新Yuque服务器的baseUrl处理逻辑

* 在apiDataset.d.ts中将YuqueServer的baseUrl属性改为必填
* 更新ApiDatasetForm组件,调整baseUrl的状态管理和路径加载逻辑
* 新增getcatalog API以支持获取数据集目录
* 修改相关API调用,确保baseUrl的正确传递和使用
* 优化路径返回值为中文“根目录”

* feat: 更新数据集API调用逻辑

* 将getFeishuAndYuqueDatasetFileList替换为getProApiDatasetFileListRequest,统一API调用方式
* 更新相关文件以确保新API的正确使用
* 优化代码结构,提高可读性和维护性

* 清理代码:移除ApiDatasetForm、BaseUrlSelector和相关API中的调试日志

* 删除不必要的console.log语句,提升代码整洁性
* 确保API逻辑的清晰性,避免冗余输出

* 更新数据集相关类型和API逻辑

- 在apiDataset.d.ts中添加ApiDatasetDetailResponse类型,移除不必要的字段。
- 在proApi.ts中新增DETAIL操作类型及相关参数类型。
- 修改ApiDatasetForm.tsx以支持新的API调用逻辑,统一路径获取方式。
- 更新BaseUrlSelector组件,简化目录选择逻辑。
- 优化getpath.ts和getcatalog.ts中的路径处理逻辑,确保API调用的一致性和正确性。
- 清理不必要的代码和注释,提高代码可读性。

* 清理ApiDatasetForm组件中的调试日志,移除console.log语句以提升代码整洁性和可读性。

* fix

* updata apidatasetform

* remove console

* updata

* updata

* updata editapiservermodal

* updata i18n

* add type

* update getpath

* add type

* perf: yuque dataset baseurl

* perf: remove rerank records

* fix: ts

---------

Co-authored-by: dreamer6680 <1468683855@qq.com>
This commit is contained in:
Archer
2025-05-05 18:37:14 +08:00
committed by GitHub
parent 864eff47c7
commit a6fbfac96f
21 changed files with 640 additions and 98 deletions

View File

@@ -1,6 +1,6 @@
import React from 'react';
import React, { useState, useMemo } from 'react';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { Flex, Input } from '@chakra-ui/react';
import { Flex, Input, Button, ModalBody, ModalFooter, Box } from '@chakra-ui/react';
import { UseFormReturn } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import type {
@@ -8,12 +8,28 @@ import type {
FeishuServer,
YuqueServer
} from '@fastgpt/global/core/dataset/apiDataset';
import { getApiDatasetPaths, getApiDatasetCatalog } from '@/web/core/dataset/api';
import type {
GetResourceFolderListItemResponse,
GetResourceFolderListProps,
ParentIdType
} from '@fastgpt/global/common/parentFolder/type';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import type { GetApiDatasetCataLogProps } from '@/pages/api/core/dataset/apiDataset/getCatalog';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useBoolean, useMemoizedFn, useMount } from 'ahooks';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { FolderIcon } from '@fastgpt/global/common/file/image/constants';
const ApiDatasetForm = ({
type,
datasetId,
form
}: {
type: `${DatasetTypeEnum}`;
datasetId?: string;
form: UseFormReturn<
{
apiServer?: APIFileServer;
@@ -24,22 +40,151 @@ const ApiDatasetForm = ({
>;
}) => {
const { t } = useTranslation();
const { register } = form;
const { register, setValue, watch } = form;
const yuqueServer = watch('yuqueServer');
const feishuServer = watch('feishuServer');
const apiServer = watch('apiServer');
const [pathNames, setPathNames] = useState(t('dataset:rootdirectory'));
const [
isOpenBaseurlSeletModal,
{ setTrue: openBaseurlSeletModal, setFalse: closeBaseurlSelectModal }
] = useBoolean();
const parentId = yuqueServer?.basePath || feishuServer?.folderToken || apiServer?.basePath;
const canSelectBaseUrl = useMemo(() => {
switch (type) {
case DatasetTypeEnum.yuque:
return yuqueServer?.userId && yuqueServer?.token;
case DatasetTypeEnum.feishu:
return feishuServer?.appId && feishuServer?.appSecret;
case DatasetTypeEnum.apiDataset:
return !!apiServer?.basePath;
default:
return false;
}
}, [
type,
yuqueServer?.token,
yuqueServer?.userId,
feishuServer?.appId,
feishuServer?.appSecret,
apiServer?.basePath
]);
// Unified function to get the current path
const { loading: isFetching } = useRequest2(
async () => {
if (!datasetId && !(yuqueServer?.userId && yuqueServer?.token)) {
return setPathNames(t('dataset:input_required_field_to_select_baseurl'));
}
if (!parentId) {
return setPathNames(t('dataset:rootdirectory'));
}
const path = await getApiDatasetPaths({
datasetId,
parentId,
yuqueServer,
feishuServer,
apiServer
});
setPathNames(path);
},
{
manual: false,
refreshDeps: [datasetId, parentId]
}
);
// Unified handling of directory selection
const onSelectBaseUrl = async (id: ParentIdType) => {
const value = id === 'root' || id === null || id === undefined ? '' : id;
switch (type) {
case DatasetTypeEnum.yuque:
setValue('yuqueServer.basePath', value);
break;
case DatasetTypeEnum.feishu:
setValue('feishuServer.folderToken', value);
break;
case DatasetTypeEnum.apiDataset:
setValue('apiServer.basePath', value);
break;
}
closeBaseurlSelectModal();
};
const renderBaseUrlSelector = () => (
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'}>
Base URL
</FormLabel>
<MyBox py={1} fontSize={'sm'} flex={'1 0 0'} overflow="auto" isLoading={isFetching}>
{pathNames}
</MyBox>
<Button
ml={2}
variant={'whiteBase'}
onClick={openBaseurlSeletModal}
isDisabled={!canSelectBaseUrl}
>
{t('dataset:selectDirectory')}
</Button>
</Flex>
);
// Render the directory selection modal
const renderDirectoryModal = () =>
isOpenBaseurlSeletModal ? (
<BaseUrlSelector
selectId={type === DatasetTypeEnum.yuque ? yuqueServer?.basePath || 'root' : 'root'}
server={async (e: GetResourceFolderListProps) => {
const params: GetApiDatasetCataLogProps = { parentId: e.parentId };
switch (type) {
case DatasetTypeEnum.yuque:
params.yuqueServer = {
userId: yuqueServer?.userId || '',
token: yuqueServer?.token || '',
basePath: ''
};
break;
// Currently, only Yuque is using it
case DatasetTypeEnum.feishu:
params.feishuServer = {
appId: feishuServer?.appId || '',
appSecret: feishuServer?.appSecret || '',
folderToken: feishuServer?.folderToken || ''
};
break;
case DatasetTypeEnum.apiDataset:
params.apiServer = {
baseUrl: apiServer?.baseUrl || '',
authorization: apiServer?.authorization || '',
basePath: ''
};
break;
}
return getApiDatasetCatalog(params);
}}
onConfirm={onSelectBaseUrl}
onClose={closeBaseurlSelectModal}
/>
) : null;
return (
<>
{type === DatasetTypeEnum.apiDataset && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
{t('dataset:api_url')}
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
@@ -47,16 +192,10 @@ const ApiDatasetForm = ({
{...register('apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
Authorization
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
@@ -64,6 +203,8 @@ const ApiDatasetForm = ({
{...register('apiServer.authorization')}
/>
</Flex>
{/* {renderBaseUrlSelector()}
{renderDirectoryModal()} */}
</>
)}
{type === DatasetTypeEnum.feishu && (
@@ -119,20 +260,16 @@ const ApiDatasetForm = ({
{...register('feishuServer.folderToken', { required: true })}
/>
</Flex>
{/* {renderBaseUrlSelector()}
{renderDirectoryModal()} */}
</>
)}
{type === DatasetTypeEnum.yuque && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
User ID
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={'User ID'}
@@ -140,16 +277,10 @@ const ApiDatasetForm = ({
{...register('yuqueServer.userId', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
Token
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={'Token'}
@@ -157,6 +288,8 @@ const ApiDatasetForm = ({
{...register('yuqueServer.token', { required: true })}
/>
</Flex>
{renderBaseUrlSelector()}
{renderDirectoryModal()}
</>
)}
</>
@@ -164,3 +297,165 @@ const ApiDatasetForm = ({
};
export default ApiDatasetForm;
type FolderItemType = {
id: string;
name: string;
open: boolean;
children?: FolderItemType[];
};
const rootId = 'root';
type Props = {
selectId: string;
server: (e: GetResourceFolderListProps) => Promise<GetResourceFolderListItemResponse[]>;
onConfirm: (id: ParentIdType) => Promise<any>;
onClose: () => void;
};
const BaseUrlSelector = ({ selectId, server, onConfirm, onClose }: Props) => {
const { t } = useTranslation();
const [selectedId, setSelectedId] = React.useState<string>(selectId);
const [requestingIdList, setRequestingIdList] = useState<ParentIdType[]>([]);
const [folderList, setFolderList] = useState<FolderItemType[]>([]);
const { runAsync: requestServer } = useRequest2(async (e: GetResourceFolderListProps) => {
if (requestingIdList.includes(e.parentId)) return Promise.reject(null);
setRequestingIdList((state) => [...state, e.parentId]);
return server(e).finally(() =>
setRequestingIdList((state) => state.filter((id) => id !== e.parentId))
);
}, {});
// Initialize the folder list
useMount(async () => {
const data = await requestServer({ parentId: null });
setFolderList([
{
id: rootId,
name: t('common:root_folder'),
open: true,
children: data.map((item) => ({
id: item.id,
name: item.name,
open: false
}))
}
]);
});
const RenderList = useMemoizedFn(
({ list, index = 0 }: { list: FolderItemType[]; index?: number }) => {
return (
<>
{list.map((item) => (
<Box key={item.id} _notLast={{ mb: 0.5 }} userSelect={'none'}>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={1}
pl={index === 0 ? '0.5rem' : `${1.75 * (index - 1) + 0.5}rem`}
pr={2}
borderRadius={'md'}
_hover={{
bg: 'myGray.100'
}}
{...(item.id === selectedId
? {
bg: 'primary.50 !important',
onClick: () => setSelectedId('')
}
: {
onClick: () => setSelectedId(item.id)
})}
>
{index !== 0 && (
<Flex
alignItems={'center'}
justifyContent={'center'}
visibility={!item.children || item.children.length > 0 ? 'visible' : 'hidden'}
w={'1.25rem'}
h={'1.25rem'}
cursor={'pointer'}
borderRadius={'xs'}
_hover={{
bg: 'rgba(31, 35, 41, 0.08)'
}}
onClick={async (e) => {
e.stopPropagation();
if (requestingIdList.includes(item.id)) return;
if (!item.children) {
const data = await requestServer({ parentId: item.id });
item.children = data.map((item) => ({
id: item.id,
name: item.name,
open: false
}));
}
item.open = !item.open;
setFolderList([...folderList]);
}}
>
<MyIcon
name={
requestingIdList.includes(item.id)
? 'common/loading'
: 'common/rightArrowFill'
}
w={'1.25rem'}
color={'myGray.500'}
transform={item.open ? 'rotate(90deg)' : 'none'}
/>
</Flex>
)}
<MyIcon ml={index !== 0 ? '0.5rem' : 0} name={FolderIcon} w={'1.25rem'} />
<Box fontSize={'sm'} ml={2}>
{item.name}
</Box>
</Flex>
{item.children && item.open && (
<Box mt={0.5}>
<RenderList list={item.children} index={index + 1} />
</Box>
)}
</Box>
))}
</>
);
}
);
const { runAsync: onConfirmSelect, loading: confirming } = useRequest2(
() => {
if (selectedId) {
return onConfirm(selectedId === rootId ? null : selectedId);
}
return Promise.reject('');
},
{
onSuccess: () => {
onClose();
}
}
);
return (
<MyModal
isLoading={folderList.length === 0}
iconSrc="/imgs/modal/move.svg"
isOpen
w={'30rem'}
title={t('dataset:selectRootFolder')}
onClose={onClose}
>
<ModalBody flex={'1 0 0'} overflow={'auto'} minH={'400px'}>
<RenderList list={folderList} />
</ModalBody>
<ModalFooter>
<Button isLoading={confirming} isDisabled={!selectedId} onClick={onConfirmSelect}>
{t('common:Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};