Yuque dataset baseurl (#4742)

* Yuque dataset baseurl (#4512)

* feat: 增加API数据集功能和国际化支持

* 在apiDataset.d.ts中添加uuid、slug、parent_uuid和children字段
* 更新英文、简体中文和繁体中文的dataset.json文件,增加多条提示信息
* 在ApiDatasetForm组件中实现目录选择功能,支持获取Yuque路径
* 新增BaseUrlSelector组件,用于选择根目录
* 实现getpath API,支持根据Yuque服务器获取路径
* 更新相关API调用,确保兼容性和功能完整性

* feat: 更新Yuque服务器的baseUrl处理逻辑

* 在apiDataset.d.ts中将YuqueServer的baseUrl属性改为必填
* 更新ApiDatasetForm组件,调整baseUrl的状态管理和路径加载逻辑
* 新增getcatalog API以支持获取数据集目录
* 修改相关API调用,确保baseUrl的正确传递和使用
* 优化路径返回值为中文“根目录”

* feat: 更新数据集API调用逻辑

* 将getFeishuAndYuqueDatasetFileList替换为getProApiDatasetFileListRequest,统一API调用方式
* 更新相关文件以确保新API的正确使用
* 优化代码结构,提高可读性和维护性

* 清理代码:移除ApiDatasetForm、BaseUrlSelector和相关API中的调试日志

* 删除不必要的console.log语句,提升代码整洁性
* 确保API逻辑的清晰性,避免冗余输出

* 更新数据集相关类型和API逻辑

- 在apiDataset.d.ts中添加ApiDatasetDetailResponse类型,移除不必要的字段。
- 在proApi.ts中新增DETAIL操作类型及相关参数类型。
- 修改ApiDatasetForm.tsx以支持新的API调用逻辑,统一路径获取方式。
- 更新BaseUrlSelector组件,简化目录选择逻辑。
- 优化getpath.ts和getcatalog.ts中的路径处理逻辑,确保API调用的一致性和正确性。
- 清理不必要的代码和注释,提高代码可读性。

* 清理ApiDatasetForm组件中的调试日志,移除console.log语句以提升代码整洁性和可读性。

* fix

* updata apidatasetform

* remove console

* updata

* updata

* updata editapiservermodal

* updata i18n

* add type

* update getpath

* add type

* perf: yuque dataset baseurl

* perf: remove rerank records

* fix: ts

---------

Co-authored-by: dreamer6680 <1468683855@qq.com>
This commit is contained in:
Archer
2025-05-05 18:37:14 +08:00
committed by GitHub
parent 864eff47c7
commit a6fbfac96f
21 changed files with 640 additions and 98 deletions

View File

@@ -13,6 +13,7 @@ weight: 792
1. 支持 Toolcalls 并行执行。
2. 将所有内置任务,从非 stream 模式调整成 stream 模式,避免部分模型不支持非 stream 模式。如需覆盖,则可以在模型`额外 Body`参数中,强制指定`stream=false`
3. qwen3 模型预设
4. 语雀知识库支持设置根目录。
## ⚙️ 优化

View File

@@ -1,8 +1,9 @@
import { RequireOnlyOne } from '../../common/type/utils';
import type { ParentIdType } from '../../common/parentFolder/type.d';
export type APIFileItem = {
id: string;
parentId: string | null;
parentId: ParentIdType;
name: string;
type: 'file' | 'folder';
updateTime: Date;
@@ -10,10 +11,24 @@ export type APIFileItem = {
hasChild?: boolean;
};
// Api dataset config
export type APIFileServer = {
baseUrl: string;
authorization: string;
authorization?: string;
basePath?: string;
};
export type FeishuServer = {
appId: string;
appSecret?: string;
folderToken: string;
};
export type YuqueServer = {
userId: string;
token?: string;
basePath?: string;
};
// Api dataset api
export type APIFileListResponse = APIFileItem[];
@@ -26,13 +41,8 @@ export type APIFileReadResponse = {
url: string;
};
export type FeishuServer = {
appId: string;
appSecret: string;
folderToken: string;
};
export type YuqueServer = {
userId: string;
token: string;
export type ApiDatasetDetailResponse = {
id: string;
name: string;
parentId: ParentIdType;
};

View File

@@ -1,4 +1,8 @@
import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
import {
ApiDatasetDetailResponse,
FeishuServer,
YuqueServer
} from '@fastgpt/global/core/dataset/apiDataset';
import {
DeepRagSearchProps,
SearchDatasetDataResponse
@@ -7,6 +11,7 @@ import { AuthOpenApiLimitProps } from '../../support/openapi/auth';
import { CreateUsageProps, ConcatUsageProps } from '@fastgpt/global/support/wallet/usage/api';
import {
GetProApiDatasetFileContentParams,
GetProApiDatasetFileDetailParams,
GetProApiDatasetFileListParams,
GetProApiDatasetFilePreviewUrlParams
} from '../../core/dataset/apiDataset/proApi';
@@ -26,4 +31,7 @@ declare global {
var getProApiDatasetFilePreviewUrl: (
data: GetProApiDatasetFilePreviewUrlParams
) => Promise<string>;
var getProApiDatasetFileDetail: (
data: GetProApiDatasetFileDetailParams
) => Promise<ApiDatasetDetailResponse>;
}

View File

@@ -4,7 +4,8 @@ import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDatas
export enum ProApiDatasetOperationTypeEnum {
LIST = 'list',
READ = 'read',
CONTENT = 'content'
CONTENT = 'content',
DETAIL = 'detail'
}
export type ProApiDatasetCommonParams = {
@@ -23,3 +24,7 @@ export type GetProApiDatasetFileContentParams = ProApiDatasetCommonParams & {
export type GetProApiDatasetFilePreviewUrlParams = ProApiDatasetCommonParams & {
apiFileId: string;
};
export type GetProApiDatasetFileDetailParams = ProApiDatasetCommonParams & {
apiFileId: string;
};

View File

@@ -9,8 +9,6 @@ import { deleteDatasetDataVector } from '../../common/vectorDB/controller';
import { MongoDatasetDataText } from './data/dataTextSchema';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { removeWebsiteSyncJobScheduler } from './websiteSync';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
/* ============= dataset ========== */
/* find all datasetId by top datasetId */

View File

@@ -109,15 +109,9 @@ const DatasetSchema = new Schema({
type: Boolean,
default: true
},
apiServer: {
type: Object
},
feishuServer: {
type: Object
},
yuqueServer: {
type: Object
},
apiServer: Object,
feishuServer: Object,
yuqueServer: Object,
// abandoned
autoSync: Boolean,

View File

@@ -181,12 +181,14 @@ export async function dispatchDatasetSearch(
inputTokens: reRankInputTokens,
modelType: ModelTypeEnum.rerank
});
nodeDispatchUsages.push({
totalPoints: reRankTotalPoints,
moduleName: node.name,
model: reRankModelName,
inputTokens: reRankInputTokens
});
if (usingReRank) {
nodeDispatchUsages.push({
totalPoints: reRankTotalPoints,
moduleName: node.name,
model: reRankModelName,
inputTokens: reRankInputTokens
});
}
// Query extension
(() => {
if (queryExtensionResult) {

View File

@@ -3,6 +3,7 @@
"add_file": "Import",
"api_file": "API Dataset",
"api_url": "API Url",
"apidataset_configuration": "Configuration information",
"auto_indexes": "Automatically generate supplementary indexes",
"auto_indexes_tips": "Additional index generation is performed through large models to improve semantic richness and improve retrieval accuracy.",
"auto_training_queue": "Enhanced index queueing",
@@ -61,6 +62,8 @@
"external_read_url": "External Preview URL",
"external_read_url_tip": "Configure the reading URL of your file library for user authentication. Use the {{fileId}} variable to refer to the external file ID.",
"external_url": "File Access URL",
"failedToLoadRootDirectories": "Failed to load root directories",
"failedToLoadSubDirectories": "Failed to load subdirectories",
"feishu_dataset": "Feishu Dataset",
"feishu_dataset_config": "Feishu Dataset Config",
"feishu_dataset_desc": "Can build a dataset using Feishu documents by configuring permissions, without secondary storage",
@@ -68,6 +71,7 @@
"file_model_function_tip": "Enhances indexing and QA generation",
"filename": "Filename",
"folder_dataset": "Folder",
"getDirectoryFailed": "Get directory failed",
"image_auto_parse": "Automatic image indexing",
"image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
"image_training_queue": "Queue of image processing",
@@ -84,9 +88,15 @@
"import_select_link": "Enter link",
"index_size": "Index size",
"index_size_tips": "When vectorized, the system will automatically further segment the blocks according to this size.",
"input_required_field_to_select_baseurl": "Please enter the required information first",
"is_open_schedule": "Enable scheduled synchronization",
"keep_image": "Keep the picture",
"loading": "Loading...",
"move.hint": "After moving, the selected knowledge base/folder will inherit the permission settings of the new folder, and the original permission settings will become invalid.",
"noChildren": "No subdirectories",
"noSelectedFolder": "No selected folder",
"noSelectedId": "No selected ID",
"noValidId": "No valid ID",
"open_auto_sync": "After scheduled synchronization is turned on, the system will try to synchronize the collection from time to time every day. During the collection synchronization period, the collection data will not be searched.",
"params_config": "Config",
"params_setting": "Parameter settings",
@@ -96,6 +106,7 @@
"permission.des.manage": "Can manage the entire knowledge base data and information",
"permission.des.read": "View knowledge base content",
"permission.des.write": "Ability to add and change knowledge base content",
"pleaseFillUserIdAndToken": "Please fill in User ID and Token",
"preview_chunk": "Preview chunks",
"preview_chunk_empty": "File content is empty",
"preview_chunk_intro": "A total of {{total}} blocks, up to 10",
@@ -112,7 +123,11 @@
"request_headers": "Request headers, will automatically append 'Bearer '",
"retain_collection": "Adjust Training Parameters",
"retrain_task_submitted": "The retraining task has been submitted",
"rootDirectoryFormatError": "Root directory data format is incorrect",
"rootdirectory": "/rootdirectory",
"same_api_collection": "The same API set exists",
"selectDirectory": "Choose",
"selectRootFolder": "Select Root Folder",
"split_chunk_char": "Block by specified splitter",
"split_chunk_size": "Block by length",
"split_sign_break": "1 newline character",
@@ -128,10 +143,10 @@
"sync_collection_failed": "Synchronization collection error, please check whether the source file can be accessed normally",
"sync_schedule": "Timing synchronization",
"sync_schedule_tip": "Only existing collections will be synchronized. \nIncludes linked collections and all collections in the API knowledge base. \nThe system will poll for updates every day, and the specific update time cannot be determined.",
"tag.add_new": "add_new",
"tag.Add_new_tag": "add_new Tag",
"tag.Edit_tag": "Edit Tag",
"tag.add": "Create",
"tag.add_new": "add_new",
"tag.cancel": "Cancel",
"tag.delete_tag_confirm": "Confirm to delete the tag?",
"tag.manage": "Tagging",

View File

@@ -3,6 +3,7 @@
"add_file": "添加文件",
"api_file": "API 文件库",
"api_url": "接口地址",
"apidataset_configuration": "配置信息",
"auto_indexes": "自动生成补充索引",
"auto_indexes_tips": "通过大模型进行额外索引生成,提高语义丰富度,提高检索的精度。",
"auto_training_queue": "增强索引排队",
@@ -61,6 +62,8 @@
"external_read_url": "外部预览地址",
"external_read_url_tip": "可以配置你文件库的阅读地址。便于对用户进行阅读鉴权操作。目前可以使用 {{fileId}} 变量来指代外部文件 ID。",
"external_url": "文件访问 URL",
"failedToLoadRootDirectories": "加载根目录失败",
"failedToLoadSubDirectories": "加载子目录失败",
"feishu_dataset": "飞书知识库",
"feishu_dataset_config": "配置飞书知识库",
"feishu_dataset_desc": "可通过配置飞书文档权限,使用飞书文档构建知识库,文档不会进行二次存储",
@@ -68,6 +71,7 @@
"file_model_function_tip": "用于增强索引和 QA 生成",
"filename": "文件名",
"folder_dataset": "文件夹",
"getDirectoryFailed": "获取目录失败",
"image_auto_parse": "图片自动索引",
"image_auto_parse_tips": "调用 VLM 自动标注文档里的图片,并生成额外的检索索引",
"image_training_queue": "图片处理排队",
@@ -84,9 +88,15 @@
"import_select_link": "输入链接",
"index_size": "索引大小",
"index_size_tips": "向量化时内容的长度,系统会自动按该大小对分块进行进一步的分割。",
"input_required_field_to_select_baseurl": "请先输入必填信息",
"is_open_schedule": "启用定时同步",
"keep_image": "保留图片",
"loading": "加载中...",
"move.hint": "移动后,所选知识库/文件夹将继承新文件夹的权限设置,原先的权限设置失效。",
"noChildren": "无子目录",
"noSelectedFolder": "没有选择文件夹",
"noSelectedId": "没有选择 ID",
"noValidId": "没有有效的 ID",
"open_auto_sync": "开启定时同步后,系统将会每天不定时尝试同步集合,集合同步期间,会出现无法搜索到该集合数据现象。",
"params_config": "配置",
"params_setting": "参数设置",
@@ -96,6 +106,7 @@
"permission.des.manage": "可管理整个知识库数据和信息",
"permission.des.read": "可查看知识库内容",
"permission.des.write": "可增加和变更知识库内容",
"pleaseFillUserIdAndToken": "请填写 User ID 和 Token",
"preview_chunk": "分块预览",
"preview_chunk_empty": "文件内容为空",
"preview_chunk_intro": "共 {{total}} 个分块,最多展示 10 个",
@@ -113,7 +124,11 @@
"request_headers": "请求头参数,会自动补充 Bearer",
"retain_collection": "调整训练参数",
"retrain_task_submitted": "重新训练任务已提交",
"rootDirectoryFormatError": "根目录数据格式不正确",
"rootdirectory": "/根目录",
"same_api_collection": "存在相同的 API 集合",
"selectDirectory": "选择",
"selectRootFolder": "选择根目录",
"split_chunk_char": "按指定分割符分块",
"split_chunk_size": "按长度分块",
"split_sign_break": "1 个换行符",
@@ -129,10 +144,10 @@
"sync_collection_failed": "同步集合错误,请检查是否能正常访问源文件",
"sync_schedule": "定时同步",
"sync_schedule_tip": "仅会同步已存在的集合。包括链接集合以及 API 知识库里所有集合。系统会每天进行轮询更新,无法确定具体的更新时间。",
"tag.add_new": "新建",
"tag.Add_new_tag": "新建标签",
"tag.Edit_tag": "编辑标签",
"tag.add": "创建",
"tag.add_new": "新建",
"tag.cancel": "取消选择",
"tag.delete_tag_confirm": "确定删除标签?",
"tag.manage": "标签管理",

View File

@@ -3,6 +3,7 @@
"add_file": "新增文件",
"api_file": "API 檔案庫",
"api_url": "介面位址",
"apidataset_configuration": "配寘資訊",
"auto_indexes": "自動生成補充索引",
"auto_indexes_tips": "透過大模型進行額外索引生成,提高語義豐富度,提高檢索的精度。",
"auto_training_queue": "增強索引排隊",
@@ -61,6 +62,8 @@
"external_read_url": "外部預覽網址",
"external_read_url_tip": "可以設定您檔案庫的讀取網址,方便對使用者進行讀取權限驗證。目前可使用 {{fileId}} 變數來代表外部檔案識別碼。",
"external_url": "檔案存取網址",
"failedToLoadRootDirectories": "加載根目錄失敗",
"failedToLoadSubDirectories": "加載子目錄失敗",
"feishu_dataset": "飛書知識庫",
"feishu_dataset_config": "設定飛書知識庫",
"feishu_dataset_desc": "可透過設定飛書文件權限,使用飛書文件建構知識庫,文件不會進行二次儲存",
@@ -68,6 +71,7 @@
"file_model_function_tip": "用於增強索引和問答生成",
"filename": "檔案名稱",
"folder_dataset": "資料夾",
"getDirectoryFailed": "獲取目錄失敗",
"image_auto_parse": "圖片自動索引",
"image_auto_parse_tips": "呼叫 VLM 自動標註文件裡的圖片,並生成額外的檢索索引",
"image_training_queue": "圖片處理排隊",
@@ -84,9 +88,15 @@
"import_select_link": "輸入連結",
"index_size": "索引大小",
"index_size_tips": "向量化時內容的長度,系統會自動按該大小對分塊進行進一步的分割。",
"input_required_field_to_select_baseurl": "請先輸入必填信息",
"is_open_schedule": "啟用定時同步",
"keep_image": "保留圖片",
"loading": "加載中...",
"move.hint": "移動後,所選資料集/資料夾將繼承新資料夾的權限設定,原先的權限設定將失效。",
"noChildren": "無子目錄",
"noSelectedFolder": "沒有選擇文件夾",
"noSelectedId": "沒有選擇 ID",
"noValidId": "沒有有效的 ID",
"open_auto_sync": "開啟定時同步後,系統將每天不定時嘗試同步集合,集合同步期間,會出現無法搜尋到該集合資料現象。",
"params_config": "設定",
"params_setting": "參數設定",
@@ -96,6 +106,7 @@
"permission.des.manage": "可管理整個資料集的資料和資訊",
"permission.des.read": "可檢視資料集內容",
"permission.des.write": "可新增和變更資料集內容",
"pleaseFillUserIdAndToken": "請填寫 User ID 和 Token",
"preview_chunk": "分塊預覽",
"preview_chunk_empty": "文件內容為空",
"preview_chunk_intro": "共 {{total}} 個分塊,最多展示 10 個",
@@ -112,7 +123,11 @@
"request_headers": "請求頭",
"retain_collection": "調整訓練參數",
"retrain_task_submitted": "重新訓練任務已提交",
"rootDirectoryFormatError": "根目錄資料格式不正確",
"rootdirectory": "/根目錄",
"same_api_collection": "存在相同的 API 集合",
"selectDirectory": "選擇",
"selectRootFolder": "選擇根目錄",
"split_chunk_char": "按指定分割符分塊",
"split_chunk_size": "按長度分塊",
"split_sign_break": "1 個換行符",
@@ -128,10 +143,10 @@
"sync_collection_failed": "同步集合錯誤,請檢查是否能正常存取來原始檔",
"sync_schedule": "定時同步",
"sync_schedule_tip": "只會同步已存在的集合。\n包括連結集合以及 API 知識庫裡所有集合。\n系統會每天進行輪詢更新無法確定特定的更新時間。",
"tag.add_new": "新增",
"tag.Add_new_tag": "新增標籤",
"tag.Edit_tag": "編輯標籤",
"tag.add": "建立",
"tag.add_new": "新增",
"tag.cancel": "取消",
"tag.delete_tag_confirm": "確定要刪除標籤嗎?",
"tag.manage": "標籤管理",

View File

@@ -1,6 +1,6 @@
import React from 'react';
import React, { useState, useMemo } from 'react';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { Flex, Input } from '@chakra-ui/react';
import { Flex, Input, Button, ModalBody, ModalFooter, Box } from '@chakra-ui/react';
import { UseFormReturn } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import type {
@@ -8,12 +8,28 @@ import type {
FeishuServer,
YuqueServer
} from '@fastgpt/global/core/dataset/apiDataset';
import { getApiDatasetPaths, getApiDatasetCatalog } from '@/web/core/dataset/api';
import type {
GetResourceFolderListItemResponse,
GetResourceFolderListProps,
ParentIdType
} from '@fastgpt/global/common/parentFolder/type';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import type { GetApiDatasetCataLogProps } from '@/pages/api/core/dataset/apiDataset/getCatalog';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useBoolean, useMemoizedFn, useMount } from 'ahooks';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { FolderIcon } from '@fastgpt/global/common/file/image/constants';
const ApiDatasetForm = ({
type,
datasetId,
form
}: {
type: `${DatasetTypeEnum}`;
datasetId?: string;
form: UseFormReturn<
{
apiServer?: APIFileServer;
@@ -24,22 +40,151 @@ const ApiDatasetForm = ({
>;
}) => {
const { t } = useTranslation();
const { register } = form;
const { register, setValue, watch } = form;
const yuqueServer = watch('yuqueServer');
const feishuServer = watch('feishuServer');
const apiServer = watch('apiServer');
const [pathNames, setPathNames] = useState(t('dataset:rootdirectory'));
const [
isOpenBaseurlSeletModal,
{ setTrue: openBaseurlSeletModal, setFalse: closeBaseurlSelectModal }
] = useBoolean();
const parentId = yuqueServer?.basePath || feishuServer?.folderToken || apiServer?.basePath;
const canSelectBaseUrl = useMemo(() => {
switch (type) {
case DatasetTypeEnum.yuque:
return yuqueServer?.userId && yuqueServer?.token;
case DatasetTypeEnum.feishu:
return feishuServer?.appId && feishuServer?.appSecret;
case DatasetTypeEnum.apiDataset:
return !!apiServer?.basePath;
default:
return false;
}
}, [
type,
yuqueServer?.token,
yuqueServer?.userId,
feishuServer?.appId,
feishuServer?.appSecret,
apiServer?.basePath
]);
// Unified function to get the current path
const { loading: isFetching } = useRequest2(
async () => {
if (!datasetId && !(yuqueServer?.userId && yuqueServer?.token)) {
return setPathNames(t('dataset:input_required_field_to_select_baseurl'));
}
if (!parentId) {
return setPathNames(t('dataset:rootdirectory'));
}
const path = await getApiDatasetPaths({
datasetId,
parentId,
yuqueServer,
feishuServer,
apiServer
});
setPathNames(path);
},
{
manual: false,
refreshDeps: [datasetId, parentId]
}
);
// Unified handling of directory selection
const onSelectBaseUrl = async (id: ParentIdType) => {
const value = id === 'root' || id === null || id === undefined ? '' : id;
switch (type) {
case DatasetTypeEnum.yuque:
setValue('yuqueServer.basePath', value);
break;
case DatasetTypeEnum.feishu:
setValue('feishuServer.folderToken', value);
break;
case DatasetTypeEnum.apiDataset:
setValue('apiServer.basePath', value);
break;
}
closeBaseurlSelectModal();
};
const renderBaseUrlSelector = () => (
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'}>
Base URL
</FormLabel>
<MyBox py={1} fontSize={'sm'} flex={'1 0 0'} overflow="auto" isLoading={isFetching}>
{pathNames}
</MyBox>
<Button
ml={2}
variant={'whiteBase'}
onClick={openBaseurlSeletModal}
isDisabled={!canSelectBaseUrl}
>
{t('dataset:selectDirectory')}
</Button>
</Flex>
);
// Render the directory selection modal
const renderDirectoryModal = () =>
isOpenBaseurlSeletModal ? (
<BaseUrlSelector
selectId={type === DatasetTypeEnum.yuque ? yuqueServer?.basePath || 'root' : 'root'}
server={async (e: GetResourceFolderListProps) => {
const params: GetApiDatasetCataLogProps = { parentId: e.parentId };
switch (type) {
case DatasetTypeEnum.yuque:
params.yuqueServer = {
userId: yuqueServer?.userId || '',
token: yuqueServer?.token || '',
basePath: ''
};
break;
// Currently, only Yuque is using it
case DatasetTypeEnum.feishu:
params.feishuServer = {
appId: feishuServer?.appId || '',
appSecret: feishuServer?.appSecret || '',
folderToken: feishuServer?.folderToken || ''
};
break;
case DatasetTypeEnum.apiDataset:
params.apiServer = {
baseUrl: apiServer?.baseUrl || '',
authorization: apiServer?.authorization || '',
basePath: ''
};
break;
}
return getApiDatasetCatalog(params);
}}
onConfirm={onSelectBaseUrl}
onClose={closeBaseurlSelectModal}
/>
) : null;
return (
<>
{type === DatasetTypeEnum.apiDataset && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
{t('dataset:api_url')}
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
@@ -47,16 +192,10 @@ const ApiDatasetForm = ({
{...register('apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
Authorization
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
@@ -64,6 +203,8 @@ const ApiDatasetForm = ({
{...register('apiServer.authorization')}
/>
</Flex>
{/* {renderBaseUrlSelector()}
{renderDirectoryModal()} */}
</>
)}
{type === DatasetTypeEnum.feishu && (
@@ -119,20 +260,16 @@ const ApiDatasetForm = ({
{...register('feishuServer.folderToken', { required: true })}
/>
</Flex>
{/* {renderBaseUrlSelector()}
{renderDirectoryModal()} */}
</>
)}
{type === DatasetTypeEnum.yuque && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
User ID
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={'User ID'}
@@ -140,16 +277,10 @@ const ApiDatasetForm = ({
{...register('yuqueServer.userId', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
<Flex mt={6} alignItems={'center'}>
<FormLabel flex={['', '0 0 110px']} fontSize={'sm'} required>
Token
</Flex>
</FormLabel>
<Input
bg={'myWhite.600'}
placeholder={'Token'}
@@ -157,6 +288,8 @@ const ApiDatasetForm = ({
{...register('yuqueServer.token', { required: true })}
/>
</Flex>
{renderBaseUrlSelector()}
{renderDirectoryModal()}
</>
)}
</>
@@ -164,3 +297,165 @@ const ApiDatasetForm = ({
};
export default ApiDatasetForm;
type FolderItemType = {
id: string;
name: string;
open: boolean;
children?: FolderItemType[];
};
const rootId = 'root';
type Props = {
selectId: string;
server: (e: GetResourceFolderListProps) => Promise<GetResourceFolderListItemResponse[]>;
onConfirm: (id: ParentIdType) => Promise<any>;
onClose: () => void;
};
const BaseUrlSelector = ({ selectId, server, onConfirm, onClose }: Props) => {
const { t } = useTranslation();
const [selectedId, setSelectedId] = React.useState<string>(selectId);
const [requestingIdList, setRequestingIdList] = useState<ParentIdType[]>([]);
const [folderList, setFolderList] = useState<FolderItemType[]>([]);
const { runAsync: requestServer } = useRequest2(async (e: GetResourceFolderListProps) => {
if (requestingIdList.includes(e.parentId)) return Promise.reject(null);
setRequestingIdList((state) => [...state, e.parentId]);
return server(e).finally(() =>
setRequestingIdList((state) => state.filter((id) => id !== e.parentId))
);
}, {});
// Initialize the folder list
useMount(async () => {
const data = await requestServer({ parentId: null });
setFolderList([
{
id: rootId,
name: t('common:root_folder'),
open: true,
children: data.map((item) => ({
id: item.id,
name: item.name,
open: false
}))
}
]);
});
const RenderList = useMemoizedFn(
({ list, index = 0 }: { list: FolderItemType[]; index?: number }) => {
return (
<>
{list.map((item) => (
<Box key={item.id} _notLast={{ mb: 0.5 }} userSelect={'none'}>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={1}
pl={index === 0 ? '0.5rem' : `${1.75 * (index - 1) + 0.5}rem`}
pr={2}
borderRadius={'md'}
_hover={{
bg: 'myGray.100'
}}
{...(item.id === selectedId
? {
bg: 'primary.50 !important',
onClick: () => setSelectedId('')
}
: {
onClick: () => setSelectedId(item.id)
})}
>
{index !== 0 && (
<Flex
alignItems={'center'}
justifyContent={'center'}
visibility={!item.children || item.children.length > 0 ? 'visible' : 'hidden'}
w={'1.25rem'}
h={'1.25rem'}
cursor={'pointer'}
borderRadius={'xs'}
_hover={{
bg: 'rgba(31, 35, 41, 0.08)'
}}
onClick={async (e) => {
e.stopPropagation();
if (requestingIdList.includes(item.id)) return;
if (!item.children) {
const data = await requestServer({ parentId: item.id });
item.children = data.map((item) => ({
id: item.id,
name: item.name,
open: false
}));
}
item.open = !item.open;
setFolderList([...folderList]);
}}
>
<MyIcon
name={
requestingIdList.includes(item.id)
? 'common/loading'
: 'common/rightArrowFill'
}
w={'1.25rem'}
color={'myGray.500'}
transform={item.open ? 'rotate(90deg)' : 'none'}
/>
</Flex>
)}
<MyIcon ml={index !== 0 ? '0.5rem' : 0} name={FolderIcon} w={'1.25rem'} />
<Box fontSize={'sm'} ml={2}>
{item.name}
</Box>
</Flex>
{item.children && item.open && (
<Box mt={0.5}>
<RenderList list={item.children} index={index + 1} />
</Box>
)}
</Box>
))}
</>
);
}
);
const { runAsync: onConfirmSelect, loading: confirming } = useRequest2(
() => {
if (selectedId) {
return onConfirm(selectedId === rootId ? null : selectedId);
}
return Promise.reject('');
},
{
onSuccess: () => {
onClose();
}
}
);
return (
<MyModal
isLoading={folderList.length === 0}
iconSrc="/imgs/modal/move.svg"
isOpen
w={'30rem'}
title={t('dataset:selectRootFolder')}
onClose={onClose}
>
<ModalBody flex={'1 0 0'} overflow={'auto'} minH={'400px'}>
<RenderList list={folderList} />
</ModalBody>
<ModalFooter>
<Button isLoading={confirming} isDisabled={!selectedId} onClick={onConfirmSelect}>
{t('common:Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};

View File

@@ -1,5 +1,5 @@
import React from 'react';
import { ModalFooter, ModalBody, Button, Flex } from '@chakra-ui/react';
import { ModalFooter, ModalBody, Button, Flex, Box } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal/index';
import { useTranslation } from 'next-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
@@ -57,20 +57,25 @@ const EditAPIDatasetInfoModal = ({
<MyModal isOpen onClose={onClose} w={'450px'} iconSrc="modal/edit" title={title}>
<ModalBody>
{datasetTypeCourseMap[type] && (
<Flex
alignItems={'center'}
justifyContent={'flex-end'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
<Flex alignItems={'center'} justifyContent={'space-between'}>
<Box color={'myGray.900'} fontSize={'sm'} fontWeight={500}>
{t('dataset:apidataset_configuration')}
</Box>
<Flex
alignItems={'center'}
justifyContent={'flex-end'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
</Flex>
</Flex>
)}
{/* @ts-ignore */}
<ApiDatasetForm type={type} form={form} />
<ApiDatasetForm datasetId={datasetDetail._id} type={type} form={form} />
</ModalBody>
<ModalFooter>
<Button isLoading={loading} onClick={form.handleSubmit(onSave)} px={6}>

View File

@@ -40,8 +40,7 @@ const CreateModal = ({
}) => {
const { t } = useTranslation();
const router = useRouter();
const { feConfigs, defaultModels, embeddingModelList, datasetModelList, getVlmModelList } =
useSystemStore();
const { defaultModels, embeddingModelList, datasetModelList, getVlmModelList } = useSystemStore();
const { isPc } = useSystem();
const datasetTypeMap = useMemo(() => {
@@ -50,14 +49,14 @@ const CreateModal = ({
name: t('dataset:common_dataset'),
icon: 'core/dataset/commonDatasetColor'
},
[DatasetTypeEnum.apiDataset]: {
name: t('dataset:api_file'),
icon: 'core/dataset/externalDatasetColor'
},
[DatasetTypeEnum.websiteDataset]: {
name: t('dataset:website_dataset'),
icon: 'core/dataset/websiteDatasetColor'
},
[DatasetTypeEnum.apiDataset]: {
name: t('dataset:api_file'),
icon: 'core/dataset/externalDatasetColor'
},
[DatasetTypeEnum.feishu]: {
name: t('dataset:feishu_dataset'),
icon: 'core/dataset/feishuDatasetColor'

View File

@@ -0,0 +1,47 @@
import { getProApiDatasetFileListRequest } from '@/service/core/dataset/apiDataset/controller';
import { NextAPI } from '@/service/middleware/entry';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import type {
APIFileItem,
APIFileServer,
YuqueServer,
FeishuServer
} from '@fastgpt/global/core/dataset/apiDataset';
import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
import { NextApiRequest } from 'next';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
export type GetApiDatasetCataLogProps = {
searchKey?: string;
parentId?: ParentIdType;
yuqueServer?: YuqueServer;
feishuServer?: FeishuServer;
apiServer?: APIFileServer;
};
export type GetApiDatasetCataLogResponse = APIFileItem[];
async function handler(req: NextApiRequest) {
let { searchKey = '', parentId = null, yuqueServer, feishuServer, apiServer } = req.body;
await authCert({ req, authToken: true });
const data = await (async () => {
if (apiServer) {
return useApiDatasetRequest({ apiServer }).listFiles({ searchKey, parentId });
}
if (feishuServer || yuqueServer) {
return getProApiDatasetFileListRequest({
feishuServer,
yuqueServer,
parentId
});
}
return Promise.reject(DatasetErrEnum.noApiServer);
})();
return data.filter((item: APIFileItem) => item.hasChild === true);
}
export default NextAPI(handler);

View File

@@ -0,0 +1,99 @@
import { NextAPI } from '@/service/middleware/entry';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import type {
APIFileServer,
YuqueServer,
FeishuServer
} from '@fastgpt/global/core/dataset/apiDataset';
import { getProApiDatasetFileDetailRequest } from '@/service/core/dataset/apiDataset/controller';
import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant';
export type GetApiDatasetPathQuery = {};
export type GetApiDatasetPathBody = {
datasetId?: string;
parentId?: ParentIdType;
yuqueServer?: YuqueServer;
feishuServer?: FeishuServer;
apiServer?: APIFileServer;
};
export type GetApiDatasetPathResponse = string;
async function handler(
req: ApiRequestProps<GetApiDatasetPathBody, any>,
res: ApiResponseType<GetApiDatasetPathResponse>
): Promise<GetApiDatasetPathResponse> {
const { datasetId, parentId } = req.body;
if (!parentId) return '';
const { yuqueServer, feishuServer, apiServer } = await (async () => {
if (datasetId) {
const { dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: ManagePermissionVal,
datasetId
});
return {
yuqueServer: req.body.yuqueServer
? { ...req.body.yuqueServer, token: dataset.yuqueServer?.token ?? '' }
: dataset.yuqueServer,
feishuServer: req.body.feishuServer
? { ...req.body.feishuServer, appSecret: dataset.feishuServer?.appSecret ?? '' }
: dataset.feishuServer,
apiServer: req.body.apiServer
? {
...req.body.apiServer,
authorization: dataset.apiServer?.authorization ?? ''
}
: dataset.apiServer
};
} else {
await authCert({ req, authToken: true });
return {
yuqueServer: req.body.yuqueServer,
feishuServer: req.body.feishuServer,
apiServer: req.body.apiServer
};
}
})();
if (!apiServer && !feishuServer && !yuqueServer) {
return Promise.reject(DatasetErrEnum.noApiServer);
}
if (apiServer || feishuServer) {
return Promise.reject('不支持获取 BaseUrl');
}
if (yuqueServer) {
const getFullPath = async (currentId: string): Promise<string> => {
const response = await getProApiDatasetFileDetailRequest({
feishuServer,
yuqueServer,
apiFileId: currentId
});
if (response.parentId) {
const parentPath = await getFullPath(response.parentId);
return `${parentPath}/${response.name}`;
}
return `/${response.name}`;
};
return await getFullPath(parentId);
}
return Promise.reject(new Error(DatasetErrEnum.noApiServer));
}
export default NextAPI(handler);

View File

@@ -54,7 +54,8 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
yuqueServer: dataset.yuqueServer
? {
userId: dataset.yuqueServer.userId,
token: ''
token: '',
basePath: dataset.yuqueServer.basePath
}
: undefined,
feishuServer: dataset.feishuServer

View File

@@ -172,6 +172,9 @@ async function handler(
}),
...(!!yuqueServer?.userId && { 'yuqueServer.userId': yuqueServer.userId }),
...(!!yuqueServer?.token && { 'yuqueServer.token': yuqueServer.token }),
...(!!yuqueServer?.basePath !== undefined && {
'yuqueServer.basePath': yuqueServer?.basePath
}),
...(!!feishuServer?.appId && { 'feishuServer.appId': feishuServer.appId }),
...(!!feishuServer?.appSecret && { 'feishuServer.appSecret': feishuServer.appSecret }),
...(!!feishuServer?.folderToken && {

View File

@@ -20,6 +20,7 @@ import { AuthOpenApiLimitProps } from '@fastgpt/service/support/openapi/auth';
import { ConcatUsageProps, CreateUsageProps } from '@fastgpt/global/support/wallet/usage/api';
import {
getProApiDatasetFileContentRequest,
getProApiDatasetFileDetailRequest,
getProApiDatasetFileListRequest,
getProApiDatasetFilePreviewUrlRequest
} from '@/service/core/dataset/apiDataset/controller';
@@ -79,6 +80,7 @@ export function initGlobalVariables() {
global.getProApiDatasetFileList = getProApiDatasetFileListRequest;
global.getProApiDatasetFileContent = getProApiDatasetFileContentRequest;
global.getProApiDatasetFilePreviewUrl = getProApiDatasetFilePreviewUrlRequest;
global.getProApiDatasetFileDetail = getProApiDatasetFileDetailRequest;
}
global.communityPlugins = [];

View File

@@ -1,7 +1,12 @@
import { APIFileItem, ApiFileReadContentResponse } from '@fastgpt/global/core/dataset/apiDataset';
import type {
APIFileItem,
ApiFileReadContentResponse,
ApiDatasetDetailResponse
} from '@fastgpt/global/core/dataset/apiDataset';
import { POST } from '@fastgpt/service/common/api/plusRequest';
import {
GetProApiDatasetFileContentParams,
GetProApiDatasetFileDetailParams,
GetProApiDatasetFileListParams,
GetProApiDatasetFilePreviewUrlParams,
ProApiDatasetOperationTypeEnum
@@ -34,3 +39,11 @@ export const getProApiDatasetFilePreviewUrlRequest = async (
});
return res;
};
export const getProApiDatasetFileDetailRequest = async (data: GetProApiDatasetFileDetailParams) => {
const res = await POST<ApiDatasetDetailResponse>('/core/dataset/systemApiDataset', {
type: ProApiDatasetOperationTypeEnum.DETAIL,
...data
});
return res;
};

View File

@@ -73,6 +73,14 @@ import type {
} from '@/pages/api/core/dataset/training/getTrainingError';
import type { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
import { GetQuoteDataProps } from '@/pages/api/core/chat/quote/getQuote';
import type {
GetApiDatasetCataLogResponse,
GetApiDatasetCataLogProps
} from '@/pages/api/core/dataset/apiDataset/getCatalog';
import type {
GetApiDatasetPathBody,
GetApiDatasetPathResponse
} from '@/pages/api/core/dataset/apiDataset/getPathNames';
/* ======================== dataset ======================= */
export const getDatasets = (data: GetDatasetListBody) =>
@@ -256,3 +264,9 @@ export const getApiDatasetFileList = (data: GetApiDatasetFileListProps) =>
POST<APIFileItem[]>('/core/dataset/apiDataset/list', data);
export const getApiDatasetFileListExistId = (data: listExistIdQuery) =>
GET<listExistIdResponse>('/core/dataset/apiDataset/listExistId', data);
export const getApiDatasetCatalog = (data: GetApiDatasetCataLogProps) =>
POST<GetApiDatasetCataLogResponse>('/core/dataset/apiDataset/getCatalog', data);
export const getApiDatasetPaths = (data: GetApiDatasetPathBody) =>
POST<GetApiDatasetPathResponse>('/core/dataset/apiDataset/getPathNames', data);

View File

@@ -101,27 +101,28 @@ export const DatasetPageContextProvider = ({
setDatasetDetail((state) => ({
...state,
...data,
agentModel: getWebLLMModel(data.agentModel),
vlmModel: getWebLLMModel(data.vlmModel),
agentModel: data.agentModel ? getWebLLMModel(data.agentModel) : state.agentModel,
vlmModel: data.vlmModel ? getWebLLMModel(data.vlmModel) : state.vlmModel,
apiServer: data.apiServer
? {
baseUrl: data.apiServer.baseUrl,
authorization: ''
}
: undefined,
: state.apiServer,
yuqueServer: data.yuqueServer
? {
userId: data.yuqueServer.userId,
token: ''
token: '',
basePath: data.yuqueServer.basePath
}
: undefined,
: state.yuqueServer,
feishuServer: data.feishuServer
? {
appId: data.feishuServer.appId,
appSecret: '',
folderToken: data.feishuServer.folderToken
}
: undefined
: state.feishuServer
}));
}
};