mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-23 21:13:50 +00:00
V4.8.15 feature (#3331)
* feat: add customize toolkit (#3205) * chaoyang * fix-auth * add toolkit * add order * plugin usage * fix * delete console: * Fix: Fix fullscreen preview top positioning and improve Markdown rendering logic (#3247) * 完成任务:修复全屏预览顶部固定问题,优化 Markdown 渲染逻辑 * 有问题修改 * 问题再修改 * 修正问题 * fix: plugin standalone display issue (#3254) * 4.8.15 test (#3246) * o1 config * perf: system plugin code * 调整系统插件代码。增加html 渲染安全配置。 (#3258) * perf: base64 picker * perf: list app or dataset * perf: plugin config code * 小窗适配等问题 (#3257) * 小窗适配等问题 * git问题 * 小窗剩余问题 * feat: system plugin auth and lock version (#3265) * feat: system plugin auth and lock version * update comment * 4.8.15 test (#3267) * tmp log * perf: login direct * perf: iframe html code * remove log * fix: plugin standalone display (#3277) * refactor: 页面拆分&i18n拆分 (#3281) * refactor: account组件拆成独立页面 * script: 新增i18n json文件创建脚本 * refactor: 页面i18n拆分 * i18n: add en&hant * 4.8.15 test (#3285) * tmp log * remove log * fix: watch avatar refresh * perf: i18n code * fix(plugin): use intro instead of userguide (#3290) * Universal SSO (#3292) * tmp log * remove log * feat: common oauth * readme * perf: sso provider * remove sso code * perf: refresh plugins * feat: add api dataset (#3272) * add api-dataset * fix api-dataset * fix api dataset * fix ts * perf: create collection code (#3301) * tmp log * remove log * perf: i18n change * update version doc * feat: question guide from chatId * perf: create collection code * fix: request api * fix: request api * fix: tts auth and response type (#3303) * perf: md splitter * fix: tts auth and response type * fix: api file dataset (#3307) * perf: api dataset init (#3310) * perf: collection schema * perf: api dataset init * refactor: 团队管理独立页面 (#3302) * ui: 团队管理独立页面 * 代码优化 * fix * perf: sync collection and ui check (#3314) * perf: sync collection * remove script * perf: update api server * perf: api dataset parent * perf: team ui * perf: team 18n * update team ui * perf: ui check * perf: i18n * fix: debug variables & cronjob & system plugin callback load (#3315) * fix: debug variables & cronjob & system plugin callback load * fix type * fix * fix * fix: plugin dataset quote;perf: system variables init (#3316) * fix: plugin dataset quote * perf: system variables init * perf: node templates ui;fix: dataset import ui (#3318) * fix: dataset import ui * perf: node templates ui * perf: ui refresh * feat:套餐改名和套餐跳转配置 (#3309) * fixing:except Sidebar * 去除了多余的代码 * 修正了套餐说明的代码 * 修正了误删除的show_git代码 * 修正了名字部分等代码 * 修正了问题,遗留了其他和ui讨论不一致的部分 * 4.8.15 test (#3319) * remove log * pref: bill ui * pref: bill ui * perf: log * html渲染文档 (#3270) * html渲染文档 * 文档有点小问题 * feat: doc (#3322) * 集合重训练 (#3282) * rebaser * 一点补充 * 小问题 * 其他问题修正,删除集合保留文件的参数还没找到... * reTraining * delete uesless * 删除了一行错误代码 * 集合重训练部分 * fixing * 删除console代码 * feat: navbar item config (#3326) * perf: custom navbar code;perf: retraining code;feat: api dataset and dataset api doc (#3329) * feat: api dataset and dataset api doc * perf: retraining code * perf: custom navbar code * fix: ts (#3330) * fix: ts * fix: ts * retraining ui * perf: api collection filter * perf: retrining button --------- Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com> Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>
This commit is contained in:
18
packages/global/core/dataset/api.d.ts
vendored
18
packages/global/core/dataset/api.d.ts
vendored
@@ -16,6 +16,7 @@ export type DatasetUpdateBody = {
|
||||
websiteConfig?: DatasetSchemaType['websiteConfig'];
|
||||
externalReadUrl?: DatasetSchemaType['externalReadUrl'];
|
||||
defaultPermission?: DatasetSchemaType['defaultPermission'];
|
||||
apiServer?: DatasetSchemaType['apiServer'];
|
||||
};
|
||||
|
||||
/* ================= collection ===================== */
|
||||
@@ -34,15 +35,18 @@ export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType &
|
||||
name: string;
|
||||
type: DatasetCollectionTypeEnum;
|
||||
|
||||
tags?: string[];
|
||||
|
||||
fileId?: string;
|
||||
rawLink?: string;
|
||||
externalFileId?: string;
|
||||
|
||||
externalFileUrl?: string;
|
||||
apiFileId?: string;
|
||||
|
||||
rawTextLength?: number;
|
||||
hashRawText?: string;
|
||||
|
||||
tags?: string[];
|
||||
|
||||
createTime?: Date;
|
||||
};
|
||||
|
||||
export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
|
||||
@@ -56,9 +60,17 @@ export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams
|
||||
export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
|
||||
link: string;
|
||||
};
|
||||
export type ApiDatasetCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
|
||||
name: string;
|
||||
apiFileId: string;
|
||||
};
|
||||
export type FileIdCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
|
||||
fileId: string;
|
||||
};
|
||||
export type reTrainingDatasetFileCollectionParams = DatasetCollectionChunkMetadataType & {
|
||||
datasetId: string;
|
||||
collectionId: string;
|
||||
};
|
||||
export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
|
||||
fileMetadata?: Record<string, any>;
|
||||
collectionMetadata?: Record<string, any>;
|
||||
|
24
packages/global/core/dataset/apiDataset.d.ts
vendored
Normal file
24
packages/global/core/dataset/apiDataset.d.ts
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
export type APIFileItem = {
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
name: string;
|
||||
type: 'file' | 'folder';
|
||||
updateTime: Date;
|
||||
createTime: Date;
|
||||
};
|
||||
|
||||
export type APIFileServer = {
|
||||
baseUrl: string;
|
||||
authorization: string;
|
||||
};
|
||||
|
||||
export type APIFileListResponse = APIFileItem[];
|
||||
|
||||
export type APIFileContentResponse = {
|
||||
content?: string;
|
||||
previewUrl?: string;
|
||||
};
|
||||
|
||||
export type APIFileReadResponse = {
|
||||
url: string;
|
||||
};
|
@@ -9,7 +9,8 @@ export const getCollectionSourceData = (
|
||||
collection?.fileId ||
|
||||
collection?.rawLink ||
|
||||
collection?.externalFileId ||
|
||||
collection?.externalFileUrl,
|
||||
collection?.externalFileUrl ||
|
||||
collection?.apiFileId,
|
||||
sourceName: collection?.name || ''
|
||||
};
|
||||
};
|
||||
|
@@ -1,9 +1,12 @@
|
||||
import { i18nT } from '../../../web/i18n/utils';
|
||||
|
||||
/* ------------ dataset -------------- */
|
||||
export enum DatasetTypeEnum {
|
||||
folder = 'folder',
|
||||
dataset = 'dataset',
|
||||
websiteDataset = 'websiteDataset', // depp link
|
||||
externalFile = 'externalFile'
|
||||
externalFile = 'externalFile',
|
||||
apiDataset = 'apiDataset'
|
||||
}
|
||||
export const DatasetTypeMap = {
|
||||
[DatasetTypeEnum.folder]: {
|
||||
@@ -25,6 +28,11 @@ export const DatasetTypeMap = {
|
||||
icon: 'core/dataset/externalDatasetOutline',
|
||||
label: 'external_file',
|
||||
collectionLabel: 'common.File'
|
||||
},
|
||||
[DatasetTypeEnum.apiDataset]: {
|
||||
icon: 'core/dataset/externalDatasetOutline',
|
||||
label: 'api_file',
|
||||
collectionLabel: 'common.File'
|
||||
}
|
||||
};
|
||||
|
||||
@@ -34,10 +42,10 @@ export enum DatasetStatusEnum {
|
||||
}
|
||||
export const DatasetStatusMap = {
|
||||
[DatasetStatusEnum.active]: {
|
||||
label: 'core.dataset.status.active'
|
||||
label: i18nT('common:core.dataset.status.active')
|
||||
},
|
||||
[DatasetStatusEnum.syncing]: {
|
||||
label: 'core.dataset.status.syncing'
|
||||
label: i18nT('common:core.dataset.status.syncing')
|
||||
}
|
||||
};
|
||||
|
||||
@@ -48,23 +56,27 @@ export enum DatasetCollectionTypeEnum {
|
||||
|
||||
file = 'file',
|
||||
link = 'link', // one link
|
||||
externalFile = 'externalFile'
|
||||
externalFile = 'externalFile',
|
||||
apiFile = 'apiFile'
|
||||
}
|
||||
export const DatasetCollectionTypeMap = {
|
||||
[DatasetCollectionTypeEnum.folder]: {
|
||||
name: 'core.dataset.folder'
|
||||
name: i18nT('common:core.dataset.folder')
|
||||
},
|
||||
[DatasetCollectionTypeEnum.file]: {
|
||||
name: 'core.dataset.file'
|
||||
name: i18nT('common:core.dataset.file')
|
||||
},
|
||||
[DatasetCollectionTypeEnum.externalFile]: {
|
||||
name: 'core.dataset.externalFile'
|
||||
name: i18nT('common:core.dataset.externalFile')
|
||||
},
|
||||
[DatasetCollectionTypeEnum.link]: {
|
||||
name: 'core.dataset.link'
|
||||
name: i18nT('common:core.dataset.link')
|
||||
},
|
||||
[DatasetCollectionTypeEnum.virtual]: {
|
||||
name: 'core.dataset.Manual collection'
|
||||
name: i18nT('common:core.dataset.Manual collection')
|
||||
},
|
||||
[DatasetCollectionTypeEnum.apiFile]: {
|
||||
name: i18nT('common:core.dataset.apiFile')
|
||||
}
|
||||
};
|
||||
|
||||
@@ -74,10 +86,10 @@ export enum DatasetCollectionSyncResultEnum {
|
||||
}
|
||||
export const DatasetCollectionSyncResultMap = {
|
||||
[DatasetCollectionSyncResultEnum.sameRaw]: {
|
||||
label: 'core.dataset.collection.sync.result.sameRaw'
|
||||
label: i18nT('common:core.dataset.collection.sync.result.sameRaw')
|
||||
},
|
||||
[DatasetCollectionSyncResultEnum.success]: {
|
||||
label: 'core.dataset.collection.sync.result.success'
|
||||
label: i18nT('common:core.dataset.collection.sync.result.success')
|
||||
}
|
||||
};
|
||||
|
||||
@@ -89,7 +101,9 @@ export enum ImportDataSourceEnum {
|
||||
fileLink = 'fileLink',
|
||||
fileCustom = 'fileCustom',
|
||||
csvTable = 'csvTable',
|
||||
externalFile = 'externalFile'
|
||||
externalFile = 'externalFile',
|
||||
apiDataset = 'apiDataset',
|
||||
reTraining = 'reTraining'
|
||||
}
|
||||
|
||||
export enum TrainingModeEnum {
|
||||
@@ -100,18 +114,18 @@ export enum TrainingModeEnum {
|
||||
|
||||
export const TrainingTypeMap = {
|
||||
[TrainingModeEnum.chunk]: {
|
||||
label: 'core.dataset.training.Chunk mode',
|
||||
tooltip: 'core.dataset.import.Chunk Split Tip',
|
||||
label: i18nT('common:core.dataset.training.Chunk mode'),
|
||||
tooltip: i18nT('common:core.dataset.import.Chunk Split Tip'),
|
||||
openSource: true
|
||||
},
|
||||
[TrainingModeEnum.auto]: {
|
||||
label: 'core.dataset.training.Auto mode',
|
||||
tooltip: 'core.dataset.training.Auto mode Tip',
|
||||
label: i18nT('common:core.dataset.training.Auto mode'),
|
||||
tooltip: i18nT('common:core.dataset.training.Auto mode Tip'),
|
||||
openSource: false
|
||||
},
|
||||
[TrainingModeEnum.qa]: {
|
||||
label: 'core.dataset.training.QA mode',
|
||||
tooltip: 'core.dataset.import.QA Import Tip',
|
||||
label: i18nT('common:core.dataset.training.QA mode'),
|
||||
tooltip: i18nT('common:core.dataset.import.QA Import Tip'),
|
||||
openSource: true
|
||||
}
|
||||
};
|
||||
@@ -126,20 +140,20 @@ export enum DatasetSearchModeEnum {
|
||||
export const DatasetSearchModeMap = {
|
||||
[DatasetSearchModeEnum.embedding]: {
|
||||
icon: 'core/dataset/modeEmbedding',
|
||||
title: 'core.dataset.search.mode.embedding',
|
||||
desc: 'core.dataset.search.mode.embedding desc',
|
||||
title: i18nT('common:core.dataset.search.mode.embedding'),
|
||||
desc: i18nT('common:core.dataset.search.mode.embedding desc'),
|
||||
value: DatasetSearchModeEnum.embedding
|
||||
},
|
||||
[DatasetSearchModeEnum.fullTextRecall]: {
|
||||
icon: 'core/dataset/fullTextRecall',
|
||||
title: 'core.dataset.search.mode.fullTextRecall',
|
||||
desc: 'core.dataset.search.mode.fullTextRecall desc',
|
||||
title: i18nT('common:core.dataset.search.mode.fullTextRecall'),
|
||||
desc: i18nT('common:core.dataset.search.mode.fullTextRecall desc'),
|
||||
value: DatasetSearchModeEnum.fullTextRecall
|
||||
},
|
||||
[DatasetSearchModeEnum.mixedRecall]: {
|
||||
icon: 'core/dataset/mixedRecall',
|
||||
title: 'core.dataset.search.mode.mixedRecall',
|
||||
desc: 'core.dataset.search.mode.mixedRecall desc',
|
||||
title: i18nT('common:core.dataset.search.mode.mixedRecall'),
|
||||
desc: i18nT('common:core.dataset.search.mode.mixedRecall desc'),
|
||||
value: DatasetSearchModeEnum.mixedRecall
|
||||
}
|
||||
};
|
||||
@@ -152,23 +166,23 @@ export enum SearchScoreTypeEnum {
|
||||
}
|
||||
export const SearchScoreTypeMap = {
|
||||
[SearchScoreTypeEnum.embedding]: {
|
||||
label: 'core.dataset.search.score.embedding',
|
||||
desc: 'core.dataset.search.score.embedding desc',
|
||||
label: i18nT('common:core.dataset.search.score.embedding'),
|
||||
desc: i18nT('common:core.dataset.search.score.embedding desc'),
|
||||
showScore: true
|
||||
},
|
||||
[SearchScoreTypeEnum.fullText]: {
|
||||
label: 'core.dataset.search.score.fullText',
|
||||
desc: 'core.dataset.search.score.fullText desc',
|
||||
label: i18nT('common:core.dataset.search.score.fullText'),
|
||||
desc: i18nT('common:core.dataset.search.score.fullText desc'),
|
||||
showScore: false
|
||||
},
|
||||
[SearchScoreTypeEnum.reRank]: {
|
||||
label: 'core.dataset.search.score.reRank',
|
||||
desc: 'core.dataset.search.score.reRank desc',
|
||||
label: i18nT('common:core.dataset.search.score.reRank'),
|
||||
desc: i18nT('common:core.dataset.search.score.reRank desc'),
|
||||
showScore: true
|
||||
},
|
||||
[SearchScoreTypeEnum.rrf]: {
|
||||
label: 'core.dataset.search.score.rrf',
|
||||
desc: 'core.dataset.search.score.rrf desc',
|
||||
label: i18nT('common:core.dataset.search.score.rrf'),
|
||||
desc: i18nT('common:core.dataset.search.score.rrf desc'),
|
||||
showScore: false
|
||||
}
|
||||
};
|
||||
@@ -180,5 +194,7 @@ export const LinkCollectionIcon = 'common/linkBlue';
|
||||
export enum DatasetSourceReadTypeEnum {
|
||||
fileLocal = 'fileLocal',
|
||||
link = 'link',
|
||||
externalFile = 'externalFile'
|
||||
externalFile = 'externalFile',
|
||||
apiFile = 'apiFile',
|
||||
reTraining = 'reTraining'
|
||||
}
|
||||
|
@@ -1,14 +0,0 @@
|
||||
import { DatasetSourceReadTypeEnum, ImportDataSourceEnum } from './constants';
|
||||
|
||||
export const importType2ReadType = (type: ImportDataSourceEnum) => {
|
||||
if (type === ImportDataSourceEnum.csvTable || type === ImportDataSourceEnum.fileLocal) {
|
||||
return DatasetSourceReadTypeEnum.fileLocal;
|
||||
}
|
||||
if (type === ImportDataSourceEnum.fileLink) {
|
||||
return DatasetSourceReadTypeEnum.link;
|
||||
}
|
||||
if (type === ImportDataSourceEnum.externalFile) {
|
||||
return DatasetSourceReadTypeEnum.externalFile;
|
||||
}
|
||||
return DatasetSourceReadTypeEnum.link;
|
||||
};
|
5
packages/global/core/dataset/type.d.ts
vendored
5
packages/global/core/dataset/type.d.ts
vendored
@@ -10,6 +10,7 @@ import {
|
||||
} from './constants';
|
||||
import { DatasetPermission } from '../../support/permission/dataset/controller';
|
||||
import { Permission } from '../../support/permission/controller';
|
||||
import { APIFileServer } from './apiDataset';
|
||||
|
||||
export type DatasetSchemaType = {
|
||||
_id: string;
|
||||
@@ -30,10 +31,11 @@ export type DatasetSchemaType = {
|
||||
url: string;
|
||||
selector: string;
|
||||
};
|
||||
externalReadUrl?: string;
|
||||
inheritPermission: boolean;
|
||||
apiServer?: APIFileServer;
|
||||
|
||||
// abandon
|
||||
externalReadUrl?: string;
|
||||
defaultPermission?: number;
|
||||
};
|
||||
|
||||
@@ -64,6 +66,7 @@ export type DatasetCollectionSchemaType = {
|
||||
rawTextLength?: number;
|
||||
hashRawText?: string;
|
||||
externalFileUrl?: string; // external import url
|
||||
apiFileId?: string; // api file id
|
||||
metadata?: {
|
||||
webPageSelector?: string;
|
||||
relatedImgId?: string; // The id of the associated image collections
|
||||
|
Reference in New Issue
Block a user