V4.8.15 feature (#3331)

* feat: add customize toolkit (#3205)

* chaoyang

* fix-auth

* add toolkit

* add order

* plugin usage

* fix

* delete console:

* Fix: Fix fullscreen preview top positioning and improve Markdown rendering logic (#3247)

* 完成任务：修复全屏预览顶部固定问题，优化 Markdown 渲染逻辑

* 有问题修改

* 问题再修改

* 修正问题

* fix: plugin standalone display issue (#3254)

* 4.8.15 test (#3246)

* o1 config

* perf: system plugin code

* 调整系统插件代码。增加html 渲染安全配置。 (#3258)

* perf: base64 picker

* perf: list app or dataset

* perf: plugin config code

* 小窗适配等问题 (#3257)

* 小窗适配等问题

* git问题

* 小窗剩余问题

* feat: system plugin auth and lock version (#3265)

* feat: system plugin auth and lock version

* update comment

* 4.8.15 test (#3267)

* tmp log

* perf: login direct

* perf: iframe html code

* remove log

* fix: plugin standalone display (#3277)

* refactor: 页面拆分&i18n拆分 (#3281)

* refactor: account组件拆成独立页面

* script: 新增i18n json文件创建脚本

* refactor: 页面i18n拆分

* i18n: add en&hant

* 4.8.15 test (#3285)

* tmp log

* remove log

* fix: watch avatar refresh

* perf: i18n code

* fix(plugin): use intro instead of userguide (#3290)

* Universal SSO (#3292)

* tmp log

* remove log

* feat: common oauth

* readme

* perf: sso provider

* remove sso code

* perf: refresh plugins

* feat: add api dataset (#3272)

* add api-dataset

* fix api-dataset

* fix api dataset

* fix ts

* perf: create collection code (#3301)

* tmp log

* remove log

* perf: i18n change

* update version doc

* feat: question guide from chatId

* perf: create collection code

* fix: request api

* fix: request api

* fix: tts auth and response type (#3303)

* perf: md splitter

* fix: tts auth and response type

* fix: api file dataset (#3307)

* perf: api dataset init (#3310)

* perf: collection schema

* perf: api dataset init

* refactor: 团队管理独立页面 (#3302)

* ui: 团队管理独立页面

* 代码优化

* fix

* perf: sync collection and ui check (#3314)

* perf: sync collection

* remove script

* perf: update api server

* perf: api dataset parent

* perf: team ui

* perf: team 18n

* update team ui

* perf: ui check

* perf: i18n

* fix: debug variables & cronjob & system plugin callback load (#3315)

* fix: debug variables & cronjob & system plugin callback load

* fix type

* fix

* fix

* fix: plugin dataset quote;perf: system variables init (#3316)

* fix: plugin dataset quote

* perf: system variables init

* perf: node templates ui;fix: dataset import ui (#3318)

* fix: dataset import ui

* perf: node templates ui

* perf: ui refresh

* feat:套餐改名和套餐跳转配置 (#3309)

* fixing:except Sidebar

* 去除了多余的代码

* 修正了套餐说明的代码

* 修正了误删除的show_git代码

* 修正了名字部分等代码

* 修正了问题,遗留了其他和ui讨论不一致的部分

* 4.8.15 test (#3319)

* remove log

* pref: bill ui

* pref: bill ui

* perf: log

* html渲染文档 (#3270)

* html渲染文档

* 文档有点小问题

* feat: doc (#3322)

* 集合重训练 (#3282)

* rebaser

* 一点补充

* 小问题

* 其他问题修正，删除集合保留文件的参数还没找到...

* reTraining

* delete uesless

* 删除了一行错误代码

* 集合重训练部分

* fixing

* 删除console代码

* feat: navbar item config (#3326)

* perf: custom navbar code;perf: retraining code;feat: api dataset and dataset api doc (#3329)

* feat: api dataset and dataset api doc

* perf: retraining code

* perf: custom navbar code

* fix: ts (#3330)

* fix: ts

* fix: ts

* retraining ui

* perf: api collection filter

* perf: retrining button

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>

packages/global/core/dataset/api.d.ts

@@ -16,6 +16,7 @@ export type DatasetUpdateBody = {
   websiteConfig?: DatasetSchemaType['websiteConfig'];
   externalReadUrl?: DatasetSchemaType['externalReadUrl'];
   defaultPermission?: DatasetSchemaType['defaultPermission'];
+  apiServer?: DatasetSchemaType['apiServer'];
 };
 
 /* ================= collection ===================== */
@@ -34,15 +35,18 @@ export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType &
   name: string;
   type: DatasetCollectionTypeEnum;
 
-  tags?: string[];
-
   fileId?: string;
   rawLink?: string;
   externalFileId?: string;
-
   externalFileUrl?: string;
+  apiFileId?: string;
+
   rawTextLength?: number;
   hashRawText?: string;
+
+  tags?: string[];
+
+  createTime?: Date;
 };
 
 export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
@@ -56,9 +60,17 @@ export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams
 export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
   link: string;
 };
+export type ApiDatasetCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
+  name: string;
+  apiFileId: string;
+};
 export type FileIdCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
   fileId: string;
 };
+export type reTrainingDatasetFileCollectionParams = DatasetCollectionChunkMetadataType & {
+  datasetId: string;
+  collectionId: string;
+};
 export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
   fileMetadata?: Record<string, any>;
   collectionMetadata?: Record<string, any>;

packages/global/core/dataset/apiDataset.d.ts

@@ -0,0 +1,24 @@
+export type APIFileItem = {
+  id: string;
+  parentId: string | null;
+  name: string;
+  type: 'file' | 'folder';
+  updateTime: Date;
+  createTime: Date;
+};
+
+export type APIFileServer = {
+  baseUrl: string;
+  authorization: string;
+};
+
+export type APIFileListResponse = APIFileItem[];
+
+export type APIFileContentResponse = {
+  content?: string;
+  previewUrl?: string;
+};
+
+export type APIFileReadResponse = {
+  url: string;
+};

packages/global/core/dataset/collection/utils.ts

@@ -9,7 +9,8 @@ export const getCollectionSourceData = (
       collection?.fileId ||
       collection?.rawLink ||
       collection?.externalFileId ||
-      collection?.externalFileUrl,
+      collection?.externalFileUrl ||
+      collection?.apiFileId,
     sourceName: collection?.name || ''
   };
 };

packages/global/core/dataset/constants.ts

@@ -1,9 +1,12 @@
+import { i18nT } from '../../../web/i18n/utils';
+
 /* ------------ dataset -------------- */
 export enum DatasetTypeEnum {
   folder = 'folder',
   dataset = 'dataset',
   websiteDataset = 'websiteDataset', // depp link
-  externalFile = 'externalFile'
+  externalFile = 'externalFile',
+  apiDataset = 'apiDataset'
 }
 export const DatasetTypeMap = {
   [DatasetTypeEnum.folder]: {
@@ -25,6 +28,11 @@ export const DatasetTypeMap = {
     icon: 'core/dataset/externalDatasetOutline',
     label: 'external_file',
     collectionLabel: 'common.File'
+  },
+  [DatasetTypeEnum.apiDataset]: {
+    icon: 'core/dataset/externalDatasetOutline',
+    label: 'api_file',
+    collectionLabel: 'common.File'
   }
 };
 
@@ -34,10 +42,10 @@ export enum DatasetStatusEnum {
 }
 export const DatasetStatusMap = {
   [DatasetStatusEnum.active]: {
-    label: 'core.dataset.status.active'
+    label: i18nT('common:core.dataset.status.active')
   },
   [DatasetStatusEnum.syncing]: {
-    label: 'core.dataset.status.syncing'
+    label: i18nT('common:core.dataset.status.syncing')
   }
 };
 
@@ -48,23 +56,27 @@ export enum DatasetCollectionTypeEnum {
 
   file = 'file',
   link = 'link', // one link
-  externalFile = 'externalFile'
+  externalFile = 'externalFile',
+  apiFile = 'apiFile'
 }
 export const DatasetCollectionTypeMap = {
   [DatasetCollectionTypeEnum.folder]: {
-    name: 'core.dataset.folder'
+    name: i18nT('common:core.dataset.folder')
   },
   [DatasetCollectionTypeEnum.file]: {
-    name: 'core.dataset.file'
+    name: i18nT('common:core.dataset.file')
   },
   [DatasetCollectionTypeEnum.externalFile]: {
-    name: 'core.dataset.externalFile'
+    name: i18nT('common:core.dataset.externalFile')
   },
   [DatasetCollectionTypeEnum.link]: {
-    name: 'core.dataset.link'
+    name: i18nT('common:core.dataset.link')
   },
   [DatasetCollectionTypeEnum.virtual]: {
-    name: 'core.dataset.Manual collection'
+    name: i18nT('common:core.dataset.Manual collection')
+  },
+  [DatasetCollectionTypeEnum.apiFile]: {
+    name: i18nT('common:core.dataset.apiFile')
   }
 };
 
@@ -74,10 +86,10 @@ export enum DatasetCollectionSyncResultEnum {
 }
 export const DatasetCollectionSyncResultMap = {
   [DatasetCollectionSyncResultEnum.sameRaw]: {
-    label: 'core.dataset.collection.sync.result.sameRaw'
+    label: i18nT('common:core.dataset.collection.sync.result.sameRaw')
   },
   [DatasetCollectionSyncResultEnum.success]: {
-    label: 'core.dataset.collection.sync.result.success'
+    label: i18nT('common:core.dataset.collection.sync.result.success')
   }
 };
 
@@ -89,7 +101,9 @@ export enum ImportDataSourceEnum {
   fileLink = 'fileLink',
   fileCustom = 'fileCustom',
   csvTable = 'csvTable',
-  externalFile = 'externalFile'
+  externalFile = 'externalFile',
+  apiDataset = 'apiDataset',
+  reTraining = 'reTraining'
 }
 
 export enum TrainingModeEnum {
@@ -100,18 +114,18 @@ export enum TrainingModeEnum {
 
 export const TrainingTypeMap = {
   [TrainingModeEnum.chunk]: {
-    label: 'core.dataset.training.Chunk mode',
-    tooltip: 'core.dataset.import.Chunk Split Tip',
+    label: i18nT('common:core.dataset.training.Chunk mode'),
+    tooltip: i18nT('common:core.dataset.import.Chunk Split Tip'),
     openSource: true
   },
   [TrainingModeEnum.auto]: {
-    label: 'core.dataset.training.Auto mode',
-    tooltip: 'core.dataset.training.Auto mode Tip',
+    label: i18nT('common:core.dataset.training.Auto mode'),
+    tooltip: i18nT('common:core.dataset.training.Auto mode Tip'),
     openSource: false
   },
   [TrainingModeEnum.qa]: {
-    label: 'core.dataset.training.QA mode',
-    tooltip: 'core.dataset.import.QA Import Tip',
+    label: i18nT('common:core.dataset.training.QA mode'),
+    tooltip: i18nT('common:core.dataset.import.QA Import Tip'),
     openSource: true
   }
 };
@@ -126,20 +140,20 @@ export enum DatasetSearchModeEnum {
 export const DatasetSearchModeMap = {
   [DatasetSearchModeEnum.embedding]: {
     icon: 'core/dataset/modeEmbedding',
-    title: 'core.dataset.search.mode.embedding',
-    desc: 'core.dataset.search.mode.embedding desc',
+    title: i18nT('common:core.dataset.search.mode.embedding'),
+    desc: i18nT('common:core.dataset.search.mode.embedding desc'),
     value: DatasetSearchModeEnum.embedding
   },
   [DatasetSearchModeEnum.fullTextRecall]: {
     icon: 'core/dataset/fullTextRecall',
-    title: 'core.dataset.search.mode.fullTextRecall',
-    desc: 'core.dataset.search.mode.fullTextRecall desc',
+    title: i18nT('common:core.dataset.search.mode.fullTextRecall'),
+    desc: i18nT('common:core.dataset.search.mode.fullTextRecall desc'),
     value: DatasetSearchModeEnum.fullTextRecall
   },
   [DatasetSearchModeEnum.mixedRecall]: {
     icon: 'core/dataset/mixedRecall',
-    title: 'core.dataset.search.mode.mixedRecall',
-    desc: 'core.dataset.search.mode.mixedRecall desc',
+    title: i18nT('common:core.dataset.search.mode.mixedRecall'),
+    desc: i18nT('common:core.dataset.search.mode.mixedRecall desc'),
     value: DatasetSearchModeEnum.mixedRecall
   }
 };
@@ -152,23 +166,23 @@ export enum SearchScoreTypeEnum {
 }
 export const SearchScoreTypeMap = {
   [SearchScoreTypeEnum.embedding]: {
-    label: 'core.dataset.search.score.embedding',
-    desc: 'core.dataset.search.score.embedding desc',
+    label: i18nT('common:core.dataset.search.score.embedding'),
+    desc: i18nT('common:core.dataset.search.score.embedding desc'),
     showScore: true
   },
   [SearchScoreTypeEnum.fullText]: {
-    label: 'core.dataset.search.score.fullText',
-    desc: 'core.dataset.search.score.fullText desc',
+    label: i18nT('common:core.dataset.search.score.fullText'),
+    desc: i18nT('common:core.dataset.search.score.fullText desc'),
     showScore: false
   },
   [SearchScoreTypeEnum.reRank]: {
-    label: 'core.dataset.search.score.reRank',
-    desc: 'core.dataset.search.score.reRank desc',
+    label: i18nT('common:core.dataset.search.score.reRank'),
+    desc: i18nT('common:core.dataset.search.score.reRank desc'),
     showScore: true
   },
   [SearchScoreTypeEnum.rrf]: {
-    label: 'core.dataset.search.score.rrf',
-    desc: 'core.dataset.search.score.rrf desc',
+    label: i18nT('common:core.dataset.search.score.rrf'),
+    desc: i18nT('common:core.dataset.search.score.rrf desc'),
     showScore: false
   }
 };
@@ -180,5 +194,7 @@ export const LinkCollectionIcon = 'common/linkBlue';
 export enum DatasetSourceReadTypeEnum {
   fileLocal = 'fileLocal',
   link = 'link',
-  externalFile = 'externalFile'
+  externalFile = 'externalFile',
+  apiFile = 'apiFile',
+  reTraining = 'reTraining'
 }

packages/global/core/dataset/read.ts

@@ -1,14 +0,0 @@
-import { DatasetSourceReadTypeEnum, ImportDataSourceEnum } from './constants';
-
-export const importType2ReadType = (type: ImportDataSourceEnum) => {
-  if (type === ImportDataSourceEnum.csvTable || type === ImportDataSourceEnum.fileLocal) {
-    return DatasetSourceReadTypeEnum.fileLocal;
-  }
-  if (type === ImportDataSourceEnum.fileLink) {
-    return DatasetSourceReadTypeEnum.link;
-  }
-  if (type === ImportDataSourceEnum.externalFile) {
-    return DatasetSourceReadTypeEnum.externalFile;
-  }
-  return DatasetSourceReadTypeEnum.link;
-};

packages/global/core/dataset/type.d.ts

@@ -10,6 +10,7 @@ import {
 } from './constants';
 import { DatasetPermission } from '../../support/permission/dataset/controller';
 import { Permission } from '../../support/permission/controller';
+import { APIFileServer } from './apiDataset';
 
 export type DatasetSchemaType = {
   _id: string;
@@ -30,10 +31,11 @@ export type DatasetSchemaType = {
     url: string;
     selector: string;
   };
-  externalReadUrl?: string;
   inheritPermission: boolean;
+  apiServer?: APIFileServer;
 
   // abandon
+  externalReadUrl?: string;
   defaultPermission?: number;
 };
 
@@ -64,6 +66,7 @@ export type DatasetCollectionSchemaType = {
   rawTextLength?: number;
   hashRawText?: string;
   externalFileUrl?: string; // external import url
+  apiFileId?: string; // api file id
   metadata?: {
     webPageSelector?: string;
     relatedImgId?: string; // The id of the associated image collections