V4.8.15 feature (#3331)

* feat: add customize toolkit (#3205)

* chaoyang

* fix-auth

* add toolkit

* add order

* plugin usage

* fix

* delete console:

* Fix: Fix fullscreen preview top positioning and improve Markdown rendering logic (#3247)

* 完成任务:修复全屏预览顶部固定问题,优化 Markdown 渲染逻辑

* 有问题修改

* 问题再修改

* 修正问题

* fix: plugin standalone display issue (#3254)

* 4.8.15 test (#3246)

* o1 config

* perf: system plugin code

* 调整系统插件代码。增加html 渲染安全配置。 (#3258)

* perf: base64 picker

* perf: list app or dataset

* perf: plugin config code

* 小窗适配等问题 (#3257)

* 小窗适配等问题

* git问题

* 小窗剩余问题

* feat: system plugin auth and lock version (#3265)

* feat: system plugin auth and lock version

* update comment

* 4.8.15 test (#3267)

* tmp log

* perf: login direct

* perf: iframe html code

* remove log

* fix: plugin standalone display (#3277)

* refactor: 页面拆分&i18n拆分 (#3281)

* refactor: account组件拆成独立页面

* script: 新增i18n json文件创建脚本

* refactor: 页面i18n拆分

* i18n: add en&hant

* 4.8.15 test (#3285)

* tmp log

* remove log

* fix: watch avatar refresh

* perf: i18n code

* fix(plugin): use intro instead of userguide (#3290)

* Universal SSO (#3292)

* tmp log

* remove log

* feat: common oauth

* readme

* perf: sso provider

* remove sso code

* perf: refresh plugins

* feat: add api dataset (#3272)

* add api-dataset

* fix api-dataset

* fix api dataset

* fix ts

* perf: create collection code (#3301)

* tmp log

* remove log

* perf: i18n change

* update version doc

* feat: question guide from chatId

* perf: create collection code

* fix: request api

* fix: request api

* fix: tts auth and response type (#3303)

* perf: md splitter

* fix: tts auth and response type

* fix: api file dataset (#3307)

* perf: api dataset init (#3310)

* perf: collection schema

* perf: api dataset init

* refactor: 团队管理独立页面 (#3302)

* ui: 团队管理独立页面

* 代码优化

* fix

* perf: sync collection and ui check (#3314)

* perf: sync collection

* remove script

* perf: update api server

* perf: api dataset parent

* perf: team ui

* perf: team 18n

* update team ui

* perf: ui check

* perf: i18n

* fix: debug variables & cronjob & system plugin callback load (#3315)

* fix: debug variables & cronjob & system plugin callback load

* fix type

* fix

* fix

* fix: plugin dataset quote;perf: system variables init (#3316)

* fix: plugin dataset quote

* perf: system variables init

* perf: node templates ui;fix: dataset import ui (#3318)

* fix: dataset import ui

* perf: node templates ui

* perf: ui refresh

* feat:套餐改名和套餐跳转配置 (#3309)

* fixing:except Sidebar

* 去除了多余的代码

* 修正了套餐说明的代码

* 修正了误删除的show_git代码

* 修正了名字部分等代码

* 修正了问题,遗留了其他和ui讨论不一致的部分

* 4.8.15 test (#3319)

* remove log

* pref: bill ui

* pref: bill ui

* perf: log

* html渲染文档 (#3270)

* html渲染文档

* 文档有点小问题

* feat: doc (#3322)

* 集合重训练 (#3282)

* rebaser

* 一点补充

* 小问题

* 其他问题修正,删除集合保留文件的参数还没找到...

* reTraining

* delete uesless

* 删除了一行错误代码

* 集合重训练部分

* fixing

* 删除console代码

* feat: navbar item config (#3326)

* perf: custom navbar code;perf: retraining code;feat: api dataset and dataset api doc (#3329)

* feat: api dataset and dataset api doc

* perf: retraining code

* perf: custom navbar code

* fix: ts (#3330)

* fix: ts

* fix: ts

* retraining ui

* perf: api collection filter

* perf: retrining button

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>
This commit is contained in:
Archer
2024-12-06 10:56:53 +08:00
committed by GitHub
parent b188544386
commit 1aebe5f185
307 changed files with 7383 additions and 3981 deletions

View File

@@ -48,3 +48,5 @@ export enum AppTemplateTypeEnum {
roleplay = 'roleplay',
officeServices = 'office-services'
}
export const defaultDatasetMaxTokens = 16000;

View File

@@ -170,3 +170,9 @@ export type AppFileSelectConfigType = {
canSelectImg: boolean;
maxFiles: number;
};
export type SystemPluginListItemType = {
_id: string;
name: string;
avatar: string;
};

View File

@@ -16,6 +16,7 @@ export type DatasetUpdateBody = {
websiteConfig?: DatasetSchemaType['websiteConfig'];
externalReadUrl?: DatasetSchemaType['externalReadUrl'];
defaultPermission?: DatasetSchemaType['defaultPermission'];
apiServer?: DatasetSchemaType['apiServer'];
};
/* ================= collection ===================== */
@@ -34,15 +35,18 @@ export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType &
name: string;
type: DatasetCollectionTypeEnum;
tags?: string[];
fileId?: string;
rawLink?: string;
externalFileId?: string;
externalFileUrl?: string;
apiFileId?: string;
rawTextLength?: number;
hashRawText?: string;
tags?: string[];
createTime?: Date;
};
export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
@@ -56,9 +60,17 @@ export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams
export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
link: string;
};
export type ApiDatasetCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
name: string;
apiFileId: string;
};
export type FileIdCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
fileId: string;
};
export type reTrainingDatasetFileCollectionParams = DatasetCollectionChunkMetadataType & {
datasetId: string;
collectionId: string;
};
export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
fileMetadata?: Record<string, any>;
collectionMetadata?: Record<string, any>;

View File

@@ -0,0 +1,24 @@
export type APIFileItem = {
id: string;
parentId: string | null;
name: string;
type: 'file' | 'folder';
updateTime: Date;
createTime: Date;
};
export type APIFileServer = {
baseUrl: string;
authorization: string;
};
export type APIFileListResponse = APIFileItem[];
export type APIFileContentResponse = {
content?: string;
previewUrl?: string;
};
export type APIFileReadResponse = {
url: string;
};

View File

@@ -9,7 +9,8 @@ export const getCollectionSourceData = (
collection?.fileId ||
collection?.rawLink ||
collection?.externalFileId ||
collection?.externalFileUrl,
collection?.externalFileUrl ||
collection?.apiFileId,
sourceName: collection?.name || ''
};
};

View File

@@ -1,9 +1,12 @@
import { i18nT } from '../../../web/i18n/utils';
/* ------------ dataset -------------- */
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset',
websiteDataset = 'websiteDataset', // depp link
externalFile = 'externalFile'
externalFile = 'externalFile',
apiDataset = 'apiDataset'
}
export const DatasetTypeMap = {
[DatasetTypeEnum.folder]: {
@@ -25,6 +28,11 @@ export const DatasetTypeMap = {
icon: 'core/dataset/externalDatasetOutline',
label: 'external_file',
collectionLabel: 'common.File'
},
[DatasetTypeEnum.apiDataset]: {
icon: 'core/dataset/externalDatasetOutline',
label: 'api_file',
collectionLabel: 'common.File'
}
};
@@ -34,10 +42,10 @@ export enum DatasetStatusEnum {
}
export const DatasetStatusMap = {
[DatasetStatusEnum.active]: {
label: 'core.dataset.status.active'
label: i18nT('common:core.dataset.status.active')
},
[DatasetStatusEnum.syncing]: {
label: 'core.dataset.status.syncing'
label: i18nT('common:core.dataset.status.syncing')
}
};
@@ -48,23 +56,27 @@ export enum DatasetCollectionTypeEnum {
file = 'file',
link = 'link', // one link
externalFile = 'externalFile'
externalFile = 'externalFile',
apiFile = 'apiFile'
}
export const DatasetCollectionTypeMap = {
[DatasetCollectionTypeEnum.folder]: {
name: 'core.dataset.folder'
name: i18nT('common:core.dataset.folder')
},
[DatasetCollectionTypeEnum.file]: {
name: 'core.dataset.file'
name: i18nT('common:core.dataset.file')
},
[DatasetCollectionTypeEnum.externalFile]: {
name: 'core.dataset.externalFile'
name: i18nT('common:core.dataset.externalFile')
},
[DatasetCollectionTypeEnum.link]: {
name: 'core.dataset.link'
name: i18nT('common:core.dataset.link')
},
[DatasetCollectionTypeEnum.virtual]: {
name: 'core.dataset.Manual collection'
name: i18nT('common:core.dataset.Manual collection')
},
[DatasetCollectionTypeEnum.apiFile]: {
name: i18nT('common:core.dataset.apiFile')
}
};
@@ -74,10 +86,10 @@ export enum DatasetCollectionSyncResultEnum {
}
export const DatasetCollectionSyncResultMap = {
[DatasetCollectionSyncResultEnum.sameRaw]: {
label: 'core.dataset.collection.sync.result.sameRaw'
label: i18nT('common:core.dataset.collection.sync.result.sameRaw')
},
[DatasetCollectionSyncResultEnum.success]: {
label: 'core.dataset.collection.sync.result.success'
label: i18nT('common:core.dataset.collection.sync.result.success')
}
};
@@ -89,7 +101,9 @@ export enum ImportDataSourceEnum {
fileLink = 'fileLink',
fileCustom = 'fileCustom',
csvTable = 'csvTable',
externalFile = 'externalFile'
externalFile = 'externalFile',
apiDataset = 'apiDataset',
reTraining = 'reTraining'
}
export enum TrainingModeEnum {
@@ -100,18 +114,18 @@ export enum TrainingModeEnum {
export const TrainingTypeMap = {
[TrainingModeEnum.chunk]: {
label: 'core.dataset.training.Chunk mode',
tooltip: 'core.dataset.import.Chunk Split Tip',
label: i18nT('common:core.dataset.training.Chunk mode'),
tooltip: i18nT('common:core.dataset.import.Chunk Split Tip'),
openSource: true
},
[TrainingModeEnum.auto]: {
label: 'core.dataset.training.Auto mode',
tooltip: 'core.dataset.training.Auto mode Tip',
label: i18nT('common:core.dataset.training.Auto mode'),
tooltip: i18nT('common:core.dataset.training.Auto mode Tip'),
openSource: false
},
[TrainingModeEnum.qa]: {
label: 'core.dataset.training.QA mode',
tooltip: 'core.dataset.import.QA Import Tip',
label: i18nT('common:core.dataset.training.QA mode'),
tooltip: i18nT('common:core.dataset.import.QA Import Tip'),
openSource: true
}
};
@@ -126,20 +140,20 @@ export enum DatasetSearchModeEnum {
export const DatasetSearchModeMap = {
[DatasetSearchModeEnum.embedding]: {
icon: 'core/dataset/modeEmbedding',
title: 'core.dataset.search.mode.embedding',
desc: 'core.dataset.search.mode.embedding desc',
title: i18nT('common:core.dataset.search.mode.embedding'),
desc: i18nT('common:core.dataset.search.mode.embedding desc'),
value: DatasetSearchModeEnum.embedding
},
[DatasetSearchModeEnum.fullTextRecall]: {
icon: 'core/dataset/fullTextRecall',
title: 'core.dataset.search.mode.fullTextRecall',
desc: 'core.dataset.search.mode.fullTextRecall desc',
title: i18nT('common:core.dataset.search.mode.fullTextRecall'),
desc: i18nT('common:core.dataset.search.mode.fullTextRecall desc'),
value: DatasetSearchModeEnum.fullTextRecall
},
[DatasetSearchModeEnum.mixedRecall]: {
icon: 'core/dataset/mixedRecall',
title: 'core.dataset.search.mode.mixedRecall',
desc: 'core.dataset.search.mode.mixedRecall desc',
title: i18nT('common:core.dataset.search.mode.mixedRecall'),
desc: i18nT('common:core.dataset.search.mode.mixedRecall desc'),
value: DatasetSearchModeEnum.mixedRecall
}
};
@@ -152,23 +166,23 @@ export enum SearchScoreTypeEnum {
}
export const SearchScoreTypeMap = {
[SearchScoreTypeEnum.embedding]: {
label: 'core.dataset.search.score.embedding',
desc: 'core.dataset.search.score.embedding desc',
label: i18nT('common:core.dataset.search.score.embedding'),
desc: i18nT('common:core.dataset.search.score.embedding desc'),
showScore: true
},
[SearchScoreTypeEnum.fullText]: {
label: 'core.dataset.search.score.fullText',
desc: 'core.dataset.search.score.fullText desc',
label: i18nT('common:core.dataset.search.score.fullText'),
desc: i18nT('common:core.dataset.search.score.fullText desc'),
showScore: false
},
[SearchScoreTypeEnum.reRank]: {
label: 'core.dataset.search.score.reRank',
desc: 'core.dataset.search.score.reRank desc',
label: i18nT('common:core.dataset.search.score.reRank'),
desc: i18nT('common:core.dataset.search.score.reRank desc'),
showScore: true
},
[SearchScoreTypeEnum.rrf]: {
label: 'core.dataset.search.score.rrf',
desc: 'core.dataset.search.score.rrf desc',
label: i18nT('common:core.dataset.search.score.rrf'),
desc: i18nT('common:core.dataset.search.score.rrf desc'),
showScore: false
}
};
@@ -180,5 +194,7 @@ export const LinkCollectionIcon = 'common/linkBlue';
export enum DatasetSourceReadTypeEnum {
fileLocal = 'fileLocal',
link = 'link',
externalFile = 'externalFile'
externalFile = 'externalFile',
apiFile = 'apiFile',
reTraining = 'reTraining'
}

View File

@@ -1,14 +0,0 @@
import { DatasetSourceReadTypeEnum, ImportDataSourceEnum } from './constants';
export const importType2ReadType = (type: ImportDataSourceEnum) => {
if (type === ImportDataSourceEnum.csvTable || type === ImportDataSourceEnum.fileLocal) {
return DatasetSourceReadTypeEnum.fileLocal;
}
if (type === ImportDataSourceEnum.fileLink) {
return DatasetSourceReadTypeEnum.link;
}
if (type === ImportDataSourceEnum.externalFile) {
return DatasetSourceReadTypeEnum.externalFile;
}
return DatasetSourceReadTypeEnum.link;
};

View File

@@ -10,6 +10,7 @@ import {
} from './constants';
import { DatasetPermission } from '../../support/permission/dataset/controller';
import { Permission } from '../../support/permission/controller';
import { APIFileServer } from './apiDataset';
export type DatasetSchemaType = {
_id: string;
@@ -30,10 +31,11 @@ export type DatasetSchemaType = {
url: string;
selector: string;
};
externalReadUrl?: string;
inheritPermission: boolean;
apiServer?: APIFileServer;
// abandon
externalReadUrl?: string;
defaultPermission?: number;
};
@@ -64,6 +66,7 @@ export type DatasetCollectionSchemaType = {
rawTextLength?: number;
hashRawText?: string;
externalFileUrl?: string; // external import url
apiFileId?: string; // api file id
metadata?: {
webPageSelector?: string;
relatedImgId?: string; // The id of the associated image collections

View File

@@ -39,6 +39,7 @@ export type PluginTemplateType = PluginRuntimeType & {
};
export type PluginRuntimeType = {
id: string;
teamId?: string;
name: string;
avatar: string;
@@ -46,4 +47,6 @@ export type PluginRuntimeType = {
isTool?: boolean;
nodes: StoreNodeItemType[];
edges: StoreEdgeItemType[];
currentCost?: number;
hasTokenFee?: boolean;
};

View File

@@ -82,17 +82,6 @@ export type RuntimeNodeItemType = {
version: string;
};
export type PluginRuntimeType = {
id: string;
teamId?: string;
name: string;
avatar: string;
showStatus?: boolean;
currentCost?: number;
nodes: StoreNodeItemType[];
edges: StoreEdgeItemType[];
};
export type RuntimeEdgeItemType = StoreEdgeItemType & {
status: 'waiting' | 'active' | 'skipped';
};

View File

@@ -283,68 +283,47 @@ export const getReferenceVariableValue = ({
export function replaceEditorVariable({
text,
nodes,
variables,
runningNode
variables
}: {
text: any;
nodes: RuntimeNodeItemType[];
variables: Record<string, any>; // global variables
runningNode: RuntimeNodeItemType;
}) {
if (typeof text !== 'string') return text;
const globalVariables = Object.keys(variables).map((key) => {
return {
nodeId: VARIABLE_NODE_ID,
id: key,
value: variables[key]
};
});
const variablePattern = /\{\{\$([^.]+)\.([^$]+)\$\}\}/g;
const matches = [...text.matchAll(variablePattern)];
if (matches.length === 0) return text;
// Upstream node outputs
const nodeVariables = nodes
.map((node) => {
return node.outputs.map((output) => {
return {
nodeId: node.nodeId,
id: output.id,
value: output.value
};
});
})
.flat();
matches.forEach((match) => {
const nodeId = match[1];
const id = match[2];
// Get runningNode inputs(Will be replaced with reference)
const customInputs = runningNode.inputs.flatMap((item) => {
return [
{
id: item.key,
value: getReferenceVariableValue({
value: item.value,
nodes,
variables
}),
nodeId: runningNode.nodeId
const variableVal = (() => {
if (nodeId === VARIABLE_NODE_ID) {
return variables[id];
}
];
});
// Find upstream node input/output
const node = nodes.find((node) => node.nodeId === nodeId);
if (!node) return;
const allVariables = [...globalVariables, ...nodeVariables, ...customInputs];
const output = node.outputs.find((output) => output.id === id);
if (output) return output.value;
// Replace {{$xxx.xxx$}} to value
for (const key in allVariables) {
const variable = allVariables[key];
const val = variable.value;
const formatVal = (() => {
if (val === undefined) return '';
if (val === null) return 'null';
return typeof val === 'object' ? JSON.stringify(val) : String(val);
const input = node.inputs.find((input) => input.key === id);
if (input) return getReferenceVariableValue({ value: input.value, nodes, variables });
})();
const regex = new RegExp(`\\{\\{\\$(${variable.nodeId}\\.${variable.id})\\$\\}\\}`, 'g');
const formatVal = (() => {
if (variableVal === undefined) return 'undefined';
if (variableVal === null) return 'null';
return typeof variableVal === 'object' ? JSON.stringify(variableVal) : String(variableVal);
})();
const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g');
text = text.replace(regex, formatVal);
}
});
return text || '';
}

View File

@@ -63,15 +63,20 @@ export type TemplateMarketListItemType = {
// system plugin
export type SystemPluginTemplateItemType = WorkflowTemplateType & {
customWorkflow?: string;
associatedPluginId?: string;
userGuide?: string;
templateType: FlowNodeTemplateTypeEnum;
templateType: string;
isTool?: boolean;
// commercial plugin config
originCost: number; // n points/one time
currentCost: number;
hasTokenFee: boolean;
pluginOrder: number;
isActive?: boolean;
isOfficial?: boolean;
inputConfig?: {
// Render config input form. Find the corresponding node and replace the variable directly
key: string;

View File

@@ -54,7 +54,7 @@ type HandleType = {
// system template
export type FlowNodeTemplateType = FlowNodeCommonType & {
id: string; // node id, unique
templateType: FlowNodeTemplateTypeEnum;
templateType: string;
// show handle
sourceHandle?: HandleType;
@@ -76,7 +76,7 @@ export type NodeTemplateListItemType = {
flowNodeType: FlowNodeTypeEnum; // render node card
parentId?: ParentIdType;
isFolder?: boolean;
templateType: FlowNodeTemplateTypeEnum;
templateType: string;
avatar?: string;
name: string;
intro?: string; // template list intro
@@ -85,10 +85,12 @@ export type NodeTemplateListItemType = {
author?: string;
unique?: boolean; // 唯一的
currentCost?: number; // 当前积分消耗
hasTokenFee?: boolean; // 是否配置积分
instructions?: string; // 使用说明
};
export type NodeTemplateListType = {
type: FlowNodeTemplateTypeEnum;
type: string;
label: string;
list: NodeTemplateListItemType[];
}[];