4.6.3-website dataset (#532)

This commit is contained in:
Archer
2023-12-03 20:45:57 +08:00
committed by GitHub
parent b916183848
commit a9ae270335
122 changed files with 3793 additions and 1360 deletions

View File

@@ -5,7 +5,6 @@ export enum ChatRoleEnum {
Function = 'Function',
Tool = 'Tool'
}
export const ChatRoleMap = {
[ChatRoleEnum.System]: {
name: '系统提示词'
@@ -30,7 +29,6 @@ export enum ChatSourceEnum {
share = 'share',
api = 'api'
}
export const ChatSourceMap = {
[ChatSourceEnum.test]: {
name: 'chat.logs.test'

View File

@@ -1,8 +1,32 @@
import { DatasetDataIndexItemType } from './type';
import { DatasetDataIndexItemType, DatasetSchemaType } from './type';
import { DatasetCollectionTrainingModeEnum, DatasetCollectionTypeEnum } from './constant';
import type { LLMModelItemType } from '../ai/model.d';
/* ================= dataset ===================== */
export type DatasetUpdateBody = {
id: string;
parentId?: string;
tags?: string[];
name?: string;
avatar?: string;
permission?: DatasetSchemaType['permission'];
agentModel?: LLMModelItemType;
websiteConfig?: DatasetSchemaType['websiteConfig'];
status?: DatasetSchemaType['status'];
};
/* ================= collection ===================== */
export type CreateDatasetCollectionParams = {
datasetId: string;
parentId?: string;
name: string;
type: `${DatasetCollectionTypeEnum}`;
trainingType?: `${DatasetCollectionTrainingModeEnum}`;
chunkSize?: number;
fileId?: string;
rawLink?: string;
metadata?: Record<string, any>;
};
/* ================= data ===================== */
export type PgSearchRawType = {
@@ -18,3 +42,8 @@ export type PushDatasetDataChunkProps = {
a?: string; // bonus content
indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
};
export type PostWebsiteSyncParams = {
datasetId: string;
billId: string;
};

View File

@@ -3,15 +3,37 @@ export const PgDatasetTableName = 'modeldata';
/* ------------ dataset -------------- */
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset'
dataset = 'dataset',
websiteDataset = 'websiteDataset' // depp link
}
export const DatasetTypeMap = {
[DatasetTypeEnum.folder]: {
name: 'folder'
icon: 'core/dataset/folderDataset',
label: 'core.dataset.Folder Dataset',
collectionLabel: 'common.Folder'
},
[DatasetTypeEnum.dataset]: {
name: 'dataset'
icon: 'core/dataset/commonDataset',
label: 'core.dataset.Common Dataset',
collectionLabel: 'common.File'
},
[DatasetTypeEnum.websiteDataset]: {
icon: 'core/dataset/websiteDataset',
label: 'core.dataset.Website Dataset',
collectionLabel: 'common.Website'
}
};
export enum DatasetStatusEnum {
active = 'active',
syncing = 'syncing'
}
export const DatasetStatusMap = {
[DatasetStatusEnum.active]: {
label: 'core.dataset.status.active'
},
[DatasetStatusEnum.syncing]: {
label: 'core.dataset.status.syncing'
}
};
@@ -19,7 +41,7 @@ export const DatasetTypeMap = {
export enum DatasetCollectionTypeEnum {
folder = 'folder',
file = 'file',
link = 'link',
link = 'link', // one link
virtual = 'virtual'
}
export const DatasetCollectionTypeMap = {

View File

@@ -4,6 +4,7 @@ import { PushDatasetDataChunkProps } from './api';
import {
DatasetCollectionTypeEnum,
DatasetDataIndexTypeEnum,
DatasetStatusEnum,
DatasetTypeEnum,
TrainingModeEnum
} from './constant';
@@ -20,9 +21,14 @@ export type DatasetSchemaType = {
name: string;
vectorModel: string;
agentModel: string;
tags: string[];
intro: string;
type: `${DatasetTypeEnum}`;
status: `${DatasetStatusEnum}`;
permission: `${PermissionTypeEnum}`;
websiteConfig?: {
url: string;
selector: string;
};
};
export type DatasetCollectionSchemaType = {
@@ -39,6 +45,7 @@ export type DatasetCollectionSchemaType = {
chunkSize: number;
fileId?: string;
rawLink?: string;
metadata?: Record<string, any>;
};
export type DatasetDataIndexItemType = {
@@ -91,6 +98,18 @@ export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collect
};
/* ================= dataset ===================== */
export type DatasetListItemType = {
_id: string;
parentId: string;
avatar: string;
name: string;
intro: string;
type: `${DatasetTypeEnum}`;
isOwner: boolean;
canWrite: boolean;
permission: `${PermissionTypeEnum}`;
vectorModel: VectorModelItemType;
};
export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
vectorModel: VectorModelItemType;
agentModel: LLMModelItemType;