mirror of
https://github.com/labring/FastGPT.git
synced 2025-07-24 13:53:50 +00:00
External dataset (#1519)
* perf: local file create collection * rename middleware * perf: remove code * feat: next14 * feat: external file dataset * collection tags field * external file dataset doc * fix: ts
This commit is contained in:
16
packages/global/core/dataset/api.d.ts
vendored
16
packages/global/core/dataset/api.d.ts
vendored
@@ -26,18 +26,27 @@ export type DatasetCollectionChunkMetadataType = {
|
||||
qaPrompt?: string;
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
|
||||
// create collection params
|
||||
export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
|
||||
datasetId: string;
|
||||
name: string;
|
||||
type: `${DatasetCollectionTypeEnum}`;
|
||||
type: DatasetCollectionTypeEnum;
|
||||
|
||||
tags?: string[];
|
||||
|
||||
fileId?: string;
|
||||
rawLink?: string;
|
||||
externalFileId?: string;
|
||||
|
||||
externalFileUrl?: string;
|
||||
rawTextLength?: number;
|
||||
hashRawText?: string;
|
||||
};
|
||||
|
||||
export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
|
||||
datasetId: string;
|
||||
tags?: string[];
|
||||
};
|
||||
export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
|
||||
name: string;
|
||||
@@ -58,6 +67,11 @@ export type CsvTableCreateDatasetCollectionParams = {
|
||||
parentId?: string;
|
||||
fileId: string;
|
||||
};
|
||||
export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
|
||||
externalFileId?: string;
|
||||
externalFileUrl: string;
|
||||
filename?: string;
|
||||
};
|
||||
|
||||
/* ================= data ===================== */
|
||||
export type PgSearchRawType = {
|
||||
|
@@ -1,4 +1,4 @@
|
||||
/* sourceId = prefix-id; id=fileId;link url;externalId */
|
||||
/* sourceId = prefix-id; id=fileId;link url;externalFileId */
|
||||
export enum CollectionSourcePrefixEnum {
|
||||
local = 'local',
|
||||
link = 'link',
|
||||
|
14
packages/global/core/dataset/collection/utils.ts
Normal file
14
packages/global/core/dataset/collection/utils.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { CollectionWithDatasetType, DatasetCollectionSchemaType } from '../type';
|
||||
|
||||
export const getCollectionSourceData = (
|
||||
collection?: CollectionWithDatasetType | DatasetCollectionSchemaType
|
||||
) => {
|
||||
return {
|
||||
sourceId:
|
||||
collection?.fileId ||
|
||||
collection?.rawLink ||
|
||||
collection?.externalFileId ||
|
||||
collection?.externalFileUrl,
|
||||
sourceName: collection?.name || ''
|
||||
};
|
||||
};
|
@@ -22,7 +22,7 @@ export const DatasetTypeMap = {
|
||||
collectionLabel: 'common.Website'
|
||||
},
|
||||
[DatasetTypeEnum.externalFile]: {
|
||||
icon: 'core/dataset/commonDataset',
|
||||
icon: 'core/dataset/externalDataset',
|
||||
label: 'External File',
|
||||
collectionLabel: 'common.File'
|
||||
}
|
||||
@@ -44,9 +44,11 @@ export const DatasetStatusMap = {
|
||||
/* ------------ collection -------------- */
|
||||
export enum DatasetCollectionTypeEnum {
|
||||
folder = 'folder',
|
||||
virtual = 'virtual',
|
||||
|
||||
file = 'file',
|
||||
link = 'link', // one link
|
||||
virtual = 'virtual'
|
||||
externalFile = 'externalFile'
|
||||
}
|
||||
export const DatasetCollectionTypeMap = {
|
||||
[DatasetCollectionTypeEnum.folder]: {
|
||||
@@ -55,6 +57,9 @@ export const DatasetCollectionTypeMap = {
|
||||
[DatasetCollectionTypeEnum.file]: {
|
||||
name: 'core.dataset.file'
|
||||
},
|
||||
[DatasetCollectionTypeEnum.externalFile]: {
|
||||
name: 'core.dataset.externalFile'
|
||||
},
|
||||
[DatasetCollectionTypeEnum.link]: {
|
||||
name: 'core.dataset.link'
|
||||
},
|
||||
|
@@ -1,7 +1,5 @@
|
||||
import { DatasetSourceReadTypeEnum, ImportDataSourceEnum } from './constants';
|
||||
|
||||
export const rawTextBackupPrefix = 'index,content';
|
||||
|
||||
export const importType2ReadType = (type: ImportDataSourceEnum) => {
|
||||
if (type === ImportDataSourceEnum.csvTable || type === ImportDataSourceEnum.fileLocal) {
|
||||
return DatasetSourceReadTypeEnum.fileLocal;
|
||||
|
8
packages/global/core/dataset/type.d.ts
vendored
8
packages/global/core/dataset/type.d.ts
vendored
@@ -41,7 +41,7 @@ export type DatasetCollectionSchemaType = {
|
||||
datasetId: string;
|
||||
parentId?: string;
|
||||
name: string;
|
||||
type: `${DatasetCollectionTypeEnum}`;
|
||||
type: DatasetCollectionTypeEnum;
|
||||
createTime: Date;
|
||||
updateTime: Date;
|
||||
|
||||
@@ -50,13 +50,15 @@ export type DatasetCollectionSchemaType = {
|
||||
chunkSplitter?: string;
|
||||
qaPrompt?: string;
|
||||
|
||||
sourceId?: string; // relate CollectionSourcePrefixEnum
|
||||
tags?: string[];
|
||||
|
||||
fileId?: string; // local file id
|
||||
rawLink?: string; // link url
|
||||
externalFileId?: string; //external file id
|
||||
|
||||
rawTextLength?: number;
|
||||
hashRawText?: string;
|
||||
externalSourceUrl?: string; // external import url
|
||||
externalFileUrl?: string; // external import url
|
||||
metadata?: {
|
||||
webPageSelector?: string;
|
||||
relatedImgId?: string; // The id of the associated image collections
|
||||
|
@@ -3,7 +3,7 @@ import { getFileIcon } from '../../common/file/icon';
|
||||
import { strIsLink } from '../../common/string/tools';
|
||||
|
||||
export function getCollectionIcon(
|
||||
type: `${DatasetCollectionTypeEnum}` = DatasetCollectionTypeEnum.file,
|
||||
type: DatasetCollectionTypeEnum = DatasetCollectionTypeEnum.file,
|
||||
name = ''
|
||||
) {
|
||||
if (type === DatasetCollectionTypeEnum.folder) {
|
||||
@@ -24,13 +24,13 @@ export function getSourceNameIcon({
|
||||
sourceName: string;
|
||||
sourceId?: string;
|
||||
}) {
|
||||
if (strIsLink(sourceId)) {
|
||||
return 'common/linkBlue';
|
||||
}
|
||||
const fileIcon = getFileIcon(sourceName, '');
|
||||
const fileIcon = getFileIcon(decodeURIComponent(sourceName), '');
|
||||
if (fileIcon) {
|
||||
return fileIcon;
|
||||
}
|
||||
if (strIsLink(sourceId)) {
|
||||
return 'common/linkBlue';
|
||||
}
|
||||
|
||||
return 'file/fill/manual';
|
||||
}
|
||||
|
@@ -10,7 +10,7 @@
|
||||
"js-yaml": "^4.1.0",
|
||||
"jschardet": "3.1.1",
|
||||
"nanoid": "^4.0.1",
|
||||
"next": "13.5.2",
|
||||
"next": "14.2.3",
|
||||
"openai": "4.28.0",
|
||||
"openapi-types": "^12.1.3",
|
||||
"timezones-list": "^3.0.2"
|
||||
|
Reference in New Issue
Block a user