External dataset (#1519)

* perf: local file create collection

* rename middleware

* perf: remove code

* feat: next14

* feat: external file dataset

* collection tags field

* external file dataset doc

* fix: ts

packages/global/core/dataset/api.d.ts

@@ -26,18 +26,27 @@ export type DatasetCollectionChunkMetadataType = {
   qaPrompt?: string;
   metadata?: Record<string, any>;
 };
+
+// create collection params
 export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
   datasetId: string;
   name: string;
-  type: `${DatasetCollectionTypeEnum}`;
+  type: DatasetCollectionTypeEnum;
+
+  tags?: string[];
+
   fileId?: string;
   rawLink?: string;
+  externalFileId?: string;
+
+  externalFileUrl?: string;
   rawTextLength?: number;
   hashRawText?: string;
 };
 
 export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
   datasetId: string;
+  tags?: string[];
 };
 export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
   name: string;
@@ -58,6 +67,11 @@ export type CsvTableCreateDatasetCollectionParams = {
   parentId?: string;
   fileId: string;
 };
+export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
+  externalFileId?: string;
+  externalFileUrl: string;
+  filename?: string;
+};
 
 /* ================= data ===================== */
 export type PgSearchRawType = {

packages/global/core/dataset/collection/constants.ts

@@ -1,4 +1,4 @@
-/* sourceId = prefix-id; id=fileId;link url;externalId */
+/* sourceId = prefix-id; id=fileId;link url;externalFileId */
 export enum CollectionSourcePrefixEnum {
   local = 'local',
   link = 'link',

packages/global/core/dataset/collection/utils.ts

@@ -0,0 +1,14 @@
+import { CollectionWithDatasetType, DatasetCollectionSchemaType } from '../type';
+
+export const getCollectionSourceData = (
+  collection?: CollectionWithDatasetType | DatasetCollectionSchemaType
+) => {
+  return {
+    sourceId:
+      collection?.fileId ||
+      collection?.rawLink ||
+      collection?.externalFileId ||
+      collection?.externalFileUrl,
+    sourceName: collection?.name || ''
+  };
+};

packages/global/core/dataset/constants.ts

@@ -22,7 +22,7 @@ export const DatasetTypeMap = {
     collectionLabel: 'common.Website'
   },
   [DatasetTypeEnum.externalFile]: {
-    icon: 'core/dataset/commonDataset',
+    icon: 'core/dataset/externalDataset',
     label: 'External File',
     collectionLabel: 'common.File'
   }
@@ -44,9 +44,11 @@ export const DatasetStatusMap = {
 /* ------------ collection -------------- */
 export enum DatasetCollectionTypeEnum {
   folder = 'folder',
+  virtual = 'virtual',
+
   file = 'file',
   link = 'link', // one link
-  virtual = 'virtual'
+  externalFile = 'externalFile'
 }
 export const DatasetCollectionTypeMap = {
   [DatasetCollectionTypeEnum.folder]: {
@@ -55,6 +57,9 @@ export const DatasetCollectionTypeMap = {
   [DatasetCollectionTypeEnum.file]: {
     name: 'core.dataset.file'
   },
+  [DatasetCollectionTypeEnum.externalFile]: {
+    name: 'core.dataset.externalFile'
+  },
   [DatasetCollectionTypeEnum.link]: {
     name: 'core.dataset.link'
   },

packages/global/core/dataset/read.ts

@@ -1,7 +1,5 @@
 import { DatasetSourceReadTypeEnum, ImportDataSourceEnum } from './constants';
 
-export const rawTextBackupPrefix = 'index,content';
-
 export const importType2ReadType = (type: ImportDataSourceEnum) => {
   if (type === ImportDataSourceEnum.csvTable || type === ImportDataSourceEnum.fileLocal) {
     return DatasetSourceReadTypeEnum.fileLocal;

packages/global/core/dataset/type.d.ts

@@ -41,7 +41,7 @@ export type DatasetCollectionSchemaType = {
   datasetId: string;
   parentId?: string;
   name: string;
-  type: `${DatasetCollectionTypeEnum}`;
+  type: DatasetCollectionTypeEnum;
   createTime: Date;
   updateTime: Date;
 
@@ -50,13 +50,15 @@ export type DatasetCollectionSchemaType = {
   chunkSplitter?: string;
   qaPrompt?: string;
 
-  sourceId?: string; // relate CollectionSourcePrefixEnum
+  tags?: string[];
+
   fileId?: string; // local file id
   rawLink?: string; // link url
+  externalFileId?: string; //external file id
 
   rawTextLength?: number;
   hashRawText?: string;
-  externalSourceUrl?: string; // external import url
+  externalFileUrl?: string; // external import url
   metadata?: {
     webPageSelector?: string;
     relatedImgId?: string; // The id of the associated image collections

packages/global/core/dataset/utils.ts

@@ -3,7 +3,7 @@ import { getFileIcon } from '../../common/file/icon';
 import { strIsLink } from '../../common/string/tools';
 
 export function getCollectionIcon(
-  type: `${DatasetCollectionTypeEnum}` = DatasetCollectionTypeEnum.file,
+  type: DatasetCollectionTypeEnum = DatasetCollectionTypeEnum.file,
   name = ''
 ) {
   if (type === DatasetCollectionTypeEnum.folder) {
@@ -24,13 +24,13 @@ export function getSourceNameIcon({
   sourceName: string;
   sourceId?: string;
 }) {
-  if (strIsLink(sourceId)) {
-    return 'common/linkBlue';
-  }
-  const fileIcon = getFileIcon(sourceName, '');
+  const fileIcon = getFileIcon(decodeURIComponent(sourceName), '');
   if (fileIcon) {
     return fileIcon;
   }
+  if (strIsLink(sourceId)) {
+    return 'common/linkBlue';
+  }
 
   return 'file/fill/manual';
 }

packages/global/package.json

@@ -10,7 +10,7 @@
     "js-yaml": "^4.1.0",
     "jschardet": "3.1.1",
     "nanoid": "^4.0.1",
-    "next": "13.5.2",
+    "next": "14.2.3",
     "openai": "4.28.0",
     "openapi-types": "^12.1.3",
     "timezones-list": "^3.0.2"