External dataset (#1485)

* fix: revert version

* feat: external collection

* import context

* external ui

* doc

* fix: ts

* clear invalid data

* feat: rename sub name

* fix: node if else edge remove

* fix: init

* api size

* fix: if else node refresh

packages/service/core/dataset/collection/schema.ts

@@ -16,11 +16,6 @@ const DatasetCollectionSchema = new Schema({
     ref: DatasetColCollectionName,
     default: null
   },
-  userId: {
-    // abandoned
-    type: Schema.Types.ObjectId,
-    ref: 'user'
-  },
   teamId: {
     type: Schema.Types.ObjectId,
     ref: TeamCollectionName,
@@ -54,6 +49,7 @@ const DatasetCollectionSchema = new Schema({
     default: () => new Date()
   },
 
+  // chunk filed
   trainingType: {
     type: String,
     enum: Object.keys(TrainingTypeMap),
@@ -70,20 +66,21 @@ const DatasetCollectionSchema = new Schema({
     type: String
   },
 
+  sourceId: String,
+  // local file collection
   fileId: {
     type: Schema.Types.ObjectId,
     ref: 'dataset.files'
   },
-  rawLink: {
-    type: String
-  },
+  // web link collection
+  rawLink: String,
 
-  rawTextLength: {
-    type: Number
-  },
-  hashRawText: {
-    type: String
-  },
+  // external collection
+
+  // metadata
+  rawTextLength: Number,
+  hashRawText: String,
+  externalSourceUrl: String, // external import url
   metadata: {
     type: Object,
     default: {}

packages/service/core/dataset/schema.ts

@@ -89,7 +89,8 @@ const DatasetSchema = new Schema({
         default: 'body'
       }
     }
-  }
+  },
+  externalReadUrl: String
 });
 
 try {