External dataset (#1497)

* perf: read rawText and chunk code

* perf: read raw text

* perf: read rawtext

* perf: token count

* log

projects/app/src/service/common/system/cronTask.ts

@@ -123,7 +123,9 @@ export async function checkInvalidDatasetData(start: Date, end: Date) {
         continue;
       }
     } catch (error) {}
-    console.log(++index);
+    if (++index % 100 === 0) {
+      console.log(index);
+    }
   }
 }